1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.http;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.UnknownHostException;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Set;
40 import java.util.regex.Matcher;
41 import java.util.regex.Pattern;
42
43 import org.apache.http.Header;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpHeaders;
47 import org.apache.http.HttpHost;
48 import org.apache.http.HttpResponse;
49 import org.apache.http.HttpStatus;
50 import org.apache.http.auth.AuthSchemeProvider;
51 import org.apache.http.auth.AuthScope;
52 import org.apache.http.client.CredentialsProvider;
53 import org.apache.http.client.HttpRequestRetryHandler;
54 import org.apache.http.client.HttpResponseException;
55 import org.apache.http.client.ServiceUnavailableRetryStrategy;
56 import org.apache.http.client.config.AuthSchemes;
57 import org.apache.http.client.config.CookieSpecs;
58 import org.apache.http.client.config.RequestConfig;
59 import org.apache.http.client.methods.CloseableHttpResponse;
60 import org.apache.http.client.methods.HttpGet;
61 import org.apache.http.client.methods.HttpHead;
62 import org.apache.http.client.methods.HttpOptions;
63 import org.apache.http.client.methods.HttpPut;
64 import org.apache.http.client.methods.HttpUriRequest;
65 import org.apache.http.client.utils.DateUtils;
66 import org.apache.http.client.utils.URIUtils;
67 import org.apache.http.config.Registry;
68 import org.apache.http.config.RegistryBuilder;
69 import org.apache.http.config.SocketConfig;
70 import org.apache.http.entity.AbstractHttpEntity;
71 import org.apache.http.entity.ByteArrayEntity;
72 import org.apache.http.impl.NoConnectionReuseStrategy;
73 import org.apache.http.impl.auth.BasicScheme;
74 import org.apache.http.impl.auth.BasicSchemeFactory;
75 import org.apache.http.impl.auth.DigestSchemeFactory;
76 import org.apache.http.impl.auth.KerberosSchemeFactory;
77 import org.apache.http.impl.auth.NTLMSchemeFactory;
78 import org.apache.http.impl.auth.SPNegoSchemeFactory;
79 import org.apache.http.impl.client.CloseableHttpClient;
80 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
81 import org.apache.http.impl.client.HttpClientBuilder;
82 import org.apache.http.impl.client.LaxRedirectStrategy;
83 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
84 import org.apache.http.protocol.HttpContext;
85 import org.apache.http.util.EntityUtils;
86 import org.eclipse.aether.ConfigurationProperties;
87 import org.eclipse.aether.RepositorySystemSession;
88 import org.eclipse.aether.repository.AuthenticationContext;
89 import org.eclipse.aether.repository.Proxy;
90 import org.eclipse.aether.repository.RemoteRepository;
91 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
92 import org.eclipse.aether.spi.connector.transport.GetTask;
93 import org.eclipse.aether.spi.connector.transport.PeekTask;
94 import org.eclipse.aether.spi.connector.transport.PutTask;
95 import org.eclipse.aether.spi.connector.transport.TransportTask;
96 import org.eclipse.aether.transfer.NoTransporterException;
97 import org.eclipse.aether.transfer.TransferCancelledException;
98 import org.eclipse.aether.util.ConfigUtils;
99 import org.eclipse.aether.util.FileUtils;
100 import org.slf4j.Logger;
101 import org.slf4j.LoggerFactory;
102
103 import static java.util.Objects.requireNonNull;
104
105
106
107
108 final class HttpTransporter extends AbstractTransporter {
109
110 static final String BIND_ADDRESS = "aether.connector.bind.address";
111
112 static final String SUPPORT_WEBDAV = "aether.connector.http.supportWebDav";
113
114 static final String PREEMPTIVE_PUT_AUTH = "aether.connector.http.preemptivePutAuth";
115
116 static final String USE_SYSTEM_PROPERTIES = "aether.connector.http.useSystemProperties";
117
118 static final String HTTP_RETRY_HANDLER_NAME = "aether.connector.http.retryHandler.name";
119
120 private static final String HTTP_RETRY_HANDLER_NAME_STANDARD = "standard";
121
122 private static final String HTTP_RETRY_HANDLER_NAME_DEFAULT = "default";
123
124 static final String HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED =
125 "aether.connector.http.retryHandler.requestSentEnabled";
126
127 private static final Pattern CONTENT_RANGE_PATTERN =
128 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
129
130 private static final Logger LOGGER = LoggerFactory.getLogger(HttpTransporter.class);
131
132 private final Map<String, ChecksumExtractor> checksumExtractors;
133
134 private final AuthenticationContext repoAuthContext;
135
136 private final AuthenticationContext proxyAuthContext;
137
138 private final URI baseUri;
139
140 private final HttpHost server;
141
142 private final HttpHost proxy;
143
144 private final CloseableHttpClient client;
145
146 private final Map<?, ?> headers;
147
148 private final LocalState state;
149
150 private final boolean preemptiveAuth;
151
152 private final boolean preemptivePutAuth;
153
154 private final boolean supportWebDav;
155
156 @SuppressWarnings("checkstyle:methodlength")
157 HttpTransporter(
158 Map<String, ChecksumExtractor> checksumExtractors,
159 RemoteRepository repository,
160 RepositorySystemSession session)
161 throws NoTransporterException {
162 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
163 throw new NoTransporterException(repository);
164 }
165 this.checksumExtractors = requireNonNull(checksumExtractors, "checksum extractors must not be null");
166 try {
167 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
168 if (baseUri.isOpaque()) {
169 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
170 }
171 this.server = URIUtils.extractHost(baseUri);
172 if (server == null) {
173 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
174 }
175 } catch (URISyntaxException e) {
176 throw new NoTransporterException(repository, e.getMessage(), e);
177 }
178 this.proxy = toHost(repository.getProxy());
179
180 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
181 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
182
183 String httpsSecurityMode = ConfigUtils.getString(
184 session,
185 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
186 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
187 ConfigurationProperties.HTTPS_SECURITY_MODE);
188 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
189 session,
190 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
191 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
192 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
193 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
194 session,
195 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
196 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
197 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
198 this.state = new LocalState(
199 session,
200 repository,
201 new ConnMgrConfig(
202 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
203
204 this.headers = ConfigUtils.getMap(
205 session,
206 Collections.emptyMap(),
207 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
208 ConfigurationProperties.HTTP_HEADERS);
209
210 this.preemptiveAuth = ConfigUtils.getBoolean(
211 session,
212 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
213 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
214 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
215 this.preemptivePutAuth =
216 ConfigUtils.getBoolean(
217 session, true, PREEMPTIVE_PUT_AUTH + "." + repository.getId(), PREEMPTIVE_PUT_AUTH);
218 this.supportWebDav =
219 ConfigUtils.getBoolean(session, false, SUPPORT_WEBDAV + "." + repository.getId(), SUPPORT_WEBDAV);
220 String credentialEncoding = ConfigUtils.getString(
221 session,
222 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
223 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
224 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
225 int connectTimeout = ConfigUtils.getInteger(
226 session,
227 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
228 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
229 ConfigurationProperties.CONNECT_TIMEOUT);
230 int requestTimeout = ConfigUtils.getInteger(
231 session,
232 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
233 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
234 ConfigurationProperties.REQUEST_TIMEOUT);
235 int retryCount = ConfigUtils.getInteger(
236 session,
237 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
238 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
239 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
240 long retryInterval = ConfigUtils.getLong(
241 session,
242 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
243 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
244 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
245 long retryIntervalMax = ConfigUtils.getLong(
246 session,
247 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
248 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
249 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
250 String serviceUnavailableCodesString = ConfigUtils.getString(
251 session,
252 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
253 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
254 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
255 String retryHandlerName = ConfigUtils.getString(
256 session,
257 HTTP_RETRY_HANDLER_NAME_STANDARD,
258 HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
259 HTTP_RETRY_HANDLER_NAME);
260 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
261 session,
262 false,
263 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
264 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
265 String userAgent = ConfigUtils.getString(
266 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
267 int maxRedirects = ConfigUtils.getInteger(
268 session,
269 ConfigurationProperties.DEFAULT_HTTP_MAX_REDIRECTS,
270 ConfigurationProperties.HTTP_MAX_REDIRECTS + "." + repository.getId(),
271 ConfigurationProperties.HTTP_MAX_REDIRECTS);
272 boolean followRedirects = ConfigUtils.getBoolean(
273 session,
274 ConfigurationProperties.DEFAULT_FOLLOW_REDIRECTS,
275 ConfigurationProperties.HTTP_FOLLOW_REDIRECTS + "." + repository.getId(),
276 ConfigurationProperties.HTTP_FOLLOW_REDIRECTS);
277
278 Charset credentialsCharset = Charset.forName(credentialEncoding);
279 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
280 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
281 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
282 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
283 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
284 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
285 .build();
286 SocketConfig socketConfig =
287 SocketConfig.custom().setSoTimeout(requestTimeout).build();
288 RequestConfig requestConfig = RequestConfig.custom()
289 .setMaxRedirects(maxRedirects)
290 .setRedirectsEnabled(followRedirects)
291 .setConnectTimeout(connectTimeout)
292 .setConnectionRequestTimeout(connectTimeout)
293 .setLocalAddress(getBindAddress(session, repository))
294 .setCookieSpec(CookieSpecs.STANDARD)
295 .setSocketTimeout(requestTimeout)
296 .build();
297
298 HttpRequestRetryHandler retryHandler;
299 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
300 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
301 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
302 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
303 } else {
304 throw new IllegalArgumentException(
305 "Unsupported parameter " + HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
306 }
307 Set<Integer> serviceUnavailableCodes = new HashSet<>();
308 try {
309 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
310 serviceUnavailableCodes.add(Integer.parseInt(code));
311 }
312 } catch (NumberFormatException e) {
313 throw new IllegalArgumentException(
314 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
315 + " (list of integers): " + serviceUnavailableCodesString);
316 }
317 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
318 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
319
320 HttpClientBuilder builder = HttpClientBuilder.create()
321 .setUserAgent(userAgent)
322 .setRedirectStrategy(LaxRedirectStrategy.INSTANCE)
323 .setDefaultSocketConfig(socketConfig)
324 .setDefaultRequestConfig(requestConfig)
325 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
326 .setRetryHandler(retryHandler)
327 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
328 .setConnectionManager(state.getConnectionManager())
329 .setConnectionManagerShared(true)
330 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
331 .setProxy(proxy);
332 final boolean useSystemProperties = ConfigUtils.getBoolean(
333 session, false, USE_SYSTEM_PROPERTIES + "." + repository.getId(), USE_SYSTEM_PROPERTIES);
334 if (useSystemProperties) {
335 LOGGER.warn(
336 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
337 LOGGER.warn("Please use documented means to configure resolver transport.");
338 builder.useSystemProperties();
339 }
340
341 final String expectContinue = ConfigUtils.getString(
342 session,
343 null,
344 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
345 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
346 if (expectContinue != null) {
347 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
348 }
349
350 final boolean reuseConnections = ConfigUtils.getBoolean(
351 session,
352 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
353 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
354 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
355 if (!reuseConnections) {
356 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
357 }
358
359 this.client = builder.build();
360 }
361
362
363
364
365 private InetAddress getBindAddress(RepositorySystemSession session, RemoteRepository repository) {
366 String bindAddress =
367 ConfigUtils.getString(session, null, BIND_ADDRESS + "." + repository.getId(), BIND_ADDRESS);
368 if (bindAddress == null) {
369 return null;
370 }
371 try {
372 return InetAddress.getByName(bindAddress);
373 } catch (UnknownHostException uhe) {
374 throw new IllegalArgumentException(
375 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
376 uhe);
377 }
378 }
379
380 private static HttpHost toHost(Proxy proxy) {
381 HttpHost host = null;
382 if (proxy != null) {
383 host = new HttpHost(proxy.getHost(), proxy.getPort());
384 }
385 return host;
386 }
387
388 private static CredentialsProvider toCredentialsProvider(
389 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
390 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
391 if (proxy != null) {
392 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
393 provider = new DemuxCredentialsProvider(provider, p, proxy);
394 }
395 return provider;
396 }
397
398 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
399 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
400 if (ctx != null) {
401 AuthScope basicScope = new AuthScope(host, port);
402 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
403
404 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
405 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
406 }
407 return provider;
408 }
409
410 LocalState getState() {
411 return state;
412 }
413
414 private URI resolve(TransportTask task) {
415 return UriUtils.resolve(baseUri, task.getLocation());
416 }
417
418 @Override
419 public int classify(Throwable error) {
420 if (error instanceof HttpResponseException
421 && ((HttpResponseException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
422 return ERROR_NOT_FOUND;
423 }
424 return ERROR_OTHER;
425 }
426
427 @Override
428 protected void implPeek(PeekTask task) throws Exception {
429 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
430 execute(request, null);
431 }
432
433 @Override
434 protected void implGet(GetTask task) throws Exception {
435 boolean resume = true;
436 boolean applyChecksumExtractors = true;
437
438 EntityGetter getter = new EntityGetter(task);
439 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
440 while (true) {
441 try {
442 if (resume) {
443 resume(request, task);
444 }
445 if (applyChecksumExtractors) {
446 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
447 checksumExtractor.prepareRequest(request);
448 }
449 }
450 execute(request, getter);
451 break;
452 } catch (HttpResponseException e) {
453 if (resume
454 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
455 && request.containsHeader(HttpHeaders.RANGE)) {
456 request = commonHeaders(new HttpGet(resolve(task)));
457 resume = false;
458 continue;
459 }
460 if (applyChecksumExtractors) {
461 boolean retryWithoutExtractors = false;
462 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
463 if (checksumExtractor.retryWithoutExtractor(e)) {
464 retryWithoutExtractors = true;
465 break;
466 }
467 }
468 if (retryWithoutExtractors) {
469 request = commonHeaders(new HttpGet(resolve(task)));
470 applyChecksumExtractors = false;
471 continue;
472 }
473 }
474 throw e;
475 }
476 }
477 }
478
479 @Override
480 protected void implPut(PutTask task) throws Exception {
481 PutTaskEntity entity = new PutTaskEntity(task);
482 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
483 try {
484 execute(request, null);
485 } catch (HttpResponseException e) {
486 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
487 state.setExpectContinue(false);
488 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
489 execute(request, null);
490 return;
491 }
492 throw e;
493 }
494 }
495
496 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
497 try {
498 SharingHttpContext context = new SharingHttpContext(state);
499 prepare(request, context);
500 try (CloseableHttpResponse response = client.execute(server, request, context)) {
501 try {
502 context.close();
503 handleStatus(response);
504 if (getter != null) {
505 getter.handle(response);
506 }
507 } finally {
508 EntityUtils.consumeQuietly(response.getEntity());
509 }
510 }
511 } catch (IOException e) {
512 if (e.getCause() instanceof TransferCancelledException) {
513 throw (Exception) e.getCause();
514 }
515 throw e;
516 }
517 }
518
519 private void prepare(HttpUriRequest request, SharingHttpContext context) {
520 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
521 if (preemptiveAuth || (preemptivePutAuth && put)) {
522 context.getAuthCache().put(server, new BasicScheme());
523 }
524 if (supportWebDav) {
525 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
526 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
527 try (CloseableHttpResponse response = client.execute(server, req, context)) {
528 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
529 EntityUtils.consumeQuietly(response.getEntity());
530 } catch (IOException e) {
531 LOGGER.debug("Failed to prepare HTTP context", e);
532 }
533 }
534 if (put && Boolean.TRUE.equals(state.getWebDav())) {
535 mkdirs(request.getURI(), context);
536 }
537 }
538 }
539
540 private void mkdirs(URI uri, SharingHttpContext context) {
541 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
542 int index = 0;
543 for (; index < dirs.size(); index++) {
544 try (CloseableHttpResponse response =
545 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
546 try {
547 int status = response.getStatusLine().getStatusCode();
548 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
549 break;
550 } else if (status == HttpStatus.SC_CONFLICT) {
551 continue;
552 }
553 handleStatus(response);
554 } finally {
555 EntityUtils.consumeQuietly(response.getEntity());
556 }
557 } catch (IOException e) {
558 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
559 return;
560 }
561 }
562 for (index--; index >= 0; index--) {
563 try (CloseableHttpResponse response =
564 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
565 try {
566 handleStatus(response);
567 } finally {
568 EntityUtils.consumeQuietly(response.getEntity());
569 }
570 } catch (IOException e) {
571 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
572 return;
573 }
574 }
575 }
576
577 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
578 request.setEntity(entity);
579 return request;
580 }
581
582 private boolean isPayloadPresent(HttpUriRequest request) {
583 if (request instanceof HttpEntityEnclosingRequest) {
584 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
585 return entity != null && entity.getContentLength() != 0;
586 }
587 return false;
588 }
589
590 private <T extends HttpUriRequest> T commonHeaders(T request) {
591 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
592 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
593
594 if (state.isExpectContinue() && isPayloadPresent(request)) {
595 request.setHeader(HttpHeaders.EXPECT, "100-continue");
596 }
597
598 for (Map.Entry<?, ?> entry : headers.entrySet()) {
599 if (!(entry.getKey() instanceof String)) {
600 continue;
601 }
602 if (entry.getValue() instanceof String) {
603 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
604 } else {
605 request.removeHeaders(entry.getKey().toString());
606 }
607 }
608
609 if (!state.isExpectContinue()) {
610 request.removeHeaders(HttpHeaders.EXPECT);
611 }
612
613 return request;
614 }
615
616 private <T extends HttpUriRequest> void resume(T request, GetTask task) {
617 long resumeOffset = task.getResumeOffset();
618 if (resumeOffset > 0L && task.getDataFile() != null) {
619 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
620 request.setHeader(
621 HttpHeaders.IF_UNMODIFIED_SINCE,
622 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
623 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
624 }
625 }
626
627 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
628 int status = response.getStatusLine().getStatusCode();
629 if (status >= 300) {
630 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
631 }
632 }
633
634 @Override
635 protected void implClose() {
636 try {
637 client.close();
638 } catch (IOException e) {
639 throw new UncheckedIOException(e);
640 }
641 AuthenticationContext.close(repoAuthContext);
642 AuthenticationContext.close(proxyAuthContext);
643 state.close();
644 }
645
646 private class EntityGetter {
647
648 private final GetTask task;
649
650 EntityGetter(GetTask task) {
651 this.task = task;
652 }
653
654 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
655 HttpEntity entity = response.getEntity();
656 if (entity == null) {
657 entity = new ByteArrayEntity(new byte[0]);
658 }
659
660 long offset = 0L, length = entity.getContentLength();
661 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
662 String range = rangeHeader != null ? rangeHeader.getValue() : null;
663 if (range != null) {
664 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
665 if (!m.matches()) {
666 throw new IOException("Invalid Content-Range header for partial download: " + range);
667 }
668 offset = Long.parseLong(m.group(1));
669 length = Long.parseLong(m.group(2)) + 1L;
670 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
671 throw new IOException("Invalid Content-Range header for partial download from offset "
672 + task.getResumeOffset() + ": " + range);
673 }
674 }
675
676 final boolean resume = offset > 0L;
677 final File dataFile = task.getDataFile();
678 if (dataFile == null) {
679 try (InputStream is = entity.getContent()) {
680 utilGet(task, is, true, length, resume);
681 extractChecksums(response);
682 }
683 } else {
684 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
685 task.setDataFile(tempFile.getPath().toFile(), resume);
686 if (resume && Files.isRegularFile(dataFile.toPath())) {
687 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
688 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
689 }
690 }
691 try (InputStream is = entity.getContent()) {
692 utilGet(task, is, true, length, resume);
693 }
694 tempFile.move();
695 } finally {
696 task.setDataFile(dataFile);
697 }
698 }
699 if (task.getDataFile() != null) {
700 Header lastModifiedHeader =
701 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
702 if (lastModifiedHeader != null) {
703 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
704 if (lastModified != null) {
705 task.getDataFile().setLastModified(lastModified.getTime());
706 }
707 }
708 }
709 extractChecksums(response);
710 }
711
712 private void extractChecksums(CloseableHttpResponse response) {
713 for (Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet()) {
714 Map<String, String> checksums = extractorEntry.getValue().extractChecksums(response);
715 if (checksums != null) {
716 checksums.forEach(task::setChecksum);
717 return;
718 }
719 }
720 }
721 }
722
723 private class PutTaskEntity extends AbstractHttpEntity {
724
725 private final PutTask task;
726
727 PutTaskEntity(PutTask task) {
728 this.task = task;
729 }
730
731 @Override
732 public boolean isRepeatable() {
733 return true;
734 }
735
736 @Override
737 public boolean isStreaming() {
738 return false;
739 }
740
741 @Override
742 public long getContentLength() {
743 return task.getDataLength();
744 }
745
746 @Override
747 public InputStream getContent() throws IOException {
748 return task.newInputStream();
749 }
750
751 @Override
752 public void writeTo(OutputStream os) throws IOException {
753 try {
754 utilPut(task, os, false);
755 } catch (TransferCancelledException e) {
756 throw (IOException) new InterruptedIOException().initCause(e);
757 }
758 }
759 }
760
761 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
762 private final int retryCount;
763
764 private final long retryInterval;
765
766 private final long retryIntervalMax;
767
768 private final Set<Integer> serviceUnavailableHttpCodes;
769
770
771
772
773
774
775 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
776
777 private ResolverServiceUnavailableRetryStrategy(
778 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
779 if (retryCount < 0) {
780 throw new IllegalArgumentException("retryCount must be >= 0");
781 }
782 if (retryInterval < 0L) {
783 throw new IllegalArgumentException("retryInterval must be >= 0");
784 }
785 if (retryIntervalMax < 0L) {
786 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
787 }
788 this.retryCount = retryCount;
789 this.retryInterval = retryInterval;
790 this.retryIntervalMax = retryIntervalMax;
791 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
792 }
793
794 @Override
795 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
796 final boolean retry = executionCount <= retryCount
797 && (serviceUnavailableHttpCodes.contains(
798 response.getStatusLine().getStatusCode()));
799 if (retry) {
800 Long retryInterval = retryInterval(response, executionCount);
801 if (retryInterval != null) {
802 RETRY_INTERVAL_HOLDER.set(retryInterval);
803 return true;
804 }
805 }
806 RETRY_INTERVAL_HOLDER.remove();
807 return false;
808 }
809
810
811
812
813
814
815
816
817 private Long retryInterval(HttpResponse httpResponse, int executionCount) {
818 Long result = null;
819 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
820 if (header != null && header.getValue() != null) {
821 String headerValue = header.getValue();
822 if (headerValue.contains(":")) {
823 Date when = DateUtils.parseDate(headerValue);
824 if (when != null) {
825 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
826 }
827 } else {
828 try {
829 result = Long.parseLong(headerValue) * 1000L;
830 } catch (NumberFormatException e) {
831
832 }
833 }
834 }
835 if (result == null) {
836 result = executionCount * this.retryInterval;
837 }
838 if (result > retryIntervalMax) {
839 return null;
840 }
841 return result;
842 }
843
844 @Override
845 public long getRetryInterval() {
846 Long ri = RETRY_INTERVAL_HOLDER.get();
847 if (ri == null) {
848 return 0L;
849 }
850 RETRY_INTERVAL_HOLDER.remove();
851 return ri;
852 }
853 }
854 }