1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.http;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.UnknownHostException;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Set;
40 import java.util.regex.Matcher;
41 import java.util.regex.Pattern;
42
43 import org.apache.http.Header;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpHeaders;
47 import org.apache.http.HttpHost;
48 import org.apache.http.HttpResponse;
49 import org.apache.http.HttpStatus;
50 import org.apache.http.auth.AuthSchemeProvider;
51 import org.apache.http.auth.AuthScope;
52 import org.apache.http.client.CredentialsProvider;
53 import org.apache.http.client.HttpRequestRetryHandler;
54 import org.apache.http.client.HttpResponseException;
55 import org.apache.http.client.ServiceUnavailableRetryStrategy;
56 import org.apache.http.client.config.AuthSchemes;
57 import org.apache.http.client.config.RequestConfig;
58 import org.apache.http.client.methods.CloseableHttpResponse;
59 import org.apache.http.client.methods.HttpGet;
60 import org.apache.http.client.methods.HttpHead;
61 import org.apache.http.client.methods.HttpOptions;
62 import org.apache.http.client.methods.HttpPut;
63 import org.apache.http.client.methods.HttpUriRequest;
64 import org.apache.http.client.utils.DateUtils;
65 import org.apache.http.client.utils.URIUtils;
66 import org.apache.http.config.Registry;
67 import org.apache.http.config.RegistryBuilder;
68 import org.apache.http.config.SocketConfig;
69 import org.apache.http.entity.AbstractHttpEntity;
70 import org.apache.http.entity.ByteArrayEntity;
71 import org.apache.http.impl.NoConnectionReuseStrategy;
72 import org.apache.http.impl.auth.BasicScheme;
73 import org.apache.http.impl.auth.BasicSchemeFactory;
74 import org.apache.http.impl.auth.DigestSchemeFactory;
75 import org.apache.http.impl.auth.KerberosSchemeFactory;
76 import org.apache.http.impl.auth.NTLMSchemeFactory;
77 import org.apache.http.impl.auth.SPNegoSchemeFactory;
78 import org.apache.http.impl.client.CloseableHttpClient;
79 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
80 import org.apache.http.impl.client.HttpClientBuilder;
81 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
82 import org.apache.http.protocol.HttpContext;
83 import org.apache.http.util.EntityUtils;
84 import org.eclipse.aether.ConfigurationProperties;
85 import org.eclipse.aether.RepositorySystemSession;
86 import org.eclipse.aether.repository.AuthenticationContext;
87 import org.eclipse.aether.repository.Proxy;
88 import org.eclipse.aether.repository.RemoteRepository;
89 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
90 import org.eclipse.aether.spi.connector.transport.GetTask;
91 import org.eclipse.aether.spi.connector.transport.PeekTask;
92 import org.eclipse.aether.spi.connector.transport.PutTask;
93 import org.eclipse.aether.spi.connector.transport.TransportTask;
94 import org.eclipse.aether.transfer.NoTransporterException;
95 import org.eclipse.aether.transfer.TransferCancelledException;
96 import org.eclipse.aether.util.ConfigUtils;
97 import org.eclipse.aether.util.FileUtils;
98 import org.slf4j.Logger;
99 import org.slf4j.LoggerFactory;
100
101 import static java.util.Objects.requireNonNull;
102
103
104
105
106 final class HttpTransporter extends AbstractTransporter {
107
108 static final String BIND_ADDRESS = "aether.connector.bind.address";
109
110 static final String SUPPORT_WEBDAV = "aether.connector.http.supportWebDav";
111
112 static final String PREEMPTIVE_PUT_AUTH = "aether.connector.http.preemptivePutAuth";
113
114 static final String USE_SYSTEM_PROPERTIES = "aether.connector.http.useSystemProperties";
115
116 static final String HTTP_RETRY_HANDLER_NAME = "aether.connector.http.retryHandler.name";
117
118 private static final String HTTP_RETRY_HANDLER_NAME_STANDARD = "standard";
119
120 private static final String HTTP_RETRY_HANDLER_NAME_DEFAULT = "default";
121
122 static final String HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED =
123 "aether.connector.http.retryHandler.requestSentEnabled";
124
125 private static final Pattern CONTENT_RANGE_PATTERN =
126 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
127
128 private static final Logger LOGGER = LoggerFactory.getLogger(HttpTransporter.class);
129
130 private final Map<String, ChecksumExtractor> checksumExtractors;
131
132 private final AuthenticationContext repoAuthContext;
133
134 private final AuthenticationContext proxyAuthContext;
135
136 private final URI baseUri;
137
138 private final HttpHost server;
139
140 private final HttpHost proxy;
141
142 private final CloseableHttpClient client;
143
144 private final Map<?, ?> headers;
145
146 private final LocalState state;
147
148 private final boolean preemptiveAuth;
149
150 private final boolean preemptivePutAuth;
151
152 private final boolean supportWebDav;
153
154 @SuppressWarnings("checkstyle:methodlength")
155 HttpTransporter(
156 Map<String, ChecksumExtractor> checksumExtractors,
157 RemoteRepository repository,
158 RepositorySystemSession session)
159 throws NoTransporterException {
160 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
161 throw new NoTransporterException(repository);
162 }
163 this.checksumExtractors = requireNonNull(checksumExtractors, "checksum extractors must not be null");
164 try {
165 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
166 if (baseUri.isOpaque()) {
167 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
168 }
169 this.server = URIUtils.extractHost(baseUri);
170 if (server == null) {
171 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
172 }
173 } catch (URISyntaxException e) {
174 throw new NoTransporterException(repository, e.getMessage(), e);
175 }
176 this.proxy = toHost(repository.getProxy());
177
178 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
179 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
180
181 String httpsSecurityMode = ConfigUtils.getString(
182 session,
183 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
184 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
185 ConfigurationProperties.HTTPS_SECURITY_MODE);
186 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
187 session,
188 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
189 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
190 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
191 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
192 session,
193 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
194 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
195 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
196 this.state = new LocalState(
197 session,
198 repository,
199 new ConnMgrConfig(
200 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
201
202 this.headers = ConfigUtils.getMap(
203 session,
204 Collections.emptyMap(),
205 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
206 ConfigurationProperties.HTTP_HEADERS);
207
208 this.preemptiveAuth = ConfigUtils.getBoolean(
209 session,
210 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
211 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
212 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
213 this.preemptivePutAuth =
214 ConfigUtils.getBoolean(
215 session, true, PREEMPTIVE_PUT_AUTH + "." + repository.getId(), PREEMPTIVE_PUT_AUTH);
216 this.supportWebDav =
217 ConfigUtils.getBoolean(session, false, SUPPORT_WEBDAV + "." + repository.getId(), SUPPORT_WEBDAV);
218 String credentialEncoding = ConfigUtils.getString(
219 session,
220 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
221 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
222 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
223 int connectTimeout = ConfigUtils.getInteger(
224 session,
225 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
226 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
227 ConfigurationProperties.CONNECT_TIMEOUT);
228 int requestTimeout = ConfigUtils.getInteger(
229 session,
230 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
231 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
232 ConfigurationProperties.REQUEST_TIMEOUT);
233 int retryCount = ConfigUtils.getInteger(
234 session,
235 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
236 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
237 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
238 long retryInterval = ConfigUtils.getLong(
239 session,
240 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
241 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
242 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
243 long retryIntervalMax = ConfigUtils.getLong(
244 session,
245 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
246 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
247 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
248 String serviceUnavailableCodesString = ConfigUtils.getString(
249 session,
250 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
251 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
252 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
253 String retryHandlerName = ConfigUtils.getString(
254 session,
255 HTTP_RETRY_HANDLER_NAME_STANDARD,
256 HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
257 HTTP_RETRY_HANDLER_NAME);
258 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
259 session,
260 false,
261 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
262 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
263 String userAgent = ConfigUtils.getString(
264 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
265
266 Charset credentialsCharset = Charset.forName(credentialEncoding);
267 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
268 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
269 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
270 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
271 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
272 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
273 .build();
274 SocketConfig socketConfig =
275 SocketConfig.custom().setSoTimeout(requestTimeout).build();
276 RequestConfig requestConfig = RequestConfig.custom()
277 .setConnectTimeout(connectTimeout)
278 .setConnectionRequestTimeout(connectTimeout)
279 .setLocalAddress(getBindAddress(session, repository))
280 .setSocketTimeout(requestTimeout)
281 .build();
282
283 HttpRequestRetryHandler retryHandler;
284 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
285 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
286 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
287 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
288 } else {
289 throw new IllegalArgumentException(
290 "Unsupported parameter " + HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
291 }
292 Set<Integer> serviceUnavailableCodes = new HashSet<>();
293 try {
294 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
295 serviceUnavailableCodes.add(Integer.parseInt(code));
296 }
297 } catch (NumberFormatException e) {
298 throw new IllegalArgumentException(
299 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
300 + " (list of integers): " + serviceUnavailableCodesString);
301 }
302 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
303 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
304
305 HttpClientBuilder builder = HttpClientBuilder.create()
306 .setUserAgent(userAgent)
307 .setDefaultSocketConfig(socketConfig)
308 .setDefaultRequestConfig(requestConfig)
309 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
310 .setRetryHandler(retryHandler)
311 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
312 .setConnectionManager(state.getConnectionManager())
313 .setConnectionManagerShared(true)
314 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
315 .setProxy(proxy);
316 final boolean useSystemProperties = ConfigUtils.getBoolean(
317 session, false, USE_SYSTEM_PROPERTIES + "." + repository.getId(), USE_SYSTEM_PROPERTIES);
318 if (useSystemProperties) {
319 LOGGER.warn(
320 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
321 LOGGER.warn("Please use documented means to configure resolver transport.");
322 builder.useSystemProperties();
323 }
324
325 final String expectContinue = ConfigUtils.getString(
326 session,
327 null,
328 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
329 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
330 if (expectContinue != null) {
331 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
332 }
333
334 final boolean reuseConnections = ConfigUtils.getBoolean(
335 session,
336 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
337 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
338 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
339 if (!reuseConnections) {
340 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
341 }
342
343 this.client = builder.build();
344 }
345
346
347
348
349 private InetAddress getBindAddress(RepositorySystemSession session, RemoteRepository repository) {
350 String bindAddress =
351 ConfigUtils.getString(session, null, BIND_ADDRESS + "." + repository.getId(), BIND_ADDRESS);
352 if (bindAddress == null) {
353 return null;
354 }
355 try {
356 return InetAddress.getByName(bindAddress);
357 } catch (UnknownHostException uhe) {
358 throw new IllegalArgumentException(
359 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
360 uhe);
361 }
362 }
363
364 private static HttpHost toHost(Proxy proxy) {
365 HttpHost host = null;
366 if (proxy != null) {
367 host = new HttpHost(proxy.getHost(), proxy.getPort());
368 }
369 return host;
370 }
371
372 private static CredentialsProvider toCredentialsProvider(
373 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
374 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
375 if (proxy != null) {
376 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
377 provider = new DemuxCredentialsProvider(provider, p, proxy);
378 }
379 return provider;
380 }
381
382 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
383 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
384 if (ctx != null) {
385 AuthScope basicScope = new AuthScope(host, port);
386 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
387
388 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
389 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
390 }
391 return provider;
392 }
393
394 LocalState getState() {
395 return state;
396 }
397
398 private URI resolve(TransportTask task) {
399 return UriUtils.resolve(baseUri, task.getLocation());
400 }
401
402 @Override
403 public int classify(Throwable error) {
404 if (error instanceof HttpResponseException
405 && ((HttpResponseException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
406 return ERROR_NOT_FOUND;
407 }
408 return ERROR_OTHER;
409 }
410
411 @Override
412 protected void implPeek(PeekTask task) throws Exception {
413 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
414 execute(request, null);
415 }
416
417 @Override
418 protected void implGet(GetTask task) throws Exception {
419 boolean resume = true;
420 boolean applyChecksumExtractors = true;
421
422 EntityGetter getter = new EntityGetter(task);
423 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
424 while (true) {
425 try {
426 if (resume) {
427 resume(request, task);
428 }
429 if (applyChecksumExtractors) {
430 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
431 checksumExtractor.prepareRequest(request);
432 }
433 }
434 execute(request, getter);
435 break;
436 } catch (HttpResponseException e) {
437 if (resume
438 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
439 && request.containsHeader(HttpHeaders.RANGE)) {
440 request = commonHeaders(new HttpGet(resolve(task)));
441 resume = false;
442 continue;
443 }
444 if (applyChecksumExtractors) {
445 boolean retryWithoutExtractors = false;
446 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
447 if (checksumExtractor.retryWithoutExtractor(e)) {
448 retryWithoutExtractors = true;
449 break;
450 }
451 }
452 if (retryWithoutExtractors) {
453 request = commonHeaders(new HttpGet(resolve(task)));
454 applyChecksumExtractors = false;
455 continue;
456 }
457 }
458 throw e;
459 }
460 }
461 }
462
463 @Override
464 protected void implPut(PutTask task) throws Exception {
465 PutTaskEntity entity = new PutTaskEntity(task);
466 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
467 try {
468 execute(request, null);
469 } catch (HttpResponseException e) {
470 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
471 state.setExpectContinue(false);
472 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
473 execute(request, null);
474 return;
475 }
476 throw e;
477 }
478 }
479
480 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
481 try {
482 SharingHttpContext context = new SharingHttpContext(state);
483 prepare(request, context);
484 try (CloseableHttpResponse response = client.execute(server, request, context)) {
485 try {
486 context.close();
487 handleStatus(response);
488 if (getter != null) {
489 getter.handle(response);
490 }
491 } finally {
492 EntityUtils.consumeQuietly(response.getEntity());
493 }
494 }
495 } catch (IOException e) {
496 if (e.getCause() instanceof TransferCancelledException) {
497 throw (Exception) e.getCause();
498 }
499 throw e;
500 }
501 }
502
503 private void prepare(HttpUriRequest request, SharingHttpContext context) {
504 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
505 if (preemptiveAuth || (preemptivePutAuth && put)) {
506 context.getAuthCache().put(server, new BasicScheme());
507 }
508 if (supportWebDav) {
509 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
510 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
511 try (CloseableHttpResponse response = client.execute(server, req, context)) {
512 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
513 EntityUtils.consumeQuietly(response.getEntity());
514 } catch (IOException e) {
515 LOGGER.debug("Failed to prepare HTTP context", e);
516 }
517 }
518 if (put && Boolean.TRUE.equals(state.getWebDav())) {
519 mkdirs(request.getURI(), context);
520 }
521 }
522 }
523
524 @SuppressWarnings("checkstyle:magicnumber")
525 private void mkdirs(URI uri, SharingHttpContext context) {
526 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
527 int index = 0;
528 for (; index < dirs.size(); index++) {
529 try (CloseableHttpResponse response =
530 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
531 try {
532 int status = response.getStatusLine().getStatusCode();
533 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
534 break;
535 } else if (status == HttpStatus.SC_CONFLICT) {
536 continue;
537 }
538 handleStatus(response);
539 } finally {
540 EntityUtils.consumeQuietly(response.getEntity());
541 }
542 } catch (IOException e) {
543 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
544 return;
545 }
546 }
547 for (index--; index >= 0; index--) {
548 try (CloseableHttpResponse response =
549 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
550 try {
551 handleStatus(response);
552 } finally {
553 EntityUtils.consumeQuietly(response.getEntity());
554 }
555 } catch (IOException e) {
556 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
557 return;
558 }
559 }
560 }
561
562 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
563 request.setEntity(entity);
564 return request;
565 }
566
567 private boolean isPayloadPresent(HttpUriRequest request) {
568 if (request instanceof HttpEntityEnclosingRequest) {
569 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
570 return entity != null && entity.getContentLength() != 0;
571 }
572 return false;
573 }
574
575 private <T extends HttpUriRequest> T commonHeaders(T request) {
576 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
577 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
578
579 if (state.isExpectContinue() && isPayloadPresent(request)) {
580 request.setHeader(HttpHeaders.EXPECT, "100-continue");
581 }
582
583 for (Map.Entry<?, ?> entry : headers.entrySet()) {
584 if (!(entry.getKey() instanceof String)) {
585 continue;
586 }
587 if (entry.getValue() instanceof String) {
588 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
589 } else {
590 request.removeHeaders(entry.getKey().toString());
591 }
592 }
593
594 if (!state.isExpectContinue()) {
595 request.removeHeaders(HttpHeaders.EXPECT);
596 }
597
598 return request;
599 }
600
601 @SuppressWarnings("checkstyle:magicnumber")
602 private <T extends HttpUriRequest> T resume(T request, GetTask task) {
603 long resumeOffset = task.getResumeOffset();
604 if (resumeOffset > 0L && task.getDataFile() != null) {
605 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
606 request.setHeader(
607 HttpHeaders.IF_UNMODIFIED_SINCE,
608 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
609 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
610 }
611 return request;
612 }
613
614 @SuppressWarnings("checkstyle:magicnumber")
615 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
616 int status = response.getStatusLine().getStatusCode();
617 if (status >= 300) {
618 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
619 }
620 }
621
622 @Override
623 protected void implClose() {
624 try {
625 client.close();
626 } catch (IOException e) {
627 throw new UncheckedIOException(e);
628 }
629 AuthenticationContext.close(repoAuthContext);
630 AuthenticationContext.close(proxyAuthContext);
631 state.close();
632 }
633
634 private class EntityGetter {
635
636 private final GetTask task;
637
638 EntityGetter(GetTask task) {
639 this.task = task;
640 }
641
642 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
643 HttpEntity entity = response.getEntity();
644 if (entity == null) {
645 entity = new ByteArrayEntity(new byte[0]);
646 }
647
648 long offset = 0L, length = entity.getContentLength();
649 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
650 String range = rangeHeader != null ? rangeHeader.getValue() : null;
651 if (range != null) {
652 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
653 if (!m.matches()) {
654 throw new IOException("Invalid Content-Range header for partial download: " + range);
655 }
656 offset = Long.parseLong(m.group(1));
657 length = Long.parseLong(m.group(2)) + 1L;
658 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
659 throw new IOException("Invalid Content-Range header for partial download from offset "
660 + task.getResumeOffset() + ": " + range);
661 }
662 }
663
664 final boolean resume = offset > 0L;
665 final File dataFile = task.getDataFile();
666 if (dataFile == null) {
667 try (InputStream is = entity.getContent()) {
668 utilGet(task, is, true, length, resume);
669 extractChecksums(response);
670 }
671 } else {
672 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
673 task.setDataFile(tempFile.getPath().toFile(), resume);
674 if (resume && Files.isRegularFile(dataFile.toPath())) {
675 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
676 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
677 }
678 }
679 try (InputStream is = entity.getContent()) {
680 utilGet(task, is, true, length, resume);
681 }
682 tempFile.move();
683 } finally {
684 task.setDataFile(dataFile);
685 }
686 }
687 if (task.getDataFile() != null) {
688 Header lastModifiedHeader =
689 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
690 if (lastModifiedHeader != null) {
691 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
692 if (lastModified != null) {
693 task.getDataFile().setLastModified(lastModified.getTime());
694 }
695 }
696 }
697 extractChecksums(response);
698 }
699
700 private void extractChecksums(CloseableHttpResponse response) {
701 for (Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet()) {
702 Map<String, String> checksums = extractorEntry.getValue().extractChecksums(response);
703 if (checksums != null) {
704 checksums.forEach(task::setChecksum);
705 return;
706 }
707 }
708 }
709 }
710
711 private class PutTaskEntity extends AbstractHttpEntity {
712
713 private final PutTask task;
714
715 PutTaskEntity(PutTask task) {
716 this.task = task;
717 }
718
719 @Override
720 public boolean isRepeatable() {
721 return true;
722 }
723
724 @Override
725 public boolean isStreaming() {
726 return false;
727 }
728
729 @Override
730 public long getContentLength() {
731 return task.getDataLength();
732 }
733
734 @Override
735 public InputStream getContent() throws IOException {
736 return task.newInputStream();
737 }
738
739 @Override
740 public void writeTo(OutputStream os) throws IOException {
741 try {
742 utilPut(task, os, false);
743 } catch (TransferCancelledException e) {
744 throw (IOException) new InterruptedIOException().initCause(e);
745 }
746 }
747 }
748
749 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
750 private final int retryCount;
751
752 private final long retryInterval;
753
754 private final long retryIntervalMax;
755
756 private final Set<Integer> serviceUnavailableHttpCodes;
757
758
759
760
761
762
763 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
764
765 private ResolverServiceUnavailableRetryStrategy(
766 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
767 if (retryCount < 0) {
768 throw new IllegalArgumentException("retryCount must be >= 0");
769 }
770 if (retryInterval < 0L) {
771 throw new IllegalArgumentException("retryInterval must be >= 0");
772 }
773 if (retryIntervalMax < 0L) {
774 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
775 }
776 this.retryCount = retryCount;
777 this.retryInterval = retryInterval;
778 this.retryIntervalMax = retryIntervalMax;
779 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
780 }
781
782 @Override
783 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
784 final boolean retry = executionCount <= retryCount
785 && (serviceUnavailableHttpCodes.contains(
786 response.getStatusLine().getStatusCode()));
787 if (retry) {
788 Long retryInterval = retryInterval(response, executionCount, context);
789 if (retryInterval != null) {
790 RETRY_INTERVAL_HOLDER.set(retryInterval);
791 return true;
792 }
793 }
794 RETRY_INTERVAL_HOLDER.remove();
795 return false;
796 }
797
798
799
800
801
802
803
804
805 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
806 Long result = null;
807 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
808 if (header != null && header.getValue() != null) {
809 String headerValue = header.getValue();
810 if (headerValue.contains(":")) {
811 Date when = DateUtils.parseDate(headerValue);
812 if (when != null) {
813 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
814 }
815 } else {
816 try {
817 result = Long.parseLong(headerValue) * 1000L;
818 } catch (NumberFormatException e) {
819
820 }
821 }
822 }
823 if (result == null) {
824 result = executionCount * this.retryInterval;
825 }
826 if (result > retryIntervalMax) {
827 return null;
828 }
829 return result;
830 }
831
832 @Override
833 public long getRetryInterval() {
834 Long ri = RETRY_INTERVAL_HOLDER.get();
835 if (ri == null) {
836 return 0L;
837 }
838 RETRY_INTERVAL_HOLDER.remove();
839 return ri;
840 }
841 }
842 }