1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.apache;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.InterruptedIOException;
24 import java.io.OutputStream;
25 import java.io.UncheckedIOException;
26 import java.net.InetAddress;
27 import java.net.URI;
28 import java.net.URISyntaxException;
29 import java.net.UnknownHostException;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Set;
40 import java.util.function.Function;
41 import java.util.regex.Matcher;
42
43 import org.apache.http.Header;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpHeaders;
47 import org.apache.http.HttpHost;
48 import org.apache.http.HttpResponse;
49 import org.apache.http.HttpStatus;
50 import org.apache.http.auth.AuthSchemeProvider;
51 import org.apache.http.auth.AuthScope;
52 import org.apache.http.client.CredentialsProvider;
53 import org.apache.http.client.HttpRequestRetryHandler;
54 import org.apache.http.client.HttpResponseException;
55 import org.apache.http.client.ServiceUnavailableRetryStrategy;
56 import org.apache.http.client.config.AuthSchemes;
57 import org.apache.http.client.config.RequestConfig;
58 import org.apache.http.client.methods.CloseableHttpResponse;
59 import org.apache.http.client.methods.HttpGet;
60 import org.apache.http.client.methods.HttpHead;
61 import org.apache.http.client.methods.HttpOptions;
62 import org.apache.http.client.methods.HttpPut;
63 import org.apache.http.client.methods.HttpUriRequest;
64 import org.apache.http.client.utils.DateUtils;
65 import org.apache.http.client.utils.URIUtils;
66 import org.apache.http.config.Registry;
67 import org.apache.http.config.RegistryBuilder;
68 import org.apache.http.config.SocketConfig;
69 import org.apache.http.entity.AbstractHttpEntity;
70 import org.apache.http.entity.ByteArrayEntity;
71 import org.apache.http.impl.NoConnectionReuseStrategy;
72 import org.apache.http.impl.auth.BasicScheme;
73 import org.apache.http.impl.auth.BasicSchemeFactory;
74 import org.apache.http.impl.auth.DigestSchemeFactory;
75 import org.apache.http.impl.auth.KerberosSchemeFactory;
76 import org.apache.http.impl.auth.NTLMSchemeFactory;
77 import org.apache.http.impl.auth.SPNegoSchemeFactory;
78 import org.apache.http.impl.client.CloseableHttpClient;
79 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
80 import org.apache.http.impl.client.HttpClientBuilder;
81 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
82 import org.apache.http.protocol.HttpContext;
83 import org.apache.http.util.EntityUtils;
84 import org.eclipse.aether.ConfigurationProperties;
85 import org.eclipse.aether.RepositorySystemSession;
86 import org.eclipse.aether.repository.AuthenticationContext;
87 import org.eclipse.aether.repository.Proxy;
88 import org.eclipse.aether.repository.RemoteRepository;
89 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
90 import org.eclipse.aether.spi.connector.transport.GetTask;
91 import org.eclipse.aether.spi.connector.transport.PeekTask;
92 import org.eclipse.aether.spi.connector.transport.PutTask;
93 import org.eclipse.aether.spi.connector.transport.TransportTask;
94 import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
95 import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
96 import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
97 import org.eclipse.aether.spi.io.PathProcessor;
98 import org.eclipse.aether.transfer.NoTransporterException;
99 import org.eclipse.aether.transfer.TransferCancelledException;
100 import org.eclipse.aether.util.ConfigUtils;
101 import org.eclipse.aether.util.FileUtils;
102 import org.slf4j.Logger;
103 import org.slf4j.LoggerFactory;
104
105 import static java.util.Objects.requireNonNull;
106 import static org.eclipse.aether.spi.connector.transport.http.HttpConstants.CONTENT_RANGE_PATTERN;
107 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
108 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
109 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
110 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
111 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
112 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
113 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
114
115
116
117
118 final class ApacheTransporter extends AbstractTransporter implements HttpTransporter {
119 private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
120
121 private final ChecksumExtractor checksumExtractor;
122
123 private final PathProcessor pathProcessor;
124
125 private final AuthenticationContext repoAuthContext;
126
127 private final AuthenticationContext proxyAuthContext;
128
129 private final URI baseUri;
130
131 private final HttpHost server;
132
133 private final HttpHost proxy;
134
135 private final CloseableHttpClient client;
136
137 private final Map<?, ?> headers;
138
139 private final LocalState state;
140
141 private final boolean preemptiveAuth;
142
143 private final boolean preemptivePutAuth;
144
145 private final boolean supportWebDav;
146
147 @SuppressWarnings("checkstyle:methodlength")
148 ApacheTransporter(
149 RemoteRepository repository,
150 RepositorySystemSession session,
151 ChecksumExtractor checksumExtractor,
152 PathProcessor pathProcessor)
153 throws NoTransporterException {
154 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
155 throw new NoTransporterException(repository);
156 }
157 this.checksumExtractor = checksumExtractor;
158 this.pathProcessor = pathProcessor;
159 try {
160 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
161 if (baseUri.isOpaque()) {
162 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
163 }
164 this.server = URIUtils.extractHost(baseUri);
165 if (server == null) {
166 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
167 }
168 } catch (URISyntaxException e) {
169 throw new NoTransporterException(repository, e.getMessage(), e);
170 }
171 this.proxy = toHost(repository.getProxy());
172
173 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
174 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
175
176 String httpsSecurityMode = ConfigUtils.getString(
177 session,
178 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
179 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
180 ConfigurationProperties.HTTPS_SECURITY_MODE);
181 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
182 session,
183 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
184 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
185 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
186 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
187 session,
188 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
189 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
190 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
191 this.state = new LocalState(
192 session,
193 repository,
194 new ConnMgrConfig(
195 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
196
197 this.headers = ConfigUtils.getMap(
198 session,
199 Collections.emptyMap(),
200 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
201 ConfigurationProperties.HTTP_HEADERS);
202
203 this.preemptiveAuth = ConfigUtils.getBoolean(
204 session,
205 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
206 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
207 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
208 this.preemptivePutAuth = ConfigUtils.getBoolean(
209 session,
210 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_PUT_AUTH,
211 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH + "." + repository.getId(),
212 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH);
213 this.supportWebDav = ConfigUtils.getBoolean(
214 session,
215 ConfigurationProperties.DEFAULT_HTTP_SUPPORT_WEBDAV,
216 ConfigurationProperties.HTTP_SUPPORT_WEBDAV + "." + repository.getId(),
217 ConfigurationProperties.HTTP_SUPPORT_WEBDAV);
218 String credentialEncoding = ConfigUtils.getString(
219 session,
220 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
221 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
222 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
223 int connectTimeout = ConfigUtils.getInteger(
224 session,
225 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
226 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
227 ConfigurationProperties.CONNECT_TIMEOUT);
228 int requestTimeout = ConfigUtils.getInteger(
229 session,
230 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
231 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
232 ConfigurationProperties.REQUEST_TIMEOUT);
233 int retryCount = ConfigUtils.getInteger(
234 session,
235 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
236 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
237 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
238 long retryInterval = ConfigUtils.getLong(
239 session,
240 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
241 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
242 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
243 long retryIntervalMax = ConfigUtils.getLong(
244 session,
245 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
246 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
247 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
248 String serviceUnavailableCodesString = ConfigUtils.getString(
249 session,
250 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
251 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
252 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
253 String retryHandlerName = ConfigUtils.getString(
254 session,
255 HTTP_RETRY_HANDLER_NAME_STANDARD,
256 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
257 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
258 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
259 session,
260 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
261 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
262 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
263 String userAgent = ConfigUtils.getString(
264 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
265
266 Charset credentialsCharset = Charset.forName(credentialEncoding);
267 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
268 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
269 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
270 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
271 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
272 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
273 .build();
274 SocketConfig socketConfig =
275 SocketConfig.custom().setSoTimeout(requestTimeout).build();
276 RequestConfig requestConfig = RequestConfig.custom()
277 .setConnectTimeout(connectTimeout)
278 .setConnectionRequestTimeout(connectTimeout)
279 .setLocalAddress(getHttpLocalAddress(session, repository))
280 .setSocketTimeout(requestTimeout)
281 .build();
282
283 HttpRequestRetryHandler retryHandler;
284 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
285 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
286 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
287 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
288 } else {
289 throw new IllegalArgumentException(
290 "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
291 }
292 Set<Integer> serviceUnavailableCodes = new HashSet<>();
293 try {
294 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
295 serviceUnavailableCodes.add(Integer.parseInt(code));
296 }
297 } catch (NumberFormatException e) {
298 throw new IllegalArgumentException(
299 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
300 + " (list of integers): " + serviceUnavailableCodesString);
301 }
302 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
303 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
304
305 HttpClientBuilder builder = HttpClientBuilder.create()
306 .setUserAgent(userAgent)
307 .setDefaultSocketConfig(socketConfig)
308 .setDefaultRequestConfig(requestConfig)
309 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
310 .setRetryHandler(retryHandler)
311 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
312 .setConnectionManager(state.getConnectionManager())
313 .setConnectionManagerShared(true)
314 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
315 .setProxy(proxy);
316 final boolean useSystemProperties = ConfigUtils.getBoolean(
317 session,
318 DEFAULT_USE_SYSTEM_PROPERTIES,
319 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
320 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
321 if (useSystemProperties) {
322 LOGGER.warn(
323 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
324 LOGGER.warn("Please use documented means to configure resolver transport.");
325 builder.useSystemProperties();
326 }
327
328 final String expectContinue = ConfigUtils.getString(
329 session,
330 null,
331 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
332 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
333 if (expectContinue != null) {
334 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
335 }
336
337 final boolean reuseConnections = ConfigUtils.getBoolean(
338 session,
339 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
340 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
341 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
342 if (!reuseConnections) {
343 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
344 }
345
346 this.client = builder.build();
347 }
348
349
350
351
352 private InetAddress getHttpLocalAddress(RepositorySystemSession session, RemoteRepository repository) {
353 String bindAddress = ConfigUtils.getString(
354 session,
355 null,
356 ConfigurationProperties.HTTP_LOCAL_ADDRESS + "." + repository.getId(),
357 ConfigurationProperties.HTTP_LOCAL_ADDRESS);
358 if (bindAddress == null) {
359 return null;
360 }
361 try {
362 return InetAddress.getByName(bindAddress);
363 } catch (UnknownHostException uhe) {
364 throw new IllegalArgumentException(
365 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
366 uhe);
367 }
368 }
369
370 private static HttpHost toHost(Proxy proxy) {
371 HttpHost host = null;
372 if (proxy != null) {
373 host = new HttpHost(proxy.getHost(), proxy.getPort());
374 }
375 return host;
376 }
377
378 private static CredentialsProvider toCredentialsProvider(
379 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
380 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
381 if (proxy != null) {
382 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
383 provider = new DemuxCredentialsProvider(provider, p, proxy);
384 }
385 return provider;
386 }
387
388 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
389 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
390 if (ctx != null) {
391 AuthScope basicScope = new AuthScope(host, port);
392 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
393
394 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
395 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
396 }
397 return provider;
398 }
399
400 LocalState getState() {
401 return state;
402 }
403
404 private URI resolve(TransportTask task) {
405 return UriUtils.resolve(baseUri, task.getLocation());
406 }
407
408 @Override
409 public int classify(Throwable error) {
410 if (error instanceof HttpTransporterException
411 && ((HttpTransporterException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
412 return ERROR_NOT_FOUND;
413 }
414 return ERROR_OTHER;
415 }
416
417 @Override
418 protected void implPeek(PeekTask task) throws Exception {
419 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
420 try {
421 execute(request, null);
422 } catch (HttpResponseException e) {
423 throw new HttpTransporterException(e.getStatusCode());
424 }
425 }
426
427 @Override
428 protected void implGet(GetTask task) throws Exception {
429 boolean resume = true;
430
431 EntityGetter getter = new EntityGetter(task);
432 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
433 while (true) {
434 try {
435 if (resume) {
436 resume(request, task);
437 }
438 execute(request, getter);
439 break;
440 } catch (HttpResponseException e) {
441 if (resume
442 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
443 && request.containsHeader(HttpHeaders.RANGE)) {
444 request = commonHeaders(new HttpGet(resolve(task)));
445 resume = false;
446 continue;
447 }
448 throw new HttpTransporterException(e.getStatusCode());
449 }
450 }
451 }
452
453 @Override
454 protected void implPut(PutTask task) throws Exception {
455 PutTaskEntity entity = new PutTaskEntity(task);
456 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
457 try {
458 execute(request, null);
459 } catch (HttpResponseException e) {
460 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
461 state.setExpectContinue(false);
462 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
463 execute(request, null);
464 return;
465 }
466 throw new HttpTransporterException(e.getStatusCode());
467 }
468 }
469
470 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
471 try {
472 SharingHttpContext context = new SharingHttpContext(state);
473 prepare(request, context);
474 try (CloseableHttpResponse response = client.execute(server, request, context)) {
475 try {
476 context.close();
477 handleStatus(response);
478 if (getter != null) {
479 getter.handle(response);
480 }
481 } finally {
482 EntityUtils.consumeQuietly(response.getEntity());
483 }
484 }
485 } catch (IOException e) {
486 if (e.getCause() instanceof TransferCancelledException) {
487 throw (Exception) e.getCause();
488 }
489 throw e;
490 }
491 }
492
493 private void prepare(HttpUriRequest request, SharingHttpContext context) throws HttpTransporterException {
494 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
495 if (preemptiveAuth || (preemptivePutAuth && put)) {
496 context.getAuthCache().put(server, new BasicScheme());
497 }
498 if (supportWebDav) {
499 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
500 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
501 try (CloseableHttpResponse response = client.execute(server, req, context)) {
502 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
503 EntityUtils.consumeQuietly(response.getEntity());
504 } catch (IOException e) {
505 LOGGER.debug("Failed to prepare HTTP context", e);
506 }
507 }
508 if (put && Boolean.TRUE.equals(state.getWebDav())) {
509 mkdirs(request.getURI(), context);
510 }
511 }
512 }
513
514 @SuppressWarnings("checkstyle:magicnumber")
515 private void mkdirs(URI uri, SharingHttpContext context) throws HttpTransporterException {
516 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
517 int index = 0;
518 for (; index < dirs.size(); index++) {
519 try (CloseableHttpResponse response =
520 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
521 try {
522 int status = response.getStatusLine().getStatusCode();
523 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
524 break;
525 } else if (status == HttpStatus.SC_CONFLICT) {
526 continue;
527 }
528 handleStatus(response);
529 } finally {
530 EntityUtils.consumeQuietly(response.getEntity());
531 }
532 } catch (IOException e) {
533 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
534 return;
535 }
536 }
537 for (index--; index >= 0; index--) {
538 try (CloseableHttpResponse response =
539 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
540 try {
541 handleStatus(response);
542 } finally {
543 EntityUtils.consumeQuietly(response.getEntity());
544 }
545 } catch (IOException e) {
546 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
547 return;
548 }
549 }
550 }
551
552 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
553 request.setEntity(entity);
554 return request;
555 }
556
557 private boolean isPayloadPresent(HttpUriRequest request) {
558 if (request instanceof HttpEntityEnclosingRequest) {
559 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
560 return entity != null && entity.getContentLength() != 0;
561 }
562 return false;
563 }
564
565 private <T extends HttpUriRequest> T commonHeaders(T request) {
566 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
567 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
568
569 if (state.isExpectContinue() && isPayloadPresent(request)) {
570 request.setHeader(HttpHeaders.EXPECT, "100-continue");
571 }
572
573 for (Map.Entry<?, ?> entry : headers.entrySet()) {
574 if (!(entry.getKey() instanceof String)) {
575 continue;
576 }
577 if (entry.getValue() instanceof String) {
578 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
579 } else {
580 request.removeHeaders(entry.getKey().toString());
581 }
582 }
583
584 if (!state.isExpectContinue()) {
585 request.removeHeaders(HttpHeaders.EXPECT);
586 }
587
588 return request;
589 }
590
591 @SuppressWarnings("checkstyle:magicnumber")
592 private <T extends HttpUriRequest> void resume(T request, GetTask task) throws IOException {
593 long resumeOffset = task.getResumeOffset();
594 if (resumeOffset > 0L && task.getDataPath() != null) {
595 long lastModified = Files.getLastModifiedTime(task.getDataPath()).toMillis();
596 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
597 request.setHeader(
598 HttpHeaders.IF_UNMODIFIED_SINCE, DateUtils.formatDate(new Date(lastModified - 60L * 1000L)));
599 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
600 }
601 }
602
603 @SuppressWarnings("checkstyle:magicnumber")
604 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
605 int status = response.getStatusLine().getStatusCode();
606 if (status >= 300) {
607 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
608 }
609 }
610
611 @Override
612 protected void implClose() {
613 try {
614 client.close();
615 } catch (IOException e) {
616 throw new UncheckedIOException(e);
617 }
618 AuthenticationContext.close(repoAuthContext);
619 AuthenticationContext.close(proxyAuthContext);
620 state.close();
621 }
622
623 private class EntityGetter {
624
625 private final GetTask task;
626
627 EntityGetter(GetTask task) {
628 this.task = task;
629 }
630
631 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
632 HttpEntity entity = response.getEntity();
633 if (entity == null) {
634 entity = new ByteArrayEntity(new byte[0]);
635 }
636
637 long offset = 0L, length = entity.getContentLength();
638 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
639 String range = rangeHeader != null ? rangeHeader.getValue() : null;
640 if (range != null) {
641 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
642 if (!m.matches()) {
643 throw new IOException("Invalid Content-Range header for partial download: " + range);
644 }
645 offset = Long.parseLong(m.group(1));
646 length = Long.parseLong(m.group(2)) + 1L;
647 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
648 throw new IOException("Invalid Content-Range header for partial download from offset "
649 + task.getResumeOffset() + ": " + range);
650 }
651 }
652
653 final boolean resume = offset > 0L;
654 final Path dataFile = task.getDataPath();
655 if (dataFile == null) {
656 try (InputStream is = entity.getContent()) {
657 utilGet(task, is, true, length, resume);
658 extractChecksums(response);
659 }
660 } else {
661 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile)) {
662 task.setDataPath(tempFile.getPath(), resume);
663 if (resume && Files.isRegularFile(dataFile)) {
664 try (InputStream inputStream = Files.newInputStream(dataFile)) {
665 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
666 }
667 }
668 try (InputStream is = entity.getContent()) {
669 utilGet(task, is, true, length, resume);
670 }
671 tempFile.move();
672 } finally {
673 task.setDataPath(dataFile);
674 }
675 }
676 if (task.getDataPath() != null) {
677 Header lastModifiedHeader =
678 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
679 if (lastModifiedHeader != null) {
680 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
681 if (lastModified != null) {
682 pathProcessor.setLastModified(task.getDataPath(), lastModified.getTime());
683 }
684 }
685 }
686 extractChecksums(response);
687 }
688
689 private void extractChecksums(CloseableHttpResponse response) {
690 Map<String, String> checksums = checksumExtractor.extractChecksums(headerGetter(response));
691 if (checksums != null && !checksums.isEmpty()) {
692 checksums.forEach(task::setChecksum);
693 }
694 }
695 }
696
697 private static Function<String, String> headerGetter(CloseableHttpResponse closeableHttpResponse) {
698 return s -> {
699 Header header = closeableHttpResponse.getFirstHeader(s);
700 return header != null ? header.getValue() : null;
701 };
702 }
703
704 private class PutTaskEntity extends AbstractHttpEntity {
705
706 private final PutTask task;
707
708 PutTaskEntity(PutTask task) {
709 this.task = task;
710 }
711
712 @Override
713 public boolean isRepeatable() {
714 return true;
715 }
716
717 @Override
718 public boolean isStreaming() {
719 return false;
720 }
721
722 @Override
723 public long getContentLength() {
724 return task.getDataLength();
725 }
726
727 @Override
728 public InputStream getContent() throws IOException {
729 return task.newInputStream();
730 }
731
732 @Override
733 public void writeTo(OutputStream os) throws IOException {
734 try {
735 utilPut(task, os, false);
736 } catch (TransferCancelledException e) {
737 throw (IOException) new InterruptedIOException().initCause(e);
738 }
739 }
740 }
741
742 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
743 private final int retryCount;
744
745 private final long retryInterval;
746
747 private final long retryIntervalMax;
748
749 private final Set<Integer> serviceUnavailableHttpCodes;
750
751
752
753
754
755
756 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
757
758 private ResolverServiceUnavailableRetryStrategy(
759 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
760 if (retryCount < 0) {
761 throw new IllegalArgumentException("retryCount must be >= 0");
762 }
763 if (retryInterval < 0L) {
764 throw new IllegalArgumentException("retryInterval must be >= 0");
765 }
766 if (retryIntervalMax < 0L) {
767 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
768 }
769 this.retryCount = retryCount;
770 this.retryInterval = retryInterval;
771 this.retryIntervalMax = retryIntervalMax;
772 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
773 }
774
775 @Override
776 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
777 final boolean retry = executionCount <= retryCount
778 && (serviceUnavailableHttpCodes.contains(
779 response.getStatusLine().getStatusCode()));
780 if (retry) {
781 Long retryInterval = retryInterval(response, executionCount, context);
782 if (retryInterval != null) {
783 RETRY_INTERVAL_HOLDER.set(retryInterval);
784 return true;
785 }
786 }
787 RETRY_INTERVAL_HOLDER.remove();
788 return false;
789 }
790
791
792
793
794
795
796
797
798 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
799 Long result = null;
800 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
801 if (header != null && header.getValue() != null) {
802 String headerValue = header.getValue();
803 if (headerValue.contains(":")) {
804 Date when = DateUtils.parseDate(headerValue);
805 if (when != null) {
806 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
807 }
808 } else {
809 try {
810 result = Long.parseLong(headerValue) * 1000L;
811 } catch (NumberFormatException e) {
812
813 }
814 }
815 }
816 if (result == null) {
817 result = executionCount * this.retryInterval;
818 }
819 if (result > retryIntervalMax) {
820 return null;
821 }
822 return result;
823 }
824
825 @Override
826 public long getRetryInterval() {
827 Long ri = RETRY_INTERVAL_HOLDER.get();
828 if (ri == null) {
829 return 0L;
830 }
831 RETRY_INTERVAL_HOLDER.remove();
832 return ri;
833 }
834 }
835 }