1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.apache;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.InterruptedIOException;
24 import java.io.OutputStream;
25 import java.io.UncheckedIOException;
26 import java.net.InetAddress;
27 import java.net.URI;
28 import java.net.URISyntaxException;
29 import java.net.UnknownHostException;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Set;
40 import java.util.function.Function;
41 import java.util.regex.Matcher;
42
43 import org.apache.http.Header;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpHeaders;
47 import org.apache.http.HttpHost;
48 import org.apache.http.HttpResponse;
49 import org.apache.http.HttpStatus;
50 import org.apache.http.auth.AuthSchemeProvider;
51 import org.apache.http.auth.AuthScope;
52 import org.apache.http.client.CredentialsProvider;
53 import org.apache.http.client.HttpRequestRetryHandler;
54 import org.apache.http.client.HttpResponseException;
55 import org.apache.http.client.ServiceUnavailableRetryStrategy;
56 import org.apache.http.client.config.AuthSchemes;
57 import org.apache.http.client.config.CookieSpecs;
58 import org.apache.http.client.config.RequestConfig;
59 import org.apache.http.client.methods.CloseableHttpResponse;
60 import org.apache.http.client.methods.HttpGet;
61 import org.apache.http.client.methods.HttpHead;
62 import org.apache.http.client.methods.HttpOptions;
63 import org.apache.http.client.methods.HttpPut;
64 import org.apache.http.client.methods.HttpUriRequest;
65 import org.apache.http.client.utils.DateUtils;
66 import org.apache.http.client.utils.URIUtils;
67 import org.apache.http.config.Registry;
68 import org.apache.http.config.RegistryBuilder;
69 import org.apache.http.config.SocketConfig;
70 import org.apache.http.entity.AbstractHttpEntity;
71 import org.apache.http.entity.ByteArrayEntity;
72 import org.apache.http.impl.NoConnectionReuseStrategy;
73 import org.apache.http.impl.auth.BasicScheme;
74 import org.apache.http.impl.auth.BasicSchemeFactory;
75 import org.apache.http.impl.auth.DigestSchemeFactory;
76 import org.apache.http.impl.auth.KerberosSchemeFactory;
77 import org.apache.http.impl.auth.NTLMSchemeFactory;
78 import org.apache.http.impl.auth.SPNegoSchemeFactory;
79 import org.apache.http.impl.client.CloseableHttpClient;
80 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
81 import org.apache.http.impl.client.HttpClientBuilder;
82 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
83 import org.apache.http.protocol.HttpContext;
84 import org.apache.http.util.EntityUtils;
85 import org.eclipse.aether.ConfigurationProperties;
86 import org.eclipse.aether.RepositorySystemSession;
87 import org.eclipse.aether.repository.AuthenticationContext;
88 import org.eclipse.aether.repository.Proxy;
89 import org.eclipse.aether.repository.RemoteRepository;
90 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
91 import org.eclipse.aether.spi.connector.transport.GetTask;
92 import org.eclipse.aether.spi.connector.transport.PeekTask;
93 import org.eclipse.aether.spi.connector.transport.PutTask;
94 import org.eclipse.aether.spi.connector.transport.TransportTask;
95 import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
96 import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
97 import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
98 import org.eclipse.aether.spi.io.PathProcessor;
99 import org.eclipse.aether.transfer.NoTransporterException;
100 import org.eclipse.aether.transfer.TransferCancelledException;
101 import org.eclipse.aether.util.ConfigUtils;
102 import org.eclipse.aether.util.FileUtils;
103 import org.slf4j.Logger;
104 import org.slf4j.LoggerFactory;
105
106 import static java.util.Objects.requireNonNull;
107 import static org.eclipse.aether.spi.connector.transport.http.HttpConstants.CONTENT_RANGE_PATTERN;
108 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
109 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
110 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
111 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
112 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
113 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
114 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
115
116
117
118
119 final class ApacheTransporter extends AbstractTransporter implements HttpTransporter {
120 private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
121
122 private final ChecksumExtractor checksumExtractor;
123
124 private final PathProcessor pathProcessor;
125
126 private final AuthenticationContext repoAuthContext;
127
128 private final AuthenticationContext proxyAuthContext;
129
130 private final URI baseUri;
131
132 private final HttpHost server;
133
134 private final HttpHost proxy;
135
136 private final CloseableHttpClient client;
137
138 private final Map<?, ?> headers;
139
140 private final LocalState state;
141
142 private final boolean preemptiveAuth;
143
144 private final boolean preemptivePutAuth;
145
146 private final boolean supportWebDav;
147
148 @SuppressWarnings("checkstyle:methodlength")
149 ApacheTransporter(
150 RemoteRepository repository,
151 RepositorySystemSession session,
152 ChecksumExtractor checksumExtractor,
153 PathProcessor pathProcessor)
154 throws NoTransporterException {
155 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
156 throw new NoTransporterException(repository);
157 }
158 this.checksumExtractor = checksumExtractor;
159 this.pathProcessor = pathProcessor;
160 try {
161 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
162 if (baseUri.isOpaque()) {
163 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
164 }
165 this.server = URIUtils.extractHost(baseUri);
166 if (server == null) {
167 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
168 }
169 } catch (URISyntaxException e) {
170 throw new NoTransporterException(repository, e.getMessage(), e);
171 }
172 this.proxy = toHost(repository.getProxy());
173
174 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
175 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
176
177 String httpsSecurityMode = ConfigUtils.getString(
178 session,
179 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
180 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
181 ConfigurationProperties.HTTPS_SECURITY_MODE);
182 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
183 session,
184 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
185 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
186 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
187 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
188 session,
189 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
190 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
191 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
192 this.state = new LocalState(
193 session,
194 repository,
195 new ConnMgrConfig(
196 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
197
198 this.headers = ConfigUtils.getMap(
199 session,
200 Collections.emptyMap(),
201 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
202 ConfigurationProperties.HTTP_HEADERS);
203
204 this.preemptiveAuth = ConfigUtils.getBoolean(
205 session,
206 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
207 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
208 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
209 this.preemptivePutAuth = ConfigUtils.getBoolean(
210 session,
211 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_PUT_AUTH,
212 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH + "." + repository.getId(),
213 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH);
214 this.supportWebDav = ConfigUtils.getBoolean(
215 session,
216 ConfigurationProperties.DEFAULT_HTTP_SUPPORT_WEBDAV,
217 ConfigurationProperties.HTTP_SUPPORT_WEBDAV + "." + repository.getId(),
218 ConfigurationProperties.HTTP_SUPPORT_WEBDAV);
219 String credentialEncoding = ConfigUtils.getString(
220 session,
221 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
222 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
223 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
224 int connectTimeout = ConfigUtils.getInteger(
225 session,
226 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
227 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
228 ConfigurationProperties.CONNECT_TIMEOUT);
229 int requestTimeout = ConfigUtils.getInteger(
230 session,
231 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
232 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
233 ConfigurationProperties.REQUEST_TIMEOUT);
234 int retryCount = ConfigUtils.getInteger(
235 session,
236 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
237 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
238 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
239 long retryInterval = ConfigUtils.getLong(
240 session,
241 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
242 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
243 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
244 long retryIntervalMax = ConfigUtils.getLong(
245 session,
246 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
247 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
248 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
249 String serviceUnavailableCodesString = ConfigUtils.getString(
250 session,
251 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
252 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
253 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
254 String retryHandlerName = ConfigUtils.getString(
255 session,
256 HTTP_RETRY_HANDLER_NAME_STANDARD,
257 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
258 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
259 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
260 session,
261 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
262 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
263 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
264 String userAgent = ConfigUtils.getString(
265 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
266
267 Charset credentialsCharset = Charset.forName(credentialEncoding);
268 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
269 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
270 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
271 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
272 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
273 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
274 .build();
275 SocketConfig socketConfig =
276 SocketConfig.custom().setSoTimeout(requestTimeout).build();
277 RequestConfig requestConfig = RequestConfig.custom()
278 .setConnectTimeout(connectTimeout)
279 .setConnectionRequestTimeout(connectTimeout)
280 .setLocalAddress(getHttpLocalAddress(session, repository))
281 .setCookieSpec(CookieSpecs.STANDARD)
282 .setSocketTimeout(requestTimeout)
283 .build();
284
285 HttpRequestRetryHandler retryHandler;
286 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
287 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
288 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
289 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
290 } else {
291 throw new IllegalArgumentException(
292 "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
293 }
294 Set<Integer> serviceUnavailableCodes = new HashSet<>();
295 try {
296 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
297 serviceUnavailableCodes.add(Integer.parseInt(code));
298 }
299 } catch (NumberFormatException e) {
300 throw new IllegalArgumentException(
301 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
302 + " (list of integers): " + serviceUnavailableCodesString);
303 }
304 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
305 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
306
307 HttpClientBuilder builder = HttpClientBuilder.create()
308 .setUserAgent(userAgent)
309 .setDefaultSocketConfig(socketConfig)
310 .setDefaultRequestConfig(requestConfig)
311 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
312 .setRetryHandler(retryHandler)
313 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
314 .setConnectionManager(state.getConnectionManager())
315 .setConnectionManagerShared(true)
316 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
317 .setProxy(proxy);
318 final boolean useSystemProperties = ConfigUtils.getBoolean(
319 session,
320 DEFAULT_USE_SYSTEM_PROPERTIES,
321 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
322 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
323 if (useSystemProperties) {
324 LOGGER.warn(
325 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
326 LOGGER.warn("Please use documented means to configure resolver transport.");
327 builder.useSystemProperties();
328 }
329
330 final String expectContinue = ConfigUtils.getString(
331 session,
332 null,
333 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
334 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
335 if (expectContinue != null) {
336 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
337 }
338
339 final boolean reuseConnections = ConfigUtils.getBoolean(
340 session,
341 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
342 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
343 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
344 if (!reuseConnections) {
345 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
346 }
347
348 this.client = builder.build();
349 }
350
351
352
353
354 private InetAddress getHttpLocalAddress(RepositorySystemSession session, RemoteRepository repository) {
355 String bindAddress = ConfigUtils.getString(
356 session,
357 null,
358 ConfigurationProperties.HTTP_LOCAL_ADDRESS + "." + repository.getId(),
359 ConfigurationProperties.HTTP_LOCAL_ADDRESS);
360 if (bindAddress == null) {
361 return null;
362 }
363 try {
364 return InetAddress.getByName(bindAddress);
365 } catch (UnknownHostException uhe) {
366 throw new IllegalArgumentException(
367 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
368 uhe);
369 }
370 }
371
372 private static HttpHost toHost(Proxy proxy) {
373 HttpHost host = null;
374 if (proxy != null) {
375 host = new HttpHost(proxy.getHost(), proxy.getPort());
376 }
377 return host;
378 }
379
380 private static CredentialsProvider toCredentialsProvider(
381 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
382 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
383 if (proxy != null) {
384 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
385 provider = new DemuxCredentialsProvider(provider, p, proxy);
386 }
387 return provider;
388 }
389
390 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
391 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
392 if (ctx != null) {
393 AuthScope basicScope = new AuthScope(host, port);
394 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
395
396 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
397 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
398 }
399 return provider;
400 }
401
402 LocalState getState() {
403 return state;
404 }
405
406 private URI resolve(TransportTask task) {
407 return UriUtils.resolve(baseUri, task.getLocation());
408 }
409
410 @Override
411 public int classify(Throwable error) {
412 if (error instanceof HttpTransporterException
413 && ((HttpTransporterException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
414 return ERROR_NOT_FOUND;
415 }
416 return ERROR_OTHER;
417 }
418
419 @Override
420 protected void implPeek(PeekTask task) throws Exception {
421 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
422 try {
423 execute(request, null);
424 } catch (HttpResponseException e) {
425 throw new HttpTransporterException(e.getStatusCode());
426 }
427 }
428
429 @Override
430 protected void implGet(GetTask task) throws Exception {
431 boolean resume = true;
432
433 EntityGetter getter = new EntityGetter(task);
434 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
435 while (true) {
436 try {
437 if (resume) {
438 resume(request, task);
439 }
440 execute(request, getter);
441 break;
442 } catch (HttpResponseException e) {
443 if (resume
444 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
445 && request.containsHeader(HttpHeaders.RANGE)) {
446 request = commonHeaders(new HttpGet(resolve(task)));
447 resume = false;
448 continue;
449 }
450 throw new HttpTransporterException(e.getStatusCode());
451 }
452 }
453 }
454
455 @Override
456 protected void implPut(PutTask task) throws Exception {
457 PutTaskEntity entity = new PutTaskEntity(task);
458 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
459 try {
460 execute(request, null);
461 } catch (HttpResponseException e) {
462 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
463 state.setExpectContinue(false);
464 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
465 execute(request, null);
466 return;
467 }
468 throw new HttpTransporterException(e.getStatusCode());
469 }
470 }
471
472 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
473 try {
474 SharingHttpContext context = new SharingHttpContext(state);
475 prepare(request, context);
476 try (CloseableHttpResponse response = client.execute(server, request, context)) {
477 try {
478 context.close();
479 handleStatus(response);
480 if (getter != null) {
481 getter.handle(response);
482 }
483 } finally {
484 EntityUtils.consumeQuietly(response.getEntity());
485 }
486 }
487 } catch (IOException e) {
488 if (e.getCause() instanceof TransferCancelledException) {
489 throw (Exception) e.getCause();
490 }
491 throw e;
492 }
493 }
494
495 private void prepare(HttpUriRequest request, SharingHttpContext context) throws HttpTransporterException {
496 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
497 if (preemptiveAuth || (preemptivePutAuth && put)) {
498 context.getAuthCache().put(server, new BasicScheme());
499 }
500 if (supportWebDav) {
501 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
502 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
503 try (CloseableHttpResponse response = client.execute(server, req, context)) {
504 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
505 EntityUtils.consumeQuietly(response.getEntity());
506 } catch (IOException e) {
507 LOGGER.debug("Failed to prepare HTTP context", e);
508 }
509 }
510 if (put && Boolean.TRUE.equals(state.getWebDav())) {
511 mkdirs(request.getURI(), context);
512 }
513 }
514 }
515
516 @SuppressWarnings("checkstyle:magicnumber")
517 private void mkdirs(URI uri, SharingHttpContext context) throws HttpTransporterException {
518 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
519 int index = 0;
520 for (; index < dirs.size(); index++) {
521 try (CloseableHttpResponse response =
522 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
523 try {
524 int status = response.getStatusLine().getStatusCode();
525 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
526 break;
527 } else if (status == HttpStatus.SC_CONFLICT) {
528 continue;
529 }
530 handleStatus(response);
531 } finally {
532 EntityUtils.consumeQuietly(response.getEntity());
533 }
534 } catch (IOException e) {
535 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
536 return;
537 }
538 }
539 for (index--; index >= 0; index--) {
540 try (CloseableHttpResponse response =
541 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
542 try {
543 handleStatus(response);
544 } finally {
545 EntityUtils.consumeQuietly(response.getEntity());
546 }
547 } catch (IOException e) {
548 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
549 return;
550 }
551 }
552 }
553
554 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
555 request.setEntity(entity);
556 return request;
557 }
558
559 private boolean isPayloadPresent(HttpUriRequest request) {
560 if (request instanceof HttpEntityEnclosingRequest) {
561 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
562 return entity != null && entity.getContentLength() != 0;
563 }
564 return false;
565 }
566
567 private <T extends HttpUriRequest> T commonHeaders(T request) {
568 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
569 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
570
571 if (state.isExpectContinue() && isPayloadPresent(request)) {
572 request.setHeader(HttpHeaders.EXPECT, "100-continue");
573 }
574
575 for (Map.Entry<?, ?> entry : headers.entrySet()) {
576 if (!(entry.getKey() instanceof String)) {
577 continue;
578 }
579 if (entry.getValue() instanceof String) {
580 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
581 } else {
582 request.removeHeaders(entry.getKey().toString());
583 }
584 }
585
586 if (!state.isExpectContinue()) {
587 request.removeHeaders(HttpHeaders.EXPECT);
588 }
589
590 return request;
591 }
592
593 @SuppressWarnings("checkstyle:magicnumber")
594 private <T extends HttpUriRequest> void resume(T request, GetTask task) throws IOException {
595 long resumeOffset = task.getResumeOffset();
596 if (resumeOffset > 0L && task.getDataPath() != null) {
597 long lastModified = Files.getLastModifiedTime(task.getDataPath()).toMillis();
598 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
599 request.setHeader(
600 HttpHeaders.IF_UNMODIFIED_SINCE, DateUtils.formatDate(new Date(lastModified - 60L * 1000L)));
601 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
602 }
603 }
604
605 @SuppressWarnings("checkstyle:magicnumber")
606 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
607 int status = response.getStatusLine().getStatusCode();
608 if (status >= 300) {
609 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
610 }
611 }
612
613 @Override
614 protected void implClose() {
615 try {
616 client.close();
617 } catch (IOException e) {
618 throw new UncheckedIOException(e);
619 }
620 AuthenticationContext.close(repoAuthContext);
621 AuthenticationContext.close(proxyAuthContext);
622 state.close();
623 }
624
625 private class EntityGetter {
626
627 private final GetTask task;
628
629 EntityGetter(GetTask task) {
630 this.task = task;
631 }
632
633 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
634 HttpEntity entity = response.getEntity();
635 if (entity == null) {
636 entity = new ByteArrayEntity(new byte[0]);
637 }
638
639 long offset = 0L, length = entity.getContentLength();
640 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
641 String range = rangeHeader != null ? rangeHeader.getValue() : null;
642 if (range != null) {
643 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
644 if (!m.matches()) {
645 throw new IOException("Invalid Content-Range header for partial download: " + range);
646 }
647 offset = Long.parseLong(m.group(1));
648 length = Long.parseLong(m.group(2)) + 1L;
649 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
650 throw new IOException("Invalid Content-Range header for partial download from offset "
651 + task.getResumeOffset() + ": " + range);
652 }
653 }
654
655 final boolean resume = offset > 0L;
656 final Path dataFile = task.getDataPath();
657 if (dataFile == null) {
658 try (InputStream is = entity.getContent()) {
659 utilGet(task, is, true, length, resume);
660 extractChecksums(response);
661 }
662 } else {
663 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile)) {
664 task.setDataPath(tempFile.getPath(), resume);
665 if (resume && Files.isRegularFile(dataFile)) {
666 try (InputStream inputStream = Files.newInputStream(dataFile)) {
667 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
668 }
669 }
670 try (InputStream is = entity.getContent()) {
671 utilGet(task, is, true, length, resume);
672 }
673 tempFile.move();
674 } finally {
675 task.setDataPath(dataFile);
676 }
677 }
678 if (task.getDataPath() != null) {
679 Header lastModifiedHeader =
680 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
681 if (lastModifiedHeader != null) {
682 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
683 if (lastModified != null) {
684 pathProcessor.setLastModified(task.getDataPath(), lastModified.getTime());
685 }
686 }
687 }
688 extractChecksums(response);
689 }
690
691 private void extractChecksums(CloseableHttpResponse response) {
692 Map<String, String> checksums = checksumExtractor.extractChecksums(headerGetter(response));
693 if (checksums != null && !checksums.isEmpty()) {
694 checksums.forEach(task::setChecksum);
695 }
696 }
697 }
698
699 private static Function<String, String> headerGetter(CloseableHttpResponse closeableHttpResponse) {
700 return s -> {
701 Header header = closeableHttpResponse.getFirstHeader(s);
702 return header != null ? header.getValue() : null;
703 };
704 }
705
706 private class PutTaskEntity extends AbstractHttpEntity {
707
708 private final PutTask task;
709
710 PutTaskEntity(PutTask task) {
711 this.task = task;
712 }
713
714 @Override
715 public boolean isRepeatable() {
716 return true;
717 }
718
719 @Override
720 public boolean isStreaming() {
721 return false;
722 }
723
724 @Override
725 public long getContentLength() {
726 return task.getDataLength();
727 }
728
729 @Override
730 public InputStream getContent() throws IOException {
731 return task.newInputStream();
732 }
733
734 @Override
735 public void writeTo(OutputStream os) throws IOException {
736 try {
737 utilPut(task, os, false);
738 } catch (TransferCancelledException e) {
739 throw (IOException) new InterruptedIOException().initCause(e);
740 }
741 }
742 }
743
744 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
745 private final int retryCount;
746
747 private final long retryInterval;
748
749 private final long retryIntervalMax;
750
751 private final Set<Integer> serviceUnavailableHttpCodes;
752
753
754
755
756
757
758 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
759
760 private ResolverServiceUnavailableRetryStrategy(
761 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
762 if (retryCount < 0) {
763 throw new IllegalArgumentException("retryCount must be >= 0");
764 }
765 if (retryInterval < 0L) {
766 throw new IllegalArgumentException("retryInterval must be >= 0");
767 }
768 if (retryIntervalMax < 0L) {
769 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
770 }
771 this.retryCount = retryCount;
772 this.retryInterval = retryInterval;
773 this.retryIntervalMax = retryIntervalMax;
774 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
775 }
776
777 @Override
778 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
779 final boolean retry = executionCount <= retryCount
780 && (serviceUnavailableHttpCodes.contains(
781 response.getStatusLine().getStatusCode()));
782 if (retry) {
783 Long retryInterval = retryInterval(response, executionCount, context);
784 if (retryInterval != null) {
785 RETRY_INTERVAL_HOLDER.set(retryInterval);
786 return true;
787 }
788 }
789 RETRY_INTERVAL_HOLDER.remove();
790 return false;
791 }
792
793
794
795
796
797
798
799
800 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
801 Long result = null;
802 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
803 if (header != null && header.getValue() != null) {
804 String headerValue = header.getValue();
805 if (headerValue.contains(":")) {
806 Date when = DateUtils.parseDate(headerValue);
807 if (when != null) {
808 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
809 }
810 } else {
811 try {
812 result = Long.parseLong(headerValue) * 1000L;
813 } catch (NumberFormatException e) {
814
815 }
816 }
817 }
818 if (result == null) {
819 result = executionCount * this.retryInterval;
820 }
821 if (result > retryIntervalMax) {
822 return null;
823 }
824 return result;
825 }
826
827 @Override
828 public long getRetryInterval() {
829 Long ri = RETRY_INTERVAL_HOLDER.get();
830 if (ri == null) {
831 return 0L;
832 }
833 RETRY_INTERVAL_HOLDER.remove();
834 return ri;
835 }
836 }
837 }