1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.http;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.UnknownHostException;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Set;
40 import java.util.regex.Matcher;
41 import java.util.regex.Pattern;
42
43 import org.apache.http.Header;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpHeaders;
47 import org.apache.http.HttpHost;
48 import org.apache.http.HttpResponse;
49 import org.apache.http.HttpStatus;
50 import org.apache.http.auth.AuthSchemeProvider;
51 import org.apache.http.auth.AuthScope;
52 import org.apache.http.client.CredentialsProvider;
53 import org.apache.http.client.HttpRequestRetryHandler;
54 import org.apache.http.client.HttpResponseException;
55 import org.apache.http.client.ServiceUnavailableRetryStrategy;
56 import org.apache.http.client.config.AuthSchemes;
57 import org.apache.http.client.config.CookieSpecs;
58 import org.apache.http.client.config.RequestConfig;
59 import org.apache.http.client.methods.CloseableHttpResponse;
60 import org.apache.http.client.methods.HttpGet;
61 import org.apache.http.client.methods.HttpHead;
62 import org.apache.http.client.methods.HttpOptions;
63 import org.apache.http.client.methods.HttpPut;
64 import org.apache.http.client.methods.HttpUriRequest;
65 import org.apache.http.client.utils.DateUtils;
66 import org.apache.http.client.utils.URIUtils;
67 import org.apache.http.config.Registry;
68 import org.apache.http.config.RegistryBuilder;
69 import org.apache.http.config.SocketConfig;
70 import org.apache.http.entity.AbstractHttpEntity;
71 import org.apache.http.entity.ByteArrayEntity;
72 import org.apache.http.impl.NoConnectionReuseStrategy;
73 import org.apache.http.impl.auth.BasicScheme;
74 import org.apache.http.impl.auth.BasicSchemeFactory;
75 import org.apache.http.impl.auth.DigestSchemeFactory;
76 import org.apache.http.impl.auth.KerberosSchemeFactory;
77 import org.apache.http.impl.auth.NTLMSchemeFactory;
78 import org.apache.http.impl.auth.SPNegoSchemeFactory;
79 import org.apache.http.impl.client.CloseableHttpClient;
80 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
81 import org.apache.http.impl.client.HttpClientBuilder;
82 import org.apache.http.impl.client.LaxRedirectStrategy;
83 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
84 import org.apache.http.protocol.HttpContext;
85 import org.apache.http.util.EntityUtils;
86 import org.eclipse.aether.ConfigurationProperties;
87 import org.eclipse.aether.RepositorySystemSession;
88 import org.eclipse.aether.repository.AuthenticationContext;
89 import org.eclipse.aether.repository.Proxy;
90 import org.eclipse.aether.repository.RemoteRepository;
91 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
92 import org.eclipse.aether.spi.connector.transport.GetTask;
93 import org.eclipse.aether.spi.connector.transport.PeekTask;
94 import org.eclipse.aether.spi.connector.transport.PutTask;
95 import org.eclipse.aether.spi.connector.transport.TransportTask;
96 import org.eclipse.aether.transfer.NoTransporterException;
97 import org.eclipse.aether.transfer.TransferCancelledException;
98 import org.eclipse.aether.transport.http.RFC9457.HttpRFC9457Exception;
99 import org.eclipse.aether.transport.http.RFC9457.RFC9457Reporter;
100 import org.eclipse.aether.util.ConfigUtils;
101 import org.eclipse.aether.util.FileUtils;
102 import org.slf4j.Logger;
103 import org.slf4j.LoggerFactory;
104
105 import static java.util.Objects.requireNonNull;
106
107
108
109
110 final class HttpTransporter extends AbstractTransporter {
111
112 static final String BIND_ADDRESS = "aether.connector.bind.address";
113
114 static final String SUPPORT_WEBDAV = "aether.connector.http.supportWebDav";
115
116 static final String PREEMPTIVE_PUT_AUTH = "aether.connector.http.preemptivePutAuth";
117
118 static final String USE_SYSTEM_PROPERTIES = "aether.connector.http.useSystemProperties";
119
120 static final String HTTP_RETRY_HANDLER_NAME = "aether.connector.http.retryHandler.name";
121
122 private static final String HTTP_RETRY_HANDLER_NAME_STANDARD = "standard";
123
124 private static final String HTTP_RETRY_HANDLER_NAME_DEFAULT = "default";
125
126 static final String HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED =
127 "aether.connector.http.retryHandler.requestSentEnabled";
128
129 private static final Pattern CONTENT_RANGE_PATTERN =
130 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
131
132 private static final Logger LOGGER = LoggerFactory.getLogger(HttpTransporter.class);
133
134 private final Map<String, ChecksumExtractor> checksumExtractors;
135
136 private final AuthenticationContext repoAuthContext;
137
138 private final AuthenticationContext proxyAuthContext;
139
140 private final URI baseUri;
141
142 private final HttpHost server;
143
144 private final HttpHost proxy;
145
146 private final CloseableHttpClient client;
147
148 private final Map<?, ?> headers;
149
150 private final LocalState state;
151
152 private final boolean preemptiveAuth;
153
154 private final boolean preemptivePutAuth;
155
156 private final boolean supportWebDav;
157
158 @SuppressWarnings("checkstyle:methodlength")
159 HttpTransporter(
160 Map<String, ChecksumExtractor> checksumExtractors,
161 RemoteRepository repository,
162 RepositorySystemSession session)
163 throws NoTransporterException {
164 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
165 throw new NoTransporterException(repository);
166 }
167 this.checksumExtractors = requireNonNull(checksumExtractors, "checksum extractors must not be null");
168 try {
169 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
170 if (baseUri.isOpaque()) {
171 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
172 }
173 this.server = URIUtils.extractHost(baseUri);
174 if (server == null) {
175 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
176 }
177 } catch (URISyntaxException e) {
178 throw new NoTransporterException(repository, e.getMessage(), e);
179 }
180 this.proxy = toHost(repository.getProxy());
181
182 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
183 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
184
185 String httpsSecurityMode = ConfigUtils.getString(
186 session,
187 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
188 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
189 ConfigurationProperties.HTTPS_SECURITY_MODE);
190 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
191 session,
192 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
193 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
194 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
195 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
196 session,
197 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
198 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
199 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
200 this.state = new LocalState(
201 session,
202 repository,
203 new ConnMgrConfig(
204 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
205
206 this.headers = ConfigUtils.getMap(
207 session,
208 Collections.emptyMap(),
209 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
210 ConfigurationProperties.HTTP_HEADERS);
211
212 this.preemptiveAuth = ConfigUtils.getBoolean(
213 session,
214 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
215 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
216 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
217 this.preemptivePutAuth =
218 ConfigUtils.getBoolean(
219 session, true, PREEMPTIVE_PUT_AUTH + "." + repository.getId(), PREEMPTIVE_PUT_AUTH);
220 this.supportWebDav =
221 ConfigUtils.getBoolean(session, false, SUPPORT_WEBDAV + "." + repository.getId(), SUPPORT_WEBDAV);
222 String credentialEncoding = ConfigUtils.getString(
223 session,
224 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
225 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
226 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
227 int connectTimeout = ConfigUtils.getInteger(
228 session,
229 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
230 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
231 ConfigurationProperties.CONNECT_TIMEOUT);
232 int requestTimeout = ConfigUtils.getInteger(
233 session,
234 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
235 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
236 ConfigurationProperties.REQUEST_TIMEOUT);
237 int retryCount = ConfigUtils.getInteger(
238 session,
239 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
240 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
241 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
242 long retryInterval = ConfigUtils.getLong(
243 session,
244 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
245 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
246 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
247 long retryIntervalMax = ConfigUtils.getLong(
248 session,
249 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
250 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
251 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
252 String serviceUnavailableCodesString = ConfigUtils.getString(
253 session,
254 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
255 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
256 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
257 String retryHandlerName = ConfigUtils.getString(
258 session,
259 HTTP_RETRY_HANDLER_NAME_STANDARD,
260 HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
261 HTTP_RETRY_HANDLER_NAME);
262 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
263 session,
264 false,
265 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
266 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
267 String userAgent = ConfigUtils.getString(
268 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
269 int maxRedirects = ConfigUtils.getInteger(
270 session,
271 ConfigurationProperties.DEFAULT_HTTP_MAX_REDIRECTS,
272 ConfigurationProperties.HTTP_MAX_REDIRECTS + "." + repository.getId(),
273 ConfigurationProperties.HTTP_MAX_REDIRECTS);
274 boolean followRedirects = ConfigUtils.getBoolean(
275 session,
276 ConfigurationProperties.DEFAULT_FOLLOW_REDIRECTS,
277 ConfigurationProperties.HTTP_FOLLOW_REDIRECTS + "." + repository.getId(),
278 ConfigurationProperties.HTTP_FOLLOW_REDIRECTS);
279
280 Charset credentialsCharset = Charset.forName(credentialEncoding);
281 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
282 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
283 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
284 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
285 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
286 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
287 .build();
288 SocketConfig socketConfig =
289 SocketConfig.custom().setSoTimeout(requestTimeout).build();
290 RequestConfig requestConfig = RequestConfig.custom()
291 .setMaxRedirects(maxRedirects)
292 .setRedirectsEnabled(followRedirects)
293 .setConnectTimeout(connectTimeout)
294 .setConnectionRequestTimeout(connectTimeout)
295 .setLocalAddress(getBindAddress(session, repository))
296 .setCookieSpec(CookieSpecs.STANDARD)
297 .setSocketTimeout(requestTimeout)
298 .build();
299
300 HttpRequestRetryHandler retryHandler;
301 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
302 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
303 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
304 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
305 } else {
306 throw new IllegalArgumentException(
307 "Unsupported parameter " + HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
308 }
309 Set<Integer> serviceUnavailableCodes = new HashSet<>();
310 try {
311 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
312 serviceUnavailableCodes.add(Integer.parseInt(code));
313 }
314 } catch (NumberFormatException e) {
315 throw new IllegalArgumentException(
316 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
317 + " (list of integers): " + serviceUnavailableCodesString);
318 }
319 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
320 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
321
322 HttpClientBuilder builder = HttpClientBuilder.create()
323 .setUserAgent(userAgent)
324 .setRedirectStrategy(LaxRedirectStrategy.INSTANCE)
325 .setDefaultSocketConfig(socketConfig)
326 .setDefaultRequestConfig(requestConfig)
327 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
328 .setRetryHandler(retryHandler)
329 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
330 .setConnectionManager(state.getConnectionManager())
331 .setConnectionManagerShared(true)
332 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
333 .setProxy(proxy);
334 final boolean useSystemProperties = ConfigUtils.getBoolean(
335 session, false, USE_SYSTEM_PROPERTIES + "." + repository.getId(), USE_SYSTEM_PROPERTIES);
336 if (useSystemProperties) {
337 LOGGER.warn(
338 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
339 LOGGER.warn("Please use documented means to configure resolver transport.");
340 builder.useSystemProperties();
341 }
342
343 final String expectContinue = ConfigUtils.getString(
344 session,
345 null,
346 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
347 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
348 if (expectContinue != null) {
349 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
350 }
351
352 final boolean reuseConnections = ConfigUtils.getBoolean(
353 session,
354 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
355 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
356 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
357 if (!reuseConnections) {
358 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
359 }
360
361 this.client = builder.build();
362 }
363
364
365
366
367 private InetAddress getBindAddress(RepositorySystemSession session, RemoteRepository repository) {
368 String bindAddress =
369 ConfigUtils.getString(session, null, BIND_ADDRESS + "." + repository.getId(), BIND_ADDRESS);
370 if (bindAddress == null) {
371 return null;
372 }
373 try {
374 return InetAddress.getByName(bindAddress);
375 } catch (UnknownHostException uhe) {
376 throw new IllegalArgumentException(
377 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
378 uhe);
379 }
380 }
381
382 private static HttpHost toHost(Proxy proxy) {
383 HttpHost host = null;
384 if (proxy != null) {
385
386
387
388
389 host = new HttpHost(proxy.getHost(), proxy.getPort());
390 }
391 return host;
392 }
393
394 private static CredentialsProvider toCredentialsProvider(
395 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
396 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
397 if (proxy != null) {
398 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
399 provider = new DemuxCredentialsProvider(provider, p, proxy);
400 }
401 return provider;
402 }
403
404 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
405 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
406 if (ctx != null) {
407 AuthScope basicScope = new AuthScope(host, port);
408 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
409
410 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
411 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
412 }
413 return provider;
414 }
415
416 LocalState getState() {
417 return state;
418 }
419
420 private URI resolve(TransportTask task) {
421 return UriUtils.resolve(baseUri, task.getLocation());
422 }
423
424 @Override
425 public int classify(Throwable error) {
426 if (error instanceof HttpResponseException
427 && ((HttpResponseException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
428 return ERROR_NOT_FOUND;
429 }
430 return ERROR_OTHER;
431 }
432
433 @Override
434 protected void implPeek(PeekTask task) throws Exception {
435 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
436 execute(request, null);
437 }
438
439 @Override
440 protected void implGet(GetTask task) throws Exception {
441 boolean resume = true;
442 boolean applyChecksumExtractors = true;
443
444 EntityGetter getter = new EntityGetter(task);
445 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
446 while (true) {
447 try {
448 if (resume) {
449 resume(request, task);
450 }
451 if (applyChecksumExtractors) {
452 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
453 checksumExtractor.prepareRequest(request);
454 }
455 }
456 execute(request, getter);
457 break;
458 } catch (HttpResponseException e) {
459 if (resume
460 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
461 && request.containsHeader(HttpHeaders.RANGE)) {
462 request = commonHeaders(new HttpGet(resolve(task)));
463 resume = false;
464 continue;
465 }
466 if (applyChecksumExtractors) {
467 boolean retryWithoutExtractors = false;
468 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
469 if (checksumExtractor.retryWithoutExtractor(e)) {
470 retryWithoutExtractors = true;
471 break;
472 }
473 }
474 if (retryWithoutExtractors) {
475 request = commonHeaders(new HttpGet(resolve(task)));
476 applyChecksumExtractors = false;
477 continue;
478 }
479 }
480 throw e;
481 }
482 }
483 }
484
485 @Override
486 protected void implPut(PutTask task) throws Exception {
487 PutTaskEntity entity = new PutTaskEntity(task);
488 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
489 try {
490 execute(request, null);
491 } catch (HttpResponseException e) {
492 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
493 state.setExpectContinue(false);
494 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
495 execute(request, null);
496 return;
497 }
498 throw e;
499 }
500 }
501
502 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
503 try {
504 SharingHttpContext context = new SharingHttpContext(state);
505 prepare(request, context);
506 try (CloseableHttpResponse response = client.execute(server, request, context)) {
507 try {
508 context.close();
509 handleStatus(response);
510 if (getter != null) {
511 getter.handle(response);
512 }
513 } finally {
514 EntityUtils.consumeQuietly(response.getEntity());
515 }
516 }
517 } catch (IOException e) {
518 if (e.getCause() instanceof TransferCancelledException) {
519 throw (Exception) e.getCause();
520 }
521 throw e;
522 }
523 }
524
525 private void prepare(HttpUriRequest request, SharingHttpContext context) {
526 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
527 if (preemptiveAuth || (preemptivePutAuth && put)) {
528 context.getAuthCache().put(server, new BasicScheme());
529 }
530 if (supportWebDav) {
531 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
532 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
533 try (CloseableHttpResponse response = client.execute(server, req, context)) {
534 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
535 EntityUtils.consumeQuietly(response.getEntity());
536 } catch (IOException e) {
537 LOGGER.debug("Failed to prepare HTTP context", e);
538 }
539 }
540 if (put && Boolean.TRUE.equals(state.getWebDav())) {
541 mkdirs(request.getURI(), context);
542 }
543 }
544 }
545
546 private void mkdirs(URI uri, SharingHttpContext context) {
547 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
548 int index = 0;
549 for (; index < dirs.size(); index++) {
550 try (CloseableHttpResponse response =
551 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
552 try {
553 int status = response.getStatusLine().getStatusCode();
554 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
555 break;
556 } else if (status == HttpStatus.SC_CONFLICT) {
557 continue;
558 }
559 handleStatus(response);
560 } finally {
561 EntityUtils.consumeQuietly(response.getEntity());
562 }
563 } catch (IOException e) {
564 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
565 return;
566 }
567 }
568 for (index--; index >= 0; index--) {
569 try (CloseableHttpResponse response =
570 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
571 try {
572 handleStatus(response);
573 } finally {
574 EntityUtils.consumeQuietly(response.getEntity());
575 }
576 } catch (IOException e) {
577 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
578 return;
579 }
580 }
581 }
582
583 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
584 request.setEntity(entity);
585 return request;
586 }
587
588 private boolean isPayloadPresent(HttpUriRequest request) {
589 if (request instanceof HttpEntityEnclosingRequest) {
590 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
591 return entity != null && entity.getContentLength() != 0;
592 }
593 return false;
594 }
595
596 private <T extends HttpUriRequest> T commonHeaders(T request) {
597 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
598 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
599
600 if (state.isExpectContinue() && isPayloadPresent(request)) {
601 request.setHeader(HttpHeaders.EXPECT, "100-continue");
602 }
603
604 for (Map.Entry<?, ?> entry : headers.entrySet()) {
605 if (!(entry.getKey() instanceof String)) {
606 continue;
607 }
608 if (entry.getValue() instanceof String) {
609 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
610 } else {
611 request.removeHeaders(entry.getKey().toString());
612 }
613 }
614
615 if (!state.isExpectContinue()) {
616 request.removeHeaders(HttpHeaders.EXPECT);
617 }
618
619 return request;
620 }
621
622 private <T extends HttpUriRequest> void resume(T request, GetTask task) {
623 long resumeOffset = task.getResumeOffset();
624 if (resumeOffset > 0L && task.getDataFile() != null) {
625 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
626 request.setHeader(
627 HttpHeaders.IF_UNMODIFIED_SINCE,
628 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
629 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
630 }
631 }
632
633 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException, HttpRFC9457Exception {
634 int status = response.getStatusLine().getStatusCode();
635 if (status >= 300) {
636 if (RFC9457Reporter.INSTANCE.isRFC9457Message(response)) {
637 RFC9457Reporter.INSTANCE.generateException(response);
638 }
639 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
640 }
641 }
642
643 @Override
644 protected void implClose() {
645 try {
646 client.close();
647 } catch (IOException e) {
648 throw new UncheckedIOException(e);
649 }
650 AuthenticationContext.close(repoAuthContext);
651 AuthenticationContext.close(proxyAuthContext);
652 state.close();
653 }
654
655 private class EntityGetter {
656
657 private final GetTask task;
658
659 EntityGetter(GetTask task) {
660 this.task = task;
661 }
662
663 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
664 HttpEntity entity = response.getEntity();
665 if (entity == null) {
666 entity = new ByteArrayEntity(new byte[0]);
667 }
668
669 long offset = 0L, length = entity.getContentLength();
670 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
671 String range = rangeHeader != null ? rangeHeader.getValue() : null;
672 if (range != null) {
673 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
674 if (!m.matches()) {
675 throw new IOException("Invalid Content-Range header for partial download: " + range);
676 }
677 offset = Long.parseLong(m.group(1));
678 length = Long.parseLong(m.group(2)) + 1L;
679 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
680 throw new IOException("Invalid Content-Range header for partial download from offset "
681 + task.getResumeOffset() + ": " + range);
682 }
683 }
684
685 final boolean resume = offset > 0L;
686 final File dataFile = task.getDataFile();
687 if (dataFile == null) {
688 try (InputStream is = entity.getContent()) {
689 utilGet(task, is, true, length, resume);
690 extractChecksums(response);
691 }
692 } else {
693 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
694 task.setDataFile(tempFile.getPath().toFile(), resume);
695 if (resume && Files.isRegularFile(dataFile.toPath())) {
696 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
697 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
698 }
699 }
700 try (InputStream is = entity.getContent()) {
701 utilGet(task, is, true, length, resume);
702 }
703 tempFile.move();
704 } finally {
705 task.setDataFile(dataFile);
706 }
707 }
708 if (task.getDataFile() != null) {
709 Header lastModifiedHeader =
710 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
711 if (lastModifiedHeader != null) {
712 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
713 if (lastModified != null) {
714 task.getDataFile().setLastModified(lastModified.getTime());
715 }
716 }
717 }
718 extractChecksums(response);
719 }
720
721 private void extractChecksums(CloseableHttpResponse response) {
722 for (Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet()) {
723 Map<String, String> checksums = extractorEntry.getValue().extractChecksums(response);
724 if (checksums != null) {
725 checksums.forEach(task::setChecksum);
726 return;
727 }
728 }
729 }
730 }
731
732 private class PutTaskEntity extends AbstractHttpEntity {
733
734 private final PutTask task;
735
736 PutTaskEntity(PutTask task) {
737 this.task = task;
738 }
739
740 @Override
741 public boolean isRepeatable() {
742 return true;
743 }
744
745 @Override
746 public boolean isStreaming() {
747 return false;
748 }
749
750 @Override
751 public long getContentLength() {
752 return task.getDataLength();
753 }
754
755 @Override
756 public InputStream getContent() throws IOException {
757 return task.newInputStream();
758 }
759
760 @Override
761 public void writeTo(OutputStream os) throws IOException {
762 try {
763 utilPut(task, os, false);
764 } catch (TransferCancelledException e) {
765 throw (IOException) new InterruptedIOException().initCause(e);
766 }
767 }
768 }
769
770 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
771 private final int retryCount;
772
773 private final long retryInterval;
774
775 private final long retryIntervalMax;
776
777 private final Set<Integer> serviceUnavailableHttpCodes;
778
779
780
781
782
783
784 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
785
786 private ResolverServiceUnavailableRetryStrategy(
787 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
788 if (retryCount < 0) {
789 throw new IllegalArgumentException("retryCount must be >= 0");
790 }
791 if (retryInterval < 0L) {
792 throw new IllegalArgumentException("retryInterval must be >= 0");
793 }
794 if (retryIntervalMax < 0L) {
795 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
796 }
797 this.retryCount = retryCount;
798 this.retryInterval = retryInterval;
799 this.retryIntervalMax = retryIntervalMax;
800 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
801 }
802
803 @Override
804 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
805 final boolean retry = executionCount <= retryCount
806 && (serviceUnavailableHttpCodes.contains(
807 response.getStatusLine().getStatusCode()));
808 if (retry) {
809 Long retryInterval = retryInterval(response, executionCount);
810 if (retryInterval != null) {
811 RETRY_INTERVAL_HOLDER.set(retryInterval);
812 return true;
813 }
814 }
815 RETRY_INTERVAL_HOLDER.remove();
816 return false;
817 }
818
819
820
821
822
823
824
825
826 private Long retryInterval(HttpResponse httpResponse, int executionCount) {
827 Long result = null;
828 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
829 if (header != null && header.getValue() != null) {
830 String headerValue = header.getValue();
831 if (headerValue.contains(":")) {
832 Date when = DateUtils.parseDate(headerValue);
833 if (when != null) {
834 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
835 }
836 } else {
837 try {
838 result = Long.parseLong(headerValue) * 1000L;
839 } catch (NumberFormatException e) {
840
841 }
842 }
843 }
844 if (result == null) {
845 result = executionCount * this.retryInterval;
846 }
847 if (result > retryIntervalMax) {
848 return null;
849 }
850 return result;
851 }
852
853 @Override
854 public long getRetryInterval() {
855 Long ri = RETRY_INTERVAL_HOLDER.get();
856 if (ri == null) {
857 return 0L;
858 }
859 RETRY_INTERVAL_HOLDER.remove();
860 return ri;
861 }
862 }
863 }