1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.http;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.UnknownHostException;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Set;
40 import java.util.regex.Matcher;
41 import java.util.regex.Pattern;
42
43 import org.apache.http.Header;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpHeaders;
47 import org.apache.http.HttpHost;
48 import org.apache.http.HttpResponse;
49 import org.apache.http.HttpStatus;
50 import org.apache.http.auth.AuthSchemeProvider;
51 import org.apache.http.auth.AuthScope;
52 import org.apache.http.client.AuthCache;
53 import org.apache.http.client.CredentialsProvider;
54 import org.apache.http.client.HttpRequestRetryHandler;
55 import org.apache.http.client.HttpResponseException;
56 import org.apache.http.client.ServiceUnavailableRetryStrategy;
57 import org.apache.http.client.config.AuthSchemes;
58 import org.apache.http.client.config.CookieSpecs;
59 import org.apache.http.client.config.RequestConfig;
60 import org.apache.http.client.methods.CloseableHttpResponse;
61 import org.apache.http.client.methods.HttpGet;
62 import org.apache.http.client.methods.HttpHead;
63 import org.apache.http.client.methods.HttpOptions;
64 import org.apache.http.client.methods.HttpPut;
65 import org.apache.http.client.methods.HttpUriRequest;
66 import org.apache.http.client.utils.DateUtils;
67 import org.apache.http.client.utils.URIUtils;
68 import org.apache.http.config.Registry;
69 import org.apache.http.config.RegistryBuilder;
70 import org.apache.http.config.SocketConfig;
71 import org.apache.http.entity.AbstractHttpEntity;
72 import org.apache.http.entity.ByteArrayEntity;
73 import org.apache.http.impl.NoConnectionReuseStrategy;
74 import org.apache.http.impl.auth.BasicScheme;
75 import org.apache.http.impl.auth.BasicSchemeFactory;
76 import org.apache.http.impl.auth.DigestSchemeFactory;
77 import org.apache.http.impl.auth.KerberosSchemeFactory;
78 import org.apache.http.impl.auth.NTLMSchemeFactory;
79 import org.apache.http.impl.auth.SPNegoSchemeFactory;
80 import org.apache.http.impl.client.BasicAuthCache;
81 import org.apache.http.impl.client.CloseableHttpClient;
82 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
83 import org.apache.http.impl.client.HttpClientBuilder;
84 import org.apache.http.impl.client.LaxRedirectStrategy;
85 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
86 import org.apache.http.protocol.HttpContext;
87 import org.apache.http.util.EntityUtils;
88 import org.eclipse.aether.ConfigurationProperties;
89 import org.eclipse.aether.RepositorySystemSession;
90 import org.eclipse.aether.repository.AuthenticationContext;
91 import org.eclipse.aether.repository.Proxy;
92 import org.eclipse.aether.repository.RemoteRepository;
93 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
94 import org.eclipse.aether.spi.connector.transport.GetTask;
95 import org.eclipse.aether.spi.connector.transport.PeekTask;
96 import org.eclipse.aether.spi.connector.transport.PutTask;
97 import org.eclipse.aether.spi.connector.transport.TransportTask;
98 import org.eclipse.aether.transfer.NoTransporterException;
99 import org.eclipse.aether.transfer.TransferCancelledException;
100 import org.eclipse.aether.transport.http.RFC9457.HttpRFC9457Exception;
101 import org.eclipse.aether.transport.http.RFC9457.RFC9457Reporter;
102 import org.eclipse.aether.util.ConfigUtils;
103 import org.eclipse.aether.util.FileUtils;
104 import org.eclipse.aether.util.StringDigestUtil;
105 import org.slf4j.Logger;
106 import org.slf4j.LoggerFactory;
107
108 import static java.util.Objects.requireNonNull;
109
110
111
112
113 final class HttpTransporter extends AbstractTransporter {
114
115 static final String BIND_ADDRESS = "aether.connector.bind.address";
116
117 static final String SUPPORT_WEBDAV = "aether.connector.http.supportWebDav";
118
119 static final String PREEMPTIVE_PUT_AUTH = "aether.connector.http.preemptivePutAuth";
120
121 static final String USE_SYSTEM_PROPERTIES = "aether.connector.http.useSystemProperties";
122
123 static final String HTTP_RETRY_HANDLER_NAME = "aether.connector.http.retryHandler.name";
124
125 private static final String HTTP_RETRY_HANDLER_NAME_STANDARD = "standard";
126
127 private static final String HTTP_RETRY_HANDLER_NAME_DEFAULT = "default";
128
129 static final String HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED =
130 "aether.connector.http.retryHandler.requestSentEnabled";
131
132 private static final Pattern CONTENT_RANGE_PATTERN =
133 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
134
135 private static final Logger LOGGER = LoggerFactory.getLogger(HttpTransporter.class);
136
137 private final Map<String, ChecksumExtractor> checksumExtractors;
138
139 private final AuthenticationContext repoAuthContext;
140
141 private final AuthenticationContext proxyAuthContext;
142
143 private final URI baseUri;
144
145 private final HttpHost server;
146
147 private final HttpHost proxy;
148
149 private final CloseableHttpClient client;
150
151 private final Map<?, ?> headers;
152
153 private final LocalState state;
154
155 private final boolean preemptiveAuth;
156
157 private final boolean preemptivePutAuth;
158
159 private final boolean supportWebDav;
160
161 private final AuthCache authCache;
162
163 private static final Object AUTH_CACHE_MUTEX = new Object();
164
165 @SuppressWarnings("checkstyle:methodlength")
166 HttpTransporter(
167 Map<String, ChecksumExtractor> checksumExtractors,
168 RemoteRepository repository,
169 RepositorySystemSession session)
170 throws NoTransporterException {
171 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
172 throw new NoTransporterException(repository);
173 }
174 this.checksumExtractors = requireNonNull(checksumExtractors, "checksum extractors must not be null");
175 try {
176 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
177 if (baseUri.isOpaque()) {
178 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
179 }
180 this.server = URIUtils.extractHost(baseUri);
181 if (server == null) {
182 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
183 }
184 } catch (URISyntaxException e) {
185 throw new NoTransporterException(repository, e.getMessage(), e);
186 }
187 this.proxy = toHost(repository.getProxy());
188
189 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
190 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
191
192 String httpsSecurityMode = ConfigUtils.getString(
193 session,
194 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
195 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
196 ConfigurationProperties.HTTPS_SECURITY_MODE);
197 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
198 session,
199 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
200 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
201 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
202 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
203 session,
204 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
205 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
206 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
207 this.state = new LocalState(
208 session,
209 repository,
210 new ConnMgrConfig(
211 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
212
213 this.headers = ConfigUtils.getMap(
214 session,
215 Collections.emptyMap(),
216 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
217 ConfigurationProperties.HTTP_HEADERS);
218
219 this.preemptiveAuth = ConfigUtils.getBoolean(
220 session,
221 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
222 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
223 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
224 this.preemptivePutAuth =
225 ConfigUtils.getBoolean(
226 session, true, PREEMPTIVE_PUT_AUTH + "." + repository.getId(), PREEMPTIVE_PUT_AUTH);
227 this.supportWebDav =
228 ConfigUtils.getBoolean(session, false, SUPPORT_WEBDAV + "." + repository.getId(), SUPPORT_WEBDAV);
229 String credentialEncoding = ConfigUtils.getString(
230 session,
231 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
232 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
233 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
234 int connectTimeout = ConfigUtils.getInteger(
235 session,
236 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
237 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
238 ConfigurationProperties.CONNECT_TIMEOUT);
239 int requestTimeout = ConfigUtils.getInteger(
240 session,
241 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
242 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
243 ConfigurationProperties.REQUEST_TIMEOUT);
244 int retryCount = ConfigUtils.getInteger(
245 session,
246 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
247 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
248 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
249 long retryInterval = ConfigUtils.getLong(
250 session,
251 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
252 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
253 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
254 long retryIntervalMax = ConfigUtils.getLong(
255 session,
256 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
257 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
258 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
259 String serviceUnavailableCodesString = ConfigUtils.getString(
260 session,
261 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
262 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
263 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
264 String retryHandlerName = ConfigUtils.getString(
265 session,
266 HTTP_RETRY_HANDLER_NAME_STANDARD,
267 HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
268 HTTP_RETRY_HANDLER_NAME);
269 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
270 session,
271 false,
272 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
273 HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
274 String userAgent = ConfigUtils.getString(
275 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
276 int maxRedirects = ConfigUtils.getInteger(
277 session,
278 ConfigurationProperties.DEFAULT_HTTP_MAX_REDIRECTS,
279 ConfigurationProperties.HTTP_MAX_REDIRECTS + "." + repository.getId(),
280 ConfigurationProperties.HTTP_MAX_REDIRECTS);
281 boolean followRedirects = ConfigUtils.getBoolean(
282 session,
283 ConfigurationProperties.DEFAULT_FOLLOW_REDIRECTS,
284 ConfigurationProperties.HTTP_FOLLOW_REDIRECTS + "." + repository.getId(),
285 ConfigurationProperties.HTTP_FOLLOW_REDIRECTS);
286
287 Charset credentialsCharset = Charset.forName(credentialEncoding);
288 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
289 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
290 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
291 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
292 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
293 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
294 .build();
295 SocketConfig socketConfig =
296 SocketConfig.custom().setSoTimeout(requestTimeout).build();
297 RequestConfig requestConfig = RequestConfig.custom()
298 .setMaxRedirects(maxRedirects)
299 .setRedirectsEnabled(followRedirects)
300 .setConnectTimeout(connectTimeout)
301 .setConnectionRequestTimeout(connectTimeout)
302 .setLocalAddress(getBindAddress(session, repository))
303 .setCookieSpec(CookieSpecs.STANDARD)
304 .setSocketTimeout(requestTimeout)
305 .build();
306
307 HttpRequestRetryHandler retryHandler;
308 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
309 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
310 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
311 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
312 } else {
313 throw new IllegalArgumentException(
314 "Unsupported parameter " + HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
315 }
316 Set<Integer> serviceUnavailableCodes = new HashSet<>();
317 try {
318 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
319 serviceUnavailableCodes.add(Integer.parseInt(code));
320 }
321 } catch (NumberFormatException e) {
322 throw new IllegalArgumentException(
323 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
324 + " (list of integers): " + serviceUnavailableCodesString);
325 }
326 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
327 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
328
329 HttpClientBuilder builder = HttpClientBuilder.create()
330 .setUserAgent(userAgent)
331 .setRedirectStrategy(LaxRedirectStrategy.INSTANCE)
332 .setDefaultSocketConfig(socketConfig)
333 .setDefaultRequestConfig(requestConfig)
334 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
335 .setRetryHandler(retryHandler)
336 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
337 .setConnectionManager(state.getConnectionManager())
338 .setConnectionManagerShared(true)
339 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
340 .setProxy(proxy);
341 final boolean useSystemProperties = ConfigUtils.getBoolean(
342 session, false, USE_SYSTEM_PROPERTIES + "." + repository.getId(), USE_SYSTEM_PROPERTIES);
343 if (useSystemProperties) {
344 LOGGER.warn(
345 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
346 LOGGER.warn("Please use documented means to configure resolver transport.");
347 builder.useSystemProperties();
348 }
349
350 final String expectContinue = ConfigUtils.getString(
351 session,
352 null,
353 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
354 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
355 if (expectContinue != null) {
356 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
357 }
358
359 final boolean reuseConnections = ConfigUtils.getBoolean(
360 session,
361 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
362 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
363 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
364 if (!reuseConnections) {
365 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
366 }
367
368 if (session.getCache() != null) {
369 String authCacheKey = getClass().getSimpleName() + "-" + repository.getId() + "-"
370 + StringDigestUtil.sha1(repository.toString());
371 synchronized (AUTH_CACHE_MUTEX) {
372 AuthCache cache = (AuthCache) session.getCache().get(session, authCacheKey);
373 if (cache == null) {
374 cache = new BasicAuthCache();
375 session.getCache().put(session, authCacheKey, cache);
376 }
377 this.authCache = cache;
378 }
379 } else {
380 this.authCache = new BasicAuthCache();
381 }
382 this.client = builder.build();
383 }
384
385
386
387
388 private InetAddress getBindAddress(RepositorySystemSession session, RemoteRepository repository) {
389 String bindAddress =
390 ConfigUtils.getString(session, null, BIND_ADDRESS + "." + repository.getId(), BIND_ADDRESS);
391 if (bindAddress == null) {
392 return null;
393 }
394 try {
395 return InetAddress.getByName(bindAddress);
396 } catch (UnknownHostException uhe) {
397 throw new IllegalArgumentException(
398 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
399 uhe);
400 }
401 }
402
403 private static HttpHost toHost(Proxy proxy) {
404 HttpHost host = null;
405 if (proxy != null) {
406
407
408
409
410 host = new HttpHost(proxy.getHost(), proxy.getPort());
411 }
412 return host;
413 }
414
415 private static CredentialsProvider toCredentialsProvider(
416 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
417 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
418 if (proxy != null) {
419 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
420 provider = new DemuxCredentialsProvider(provider, p, proxy);
421 }
422 return provider;
423 }
424
425 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
426 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
427 if (ctx != null) {
428 AuthScope basicScope = new AuthScope(host, port);
429 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
430
431 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
432 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
433 }
434 return provider;
435 }
436
437 LocalState getState() {
438 return state;
439 }
440
441 private URI resolve(TransportTask task) {
442 return UriUtils.resolve(baseUri, task.getLocation());
443 }
444
445 @Override
446 public int classify(Throwable error) {
447 if (error instanceof HttpResponseException) {
448 int statusCode = ((HttpResponseException) error).getStatusCode();
449 if (statusCode == HttpStatus.SC_NOT_FOUND || statusCode == HttpStatus.SC_GONE) {
450 return ERROR_NOT_FOUND;
451 }
452 }
453 return ERROR_OTHER;
454 }
455
456 @Override
457 protected void implPeek(PeekTask task) throws Exception {
458 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
459 execute(request, null);
460 }
461
462 @Override
463 protected void implGet(GetTask task) throws Exception {
464 boolean resume = true;
465 boolean applyChecksumExtractors = true;
466
467 EntityGetter getter = new EntityGetter(task);
468 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
469 while (true) {
470 try {
471 if (resume) {
472 resume(request, task);
473 }
474 if (applyChecksumExtractors) {
475 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
476 checksumExtractor.prepareRequest(request);
477 }
478 }
479 execute(request, getter);
480 break;
481 } catch (HttpResponseException e) {
482 if (resume
483 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
484 && request.containsHeader(HttpHeaders.RANGE)) {
485 request = commonHeaders(new HttpGet(resolve(task)));
486 resume = false;
487 continue;
488 }
489 if (applyChecksumExtractors) {
490 boolean retryWithoutExtractors = false;
491 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
492 if (checksumExtractor.retryWithoutExtractor(e)) {
493 retryWithoutExtractors = true;
494 break;
495 }
496 }
497 if (retryWithoutExtractors) {
498 request = commonHeaders(new HttpGet(resolve(task)));
499 applyChecksumExtractors = false;
500 continue;
501 }
502 }
503 throw e;
504 }
505 }
506 }
507
508 @Override
509 protected void implPut(PutTask task) throws Exception {
510 PutTaskEntity entity = new PutTaskEntity(task);
511 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
512 try {
513 execute(request, null);
514 } catch (HttpResponseException e) {
515 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
516 state.setExpectContinue(false);
517 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
518 execute(request, null);
519 return;
520 }
521 throw e;
522 }
523 }
524
525 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
526 try {
527 SharingHttpContext context = new SharingHttpContext(state);
528 context.setAuthCache(authCache);
529 prepare(request, context);
530 try (CloseableHttpResponse response = client.execute(server, request, context)) {
531 try {
532 handleStatus(response);
533 if (getter != null) {
534 getter.handle(response);
535 }
536 } finally {
537 EntityUtils.consumeQuietly(response.getEntity());
538 }
539 }
540 } catch (IOException e) {
541 if (e.getCause() instanceof TransferCancelledException) {
542 throw (Exception) e.getCause();
543 }
544 throw e;
545 }
546 }
547
548 private void prepare(HttpUriRequest request, SharingHttpContext context) {
549 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
550 if (preemptiveAuth || (preemptivePutAuth && put)) {
551 context.getAuthCache().put(server, new BasicScheme());
552 }
553 if (supportWebDav) {
554 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
555 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
556 try (CloseableHttpResponse response = client.execute(server, req, context)) {
557 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
558 EntityUtils.consumeQuietly(response.getEntity());
559 } catch (IOException e) {
560 LOGGER.debug("Failed to prepare HTTP context", e);
561 }
562 }
563 if (put && Boolean.TRUE.equals(state.getWebDav())) {
564 mkdirs(request.getURI(), context);
565 }
566 }
567 }
568
569 private void mkdirs(URI uri, SharingHttpContext context) {
570 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
571 int index = 0;
572 for (; index < dirs.size(); index++) {
573 try (CloseableHttpResponse response =
574 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
575 try {
576 int status = response.getStatusLine().getStatusCode();
577 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
578 break;
579 } else if (status == HttpStatus.SC_CONFLICT) {
580 continue;
581 }
582 handleStatus(response);
583 } finally {
584 EntityUtils.consumeQuietly(response.getEntity());
585 }
586 } catch (IOException e) {
587 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
588 return;
589 }
590 }
591 for (index--; index >= 0; index--) {
592 try (CloseableHttpResponse response =
593 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
594 try {
595 handleStatus(response);
596 } finally {
597 EntityUtils.consumeQuietly(response.getEntity());
598 }
599 } catch (IOException e) {
600 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
601 return;
602 }
603 }
604 }
605
606 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
607 request.setEntity(entity);
608 return request;
609 }
610
611 private boolean isPayloadPresent(HttpUriRequest request) {
612 if (request instanceof HttpEntityEnclosingRequest) {
613 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
614 return entity != null && entity.getContentLength() != 0;
615 }
616 return false;
617 }
618
619 private <T extends HttpUriRequest> T commonHeaders(T request) {
620 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
621 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
622
623 if (state.isExpectContinue() && isPayloadPresent(request)) {
624 request.setHeader(HttpHeaders.EXPECT, "100-continue");
625 }
626
627 for (Map.Entry<?, ?> entry : headers.entrySet()) {
628 if (!(entry.getKey() instanceof String)) {
629 continue;
630 }
631 if (entry.getValue() instanceof String) {
632 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
633 } else {
634 request.removeHeaders(entry.getKey().toString());
635 }
636 }
637
638 if (!state.isExpectContinue()) {
639 request.removeHeaders(HttpHeaders.EXPECT);
640 }
641
642 return request;
643 }
644
645 private <T extends HttpUriRequest> void resume(T request, GetTask task) {
646 long resumeOffset = task.getResumeOffset();
647 if (resumeOffset > 0L && task.getDataFile() != null) {
648 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
649 request.setHeader(
650 HttpHeaders.IF_UNMODIFIED_SINCE,
651 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
652 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
653 }
654 }
655
656 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException, HttpRFC9457Exception {
657 int status = response.getStatusLine().getStatusCode();
658 if (status >= 300) {
659 if (RFC9457Reporter.INSTANCE.isRFC9457Message(response)) {
660 RFC9457Reporter.INSTANCE.generateException(response);
661 }
662 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
663 }
664 }
665
666 @Override
667 protected void implClose() {
668 try {
669 client.close();
670 } catch (IOException e) {
671 throw new UncheckedIOException(e);
672 }
673 AuthenticationContext.close(repoAuthContext);
674 AuthenticationContext.close(proxyAuthContext);
675 state.close();
676 }
677
678 private class EntityGetter {
679
680 private final GetTask task;
681
682 EntityGetter(GetTask task) {
683 this.task = task;
684 }
685
686 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
687 HttpEntity entity = response.getEntity();
688 if (entity == null) {
689 entity = new ByteArrayEntity(new byte[0]);
690 }
691
692 long offset = 0L, length = entity.getContentLength();
693 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
694 String range = rangeHeader != null ? rangeHeader.getValue() : null;
695 if (range != null) {
696 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
697 if (!m.matches()) {
698 throw new IOException("Invalid Content-Range header for partial download: " + range);
699 }
700 offset = Long.parseLong(m.group(1));
701 length = Long.parseLong(m.group(2)) + 1L;
702 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
703 throw new IOException("Invalid Content-Range header for partial download from offset "
704 + task.getResumeOffset() + ": " + range);
705 }
706 }
707
708 final boolean resume = offset > 0L;
709 final File dataFile = task.getDataFile();
710 if (dataFile == null) {
711 try (InputStream is = entity.getContent()) {
712 utilGet(task, is, true, length, resume);
713 extractChecksums(response);
714 }
715 } else {
716 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
717 task.setDataFile(tempFile.getPath().toFile(), resume);
718 if (resume && Files.isRegularFile(dataFile.toPath())) {
719 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
720 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
721 }
722 }
723 try (InputStream is = entity.getContent()) {
724 utilGet(task, is, true, length, resume);
725 }
726 tempFile.move();
727 } finally {
728 task.setDataFile(dataFile);
729 }
730 }
731 if (task.getDataFile() != null) {
732 Header lastModifiedHeader =
733 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
734 if (lastModifiedHeader != null) {
735 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
736 if (lastModified != null) {
737 task.getDataFile().setLastModified(lastModified.getTime());
738 }
739 }
740 }
741 extractChecksums(response);
742 }
743
744 private void extractChecksums(CloseableHttpResponse response) {
745 for (Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet()) {
746 Map<String, String> checksums = extractorEntry.getValue().extractChecksums(response);
747 if (checksums != null) {
748 checksums.forEach(task::setChecksum);
749 return;
750 }
751 }
752 }
753 }
754
755 private class PutTaskEntity extends AbstractHttpEntity {
756
757 private final PutTask task;
758
759 PutTaskEntity(PutTask task) {
760 this.task = task;
761 }
762
763 @Override
764 public boolean isRepeatable() {
765 return true;
766 }
767
768 @Override
769 public boolean isStreaming() {
770 return false;
771 }
772
773 @Override
774 public long getContentLength() {
775 return task.getDataLength();
776 }
777
778 @Override
779 public InputStream getContent() throws IOException {
780 return task.newInputStream();
781 }
782
783 @Override
784 public void writeTo(OutputStream os) throws IOException {
785 try {
786 utilPut(task, os, false);
787 } catch (TransferCancelledException e) {
788 throw (IOException) new InterruptedIOException().initCause(e);
789 }
790 }
791 }
792
793 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
794 private final int retryCount;
795
796 private final long retryInterval;
797
798 private final long retryIntervalMax;
799
800 private final Set<Integer> serviceUnavailableHttpCodes;
801
802
803
804
805
806
807 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
808
809 private ResolverServiceUnavailableRetryStrategy(
810 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
811 if (retryCount < 0) {
812 throw new IllegalArgumentException("retryCount must be >= 0");
813 }
814 if (retryInterval < 0L) {
815 throw new IllegalArgumentException("retryInterval must be >= 0");
816 }
817 if (retryIntervalMax < 0L) {
818 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
819 }
820 this.retryCount = retryCount;
821 this.retryInterval = retryInterval;
822 this.retryIntervalMax = retryIntervalMax;
823 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
824 }
825
826 @Override
827 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
828 final boolean retry = executionCount <= retryCount
829 && (serviceUnavailableHttpCodes.contains(
830 response.getStatusLine().getStatusCode()));
831 if (retry) {
832 Long retryInterval = retryInterval(response, executionCount);
833 if (retryInterval != null) {
834 RETRY_INTERVAL_HOLDER.set(retryInterval);
835 return true;
836 }
837 }
838 RETRY_INTERVAL_HOLDER.remove();
839 return false;
840 }
841
842
843
844
845
846
847
848
849 private Long retryInterval(HttpResponse httpResponse, int executionCount) {
850 Long result = null;
851 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
852 if (header != null && header.getValue() != null) {
853 String headerValue = header.getValue();
854 if (headerValue.contains(":")) {
855 Date when = DateUtils.parseDate(headerValue);
856 if (when != null) {
857 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
858 }
859 } else {
860 try {
861 result = Long.parseLong(headerValue) * 1000L;
862 } catch (NumberFormatException e) {
863
864 }
865 }
866 }
867 if (result == null) {
868 result = executionCount * this.retryInterval;
869 }
870 if (result > retryIntervalMax) {
871 return null;
872 }
873 return result;
874 }
875
876 @Override
877 public long getRetryInterval() {
878 Long ri = RETRY_INTERVAL_HOLDER.get();
879 if (ri == null) {
880 return 0L;
881 }
882 RETRY_INTERVAL_HOLDER.remove();
883 return ri;
884 }
885 }
886 }