1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.apache;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.InterruptedIOException;
24 import java.io.OutputStream;
25 import java.io.UncheckedIOException;
26 import java.net.InetAddress;
27 import java.net.URI;
28 import java.net.URISyntaxException;
29 import java.net.UnknownHostException;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Map;
39 import java.util.Set;
40 import java.util.function.Function;
41 import java.util.regex.Matcher;
42
43 import org.apache.http.Header;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpHeaders;
47 import org.apache.http.HttpHost;
48 import org.apache.http.HttpResponse;
49 import org.apache.http.HttpStatus;
50 import org.apache.http.auth.AuthSchemeProvider;
51 import org.apache.http.auth.AuthScope;
52 import org.apache.http.client.CredentialsProvider;
53 import org.apache.http.client.HttpRequestRetryHandler;
54 import org.apache.http.client.HttpResponseException;
55 import org.apache.http.client.ServiceUnavailableRetryStrategy;
56 import org.apache.http.client.config.AuthSchemes;
57 import org.apache.http.client.config.CookieSpecs;
58 import org.apache.http.client.config.RequestConfig;
59 import org.apache.http.client.methods.CloseableHttpResponse;
60 import org.apache.http.client.methods.HttpGet;
61 import org.apache.http.client.methods.HttpHead;
62 import org.apache.http.client.methods.HttpOptions;
63 import org.apache.http.client.methods.HttpPut;
64 import org.apache.http.client.methods.HttpUriRequest;
65 import org.apache.http.client.utils.DateUtils;
66 import org.apache.http.client.utils.URIUtils;
67 import org.apache.http.config.Registry;
68 import org.apache.http.config.RegistryBuilder;
69 import org.apache.http.config.SocketConfig;
70 import org.apache.http.entity.AbstractHttpEntity;
71 import org.apache.http.entity.ByteArrayEntity;
72 import org.apache.http.impl.NoConnectionReuseStrategy;
73 import org.apache.http.impl.auth.BasicScheme;
74 import org.apache.http.impl.auth.BasicSchemeFactory;
75 import org.apache.http.impl.auth.DigestSchemeFactory;
76 import org.apache.http.impl.auth.KerberosSchemeFactory;
77 import org.apache.http.impl.auth.NTLMSchemeFactory;
78 import org.apache.http.impl.auth.SPNegoSchemeFactory;
79 import org.apache.http.impl.client.CloseableHttpClient;
80 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
81 import org.apache.http.impl.client.HttpClientBuilder;
82 import org.apache.http.impl.client.LaxRedirectStrategy;
83 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
84 import org.apache.http.protocol.HttpContext;
85 import org.apache.http.util.EntityUtils;
86 import org.eclipse.aether.ConfigurationProperties;
87 import org.eclipse.aether.RepositorySystemSession;
88 import org.eclipse.aether.repository.AuthenticationContext;
89 import org.eclipse.aether.repository.Proxy;
90 import org.eclipse.aether.repository.RemoteRepository;
91 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
92 import org.eclipse.aether.spi.connector.transport.GetTask;
93 import org.eclipse.aether.spi.connector.transport.PeekTask;
94 import org.eclipse.aether.spi.connector.transport.PutTask;
95 import org.eclipse.aether.spi.connector.transport.TransportTask;
96 import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
97 import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
98 import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
99 import org.eclipse.aether.spi.io.PathProcessor;
100 import org.eclipse.aether.transfer.NoTransporterException;
101 import org.eclipse.aether.transfer.TransferCancelledException;
102 import org.eclipse.aether.util.ConfigUtils;
103 import org.eclipse.aether.util.FileUtils;
104 import org.slf4j.Logger;
105 import org.slf4j.LoggerFactory;
106
107 import static java.util.Objects.requireNonNull;
108 import static org.eclipse.aether.spi.connector.transport.http.HttpConstants.CONTENT_RANGE_PATTERN;
109 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_FOLLOW_REDIRECTS;
110 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
111 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
112 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_MAX_REDIRECTS;
113 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
114 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_FOLLOW_REDIRECTS;
115 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
116 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_MAX_REDIRECTS;
117 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
118 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
119 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
120
121
122
123
124 final class ApacheTransporter extends AbstractTransporter implements HttpTransporter {
125 private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
126
127 private final ChecksumExtractor checksumExtractor;
128
129 private final PathProcessor pathProcessor;
130
131 private final AuthenticationContext repoAuthContext;
132
133 private final AuthenticationContext proxyAuthContext;
134
135 private final URI baseUri;
136
137 private final HttpHost server;
138
139 private final HttpHost proxy;
140
141 private final CloseableHttpClient client;
142
143 private final Map<?, ?> headers;
144
145 private final LocalState state;
146
147 private final boolean preemptiveAuth;
148
149 private final boolean preemptivePutAuth;
150
151 private final boolean supportWebDav;
152
153 @SuppressWarnings("checkstyle:methodlength")
154 ApacheTransporter(
155 RemoteRepository repository,
156 RepositorySystemSession session,
157 ChecksumExtractor checksumExtractor,
158 PathProcessor pathProcessor)
159 throws NoTransporterException {
160 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
161 throw new NoTransporterException(repository);
162 }
163 this.checksumExtractor = checksumExtractor;
164 this.pathProcessor = pathProcessor;
165 try {
166 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
167 if (baseUri.isOpaque()) {
168 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
169 }
170 this.server = URIUtils.extractHost(baseUri);
171 if (server == null) {
172 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
173 }
174 } catch (URISyntaxException e) {
175 throw new NoTransporterException(repository, e.getMessage(), e);
176 }
177 this.proxy = toHost(repository.getProxy());
178
179 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
180 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
181
182 String httpsSecurityMode = ConfigUtils.getString(
183 session,
184 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
185 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
186 ConfigurationProperties.HTTPS_SECURITY_MODE);
187 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
188 session,
189 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
190 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
191 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
192 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
193 session,
194 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
195 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
196 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
197 this.state = new LocalState(
198 session,
199 repository,
200 new ConnMgrConfig(
201 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
202
203 this.headers = ConfigUtils.getMap(
204 session,
205 Collections.emptyMap(),
206 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
207 ConfigurationProperties.HTTP_HEADERS);
208
209 this.preemptiveAuth = ConfigUtils.getBoolean(
210 session,
211 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
212 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
213 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
214 this.preemptivePutAuth = ConfigUtils.getBoolean(
215 session,
216 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_PUT_AUTH,
217 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH + "." + repository.getId(),
218 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH);
219 this.supportWebDav = ConfigUtils.getBoolean(
220 session,
221 ConfigurationProperties.DEFAULT_HTTP_SUPPORT_WEBDAV,
222 ConfigurationProperties.HTTP_SUPPORT_WEBDAV + "." + repository.getId(),
223 ConfigurationProperties.HTTP_SUPPORT_WEBDAV);
224 String credentialEncoding = ConfigUtils.getString(
225 session,
226 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
227 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
228 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
229 int connectTimeout = ConfigUtils.getInteger(
230 session,
231 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
232 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
233 ConfigurationProperties.CONNECT_TIMEOUT);
234 int requestTimeout = ConfigUtils.getInteger(
235 session,
236 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
237 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
238 ConfigurationProperties.REQUEST_TIMEOUT);
239 int retryCount = ConfigUtils.getInteger(
240 session,
241 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
242 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
243 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
244 long retryInterval = ConfigUtils.getLong(
245 session,
246 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
247 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
248 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
249 long retryIntervalMax = ConfigUtils.getLong(
250 session,
251 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
252 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
253 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
254 String serviceUnavailableCodesString = ConfigUtils.getString(
255 session,
256 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
257 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
258 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
259 String retryHandlerName = ConfigUtils.getString(
260 session,
261 HTTP_RETRY_HANDLER_NAME_STANDARD,
262 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
263 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
264 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
265 session,
266 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
267 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
268 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
269 int maxRedirects = ConfigUtils.getInteger(
270 session,
271 DEFAULT_MAX_REDIRECTS,
272 CONFIG_PROP_MAX_REDIRECTS + "." + repository.getId(),
273 CONFIG_PROP_MAX_REDIRECTS);
274 boolean followRedirects = ConfigUtils.getBoolean(
275 session,
276 DEFAULT_FOLLOW_REDIRECTS,
277 CONFIG_PROP_FOLLOW_REDIRECTS + "." + repository.getId(),
278 CONFIG_PROP_FOLLOW_REDIRECTS);
279 String userAgent = ConfigUtils.getString(
280 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
281
282 Charset credentialsCharset = Charset.forName(credentialEncoding);
283 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
284 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
285 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
286 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
287 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
288 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
289 .build();
290 SocketConfig socketConfig =
291
292 SocketConfig.custom().setSoTimeout(requestTimeout).build();
293 RequestConfig requestConfig = RequestConfig.custom()
294 .setMaxRedirects(maxRedirects)
295 .setRedirectsEnabled(followRedirects)
296 .setRelativeRedirectsAllowed(followRedirects)
297
298 .setSocketTimeout(requestTimeout)
299
300 .setConnectTimeout(connectTimeout)
301
302 .setConnectionRequestTimeout(connectTimeout)
303 .setLocalAddress(getHttpLocalAddress(session, repository))
304 .setCookieSpec(CookieSpecs.STANDARD)
305 .build();
306
307 HttpRequestRetryHandler retryHandler;
308 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
309 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
310 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
311 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
312 } else {
313 throw new IllegalArgumentException(
314 "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
315 }
316 Set<Integer> serviceUnavailableCodes = new HashSet<>();
317 try {
318 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
319 serviceUnavailableCodes.add(Integer.parseInt(code));
320 }
321 } catch (NumberFormatException e) {
322 throw new IllegalArgumentException(
323 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
324 + " (list of integers): " + serviceUnavailableCodesString);
325 }
326 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
327 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
328
329 HttpClientBuilder builder = HttpClientBuilder.create()
330 .setUserAgent(userAgent)
331 .setRedirectStrategy(LaxRedirectStrategy.INSTANCE)
332 .setDefaultSocketConfig(socketConfig)
333 .setDefaultRequestConfig(requestConfig)
334 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
335 .setRetryHandler(retryHandler)
336 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
337 .setConnectionManager(state.getConnectionManager())
338 .setConnectionManagerShared(true)
339 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
340 .setProxy(proxy);
341 final boolean useSystemProperties = ConfigUtils.getBoolean(
342 session,
343 DEFAULT_USE_SYSTEM_PROPERTIES,
344 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
345 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
346 if (useSystemProperties) {
347 LOGGER.warn(
348 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
349 LOGGER.warn("Please use documented means to configure resolver transport.");
350 builder.useSystemProperties();
351 }
352
353 final String expectContinue = ConfigUtils.getString(
354 session,
355 null,
356 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
357 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
358 if (expectContinue != null) {
359 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
360 }
361
362 final boolean reuseConnections = ConfigUtils.getBoolean(
363 session,
364 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
365 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
366 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
367 if (!reuseConnections) {
368 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
369 }
370
371 this.client = builder.build();
372 }
373
374
375
376
377 private InetAddress getHttpLocalAddress(RepositorySystemSession session, RemoteRepository repository) {
378 String bindAddress = ConfigUtils.getString(
379 session,
380 null,
381 ConfigurationProperties.HTTP_LOCAL_ADDRESS + "." + repository.getId(),
382 ConfigurationProperties.HTTP_LOCAL_ADDRESS);
383 if (bindAddress == null) {
384 return null;
385 }
386 try {
387 return InetAddress.getByName(bindAddress);
388 } catch (UnknownHostException uhe) {
389 throw new IllegalArgumentException(
390 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
391 uhe);
392 }
393 }
394
395 private static HttpHost toHost(Proxy proxy) {
396 HttpHost host = null;
397 if (proxy != null) {
398 host = new HttpHost(proxy.getHost(), proxy.getPort());
399 }
400 return host;
401 }
402
403 private static CredentialsProvider toCredentialsProvider(
404 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
405 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
406 if (proxy != null) {
407 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
408 provider = new DemuxCredentialsProvider(provider, p, proxy);
409 }
410 return provider;
411 }
412
413 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
414 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
415 if (ctx != null) {
416 AuthScope basicScope = new AuthScope(host, port);
417 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
418
419 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
420 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
421 }
422 return provider;
423 }
424
425 LocalState getState() {
426 return state;
427 }
428
429 private URI resolve(TransportTask task) {
430 return UriUtils.resolve(baseUri, task.getLocation());
431 }
432
433 @Override
434 public int classify(Throwable error) {
435 if (error instanceof HttpTransporterException
436 && ((HttpTransporterException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
437 return ERROR_NOT_FOUND;
438 }
439 return ERROR_OTHER;
440 }
441
442 @Override
443 protected void implPeek(PeekTask task) throws Exception {
444 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
445 try {
446 execute(request, null);
447 } catch (HttpResponseException e) {
448 throw new HttpTransporterException(e.getStatusCode());
449 }
450 }
451
452 @Override
453 protected void implGet(GetTask task) throws Exception {
454 boolean resume = true;
455
456 EntityGetter getter = new EntityGetter(task);
457 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
458 while (true) {
459 try {
460 if (resume) {
461 resume(request, task);
462 }
463 execute(request, getter);
464 break;
465 } catch (HttpResponseException e) {
466 if (resume
467 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
468 && request.containsHeader(HttpHeaders.RANGE)) {
469 request = commonHeaders(new HttpGet(resolve(task)));
470 resume = false;
471 continue;
472 }
473 throw new HttpTransporterException(e.getStatusCode());
474 }
475 }
476 }
477
478 @Override
479 protected void implPut(PutTask task) throws Exception {
480 PutTaskEntity entity = new PutTaskEntity(task);
481 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
482 try {
483 execute(request, null);
484 } catch (HttpResponseException e) {
485 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
486 state.setExpectContinue(false);
487 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
488 execute(request, null);
489 return;
490 }
491 throw new HttpTransporterException(e.getStatusCode());
492 }
493 }
494
495 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
496 try {
497 SharingHttpContext context = new SharingHttpContext(state);
498 prepare(request, context);
499 try (CloseableHttpResponse response = client.execute(server, request, context)) {
500 try {
501 context.close();
502 handleStatus(response);
503 if (getter != null) {
504 getter.handle(response);
505 }
506 } finally {
507 EntityUtils.consumeQuietly(response.getEntity());
508 }
509 }
510 } catch (IOException e) {
511 if (e.getCause() instanceof TransferCancelledException) {
512 throw (Exception) e.getCause();
513 }
514 throw e;
515 }
516 }
517
518 private void prepare(HttpUriRequest request, SharingHttpContext context) throws Exception {
519 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
520 if (preemptiveAuth || (preemptivePutAuth && put)) {
521 context.getAuthCache().put(server, new BasicScheme());
522 }
523 if (supportWebDav) {
524 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
525 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
526 try (CloseableHttpResponse response = client.execute(server, req, context)) {
527 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
528 EntityUtils.consumeQuietly(response.getEntity());
529 } catch (IOException e) {
530 LOGGER.debug("Failed to prepare HTTP context", e);
531 }
532 }
533 if (put && Boolean.TRUE.equals(state.getWebDav())) {
534 mkdirs(request.getURI(), context);
535 }
536 }
537 }
538
539 @SuppressWarnings("checkstyle:magicnumber")
540 private void mkdirs(URI uri, SharingHttpContext context) throws Exception {
541 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
542 int index = 0;
543 for (; index < dirs.size(); index++) {
544 try (CloseableHttpResponse response =
545 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
546 try {
547 int status = response.getStatusLine().getStatusCode();
548 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
549 break;
550 } else if (status == HttpStatus.SC_CONFLICT) {
551 continue;
552 }
553 handleStatus(response);
554 } finally {
555 EntityUtils.consumeQuietly(response.getEntity());
556 }
557 } catch (IOException e) {
558 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
559 return;
560 }
561 }
562 for (index--; index >= 0; index--) {
563 try (CloseableHttpResponse response =
564 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
565 try {
566 handleStatus(response);
567 } finally {
568 EntityUtils.consumeQuietly(response.getEntity());
569 }
570 } catch (IOException e) {
571 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
572 return;
573 }
574 }
575 }
576
577 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
578 request.setEntity(entity);
579 return request;
580 }
581
582 private boolean isPayloadPresent(HttpUriRequest request) {
583 if (request instanceof HttpEntityEnclosingRequest) {
584 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
585 return entity != null && entity.getContentLength() != 0;
586 }
587 return false;
588 }
589
590 private <T extends HttpUriRequest> T commonHeaders(T request) {
591 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
592 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
593
594 if (state.isExpectContinue() && isPayloadPresent(request)) {
595 request.setHeader(HttpHeaders.EXPECT, "100-continue");
596 }
597
598 for (Map.Entry<?, ?> entry : headers.entrySet()) {
599 if (!(entry.getKey() instanceof String)) {
600 continue;
601 }
602 if (entry.getValue() instanceof String) {
603 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
604 } else {
605 request.removeHeaders(entry.getKey().toString());
606 }
607 }
608
609 if (!state.isExpectContinue()) {
610 request.removeHeaders(HttpHeaders.EXPECT);
611 }
612
613 return request;
614 }
615
616 @SuppressWarnings("checkstyle:magicnumber")
617 private <T extends HttpUriRequest> void resume(T request, GetTask task) throws IOException {
618 long resumeOffset = task.getResumeOffset();
619 if (resumeOffset > 0L && task.getDataPath() != null) {
620 long lastModified = Files.getLastModifiedTime(task.getDataPath()).toMillis();
621 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
622 request.setHeader(
623 HttpHeaders.IF_UNMODIFIED_SINCE, DateUtils.formatDate(new Date(lastModified - 60L * 1000L)));
624 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
625 }
626 }
627
628 @SuppressWarnings("checkstyle:magicnumber")
629 private void handleStatus(CloseableHttpResponse response) throws Exception {
630 int status = response.getStatusLine().getStatusCode();
631 if (status >= 300) {
632 ApacheRFC9457Reporter.INSTANCE.generateException(response, (statusCode, reasonPhrase) -> {
633 throw new HttpResponseException(statusCode, reasonPhrase + " (" + statusCode + ")");
634 });
635 }
636 }
637
638 @Override
639 protected void implClose() {
640 try {
641 client.close();
642 } catch (IOException e) {
643 throw new UncheckedIOException(e);
644 }
645 AuthenticationContext.close(repoAuthContext);
646 AuthenticationContext.close(proxyAuthContext);
647 state.close();
648 }
649
650 private class EntityGetter {
651
652 private final GetTask task;
653
654 EntityGetter(GetTask task) {
655 this.task = task;
656 }
657
658 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
659 HttpEntity entity = response.getEntity();
660 if (entity == null) {
661 entity = new ByteArrayEntity(new byte[0]);
662 }
663
664 long offset = 0L, length = entity.getContentLength();
665 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
666 String range = rangeHeader != null ? rangeHeader.getValue() : null;
667 if (range != null) {
668 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
669 if (!m.matches()) {
670 throw new IOException("Invalid Content-Range header for partial download: " + range);
671 }
672 offset = Long.parseLong(m.group(1));
673 length = Long.parseLong(m.group(2)) + 1L;
674 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
675 throw new IOException("Invalid Content-Range header for partial download from offset "
676 + task.getResumeOffset() + ": " + range);
677 }
678 }
679
680 final boolean resume = offset > 0L;
681 final Path dataFile = task.getDataPath();
682 if (dataFile == null) {
683 try (InputStream is = entity.getContent()) {
684 utilGet(task, is, true, length, resume);
685 extractChecksums(response);
686 }
687 } else {
688 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile)) {
689 task.setDataPath(tempFile.getPath(), resume);
690 if (resume && Files.isRegularFile(dataFile)) {
691 try (InputStream inputStream = Files.newInputStream(dataFile)) {
692 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
693 }
694 }
695 try (InputStream is = entity.getContent()) {
696 utilGet(task, is, true, length, resume);
697 }
698 tempFile.move();
699 } finally {
700 task.setDataPath(dataFile);
701 }
702 }
703 if (task.getDataPath() != null) {
704 Header lastModifiedHeader =
705 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
706 if (lastModifiedHeader != null) {
707 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
708 if (lastModified != null) {
709 pathProcessor.setLastModified(task.getDataPath(), lastModified.getTime());
710 }
711 }
712 }
713 extractChecksums(response);
714 }
715
716 private void extractChecksums(CloseableHttpResponse response) {
717 Map<String, String> checksums = checksumExtractor.extractChecksums(headerGetter(response));
718 if (checksums != null && !checksums.isEmpty()) {
719 checksums.forEach(task::setChecksum);
720 }
721 }
722 }
723
724 private static Function<String, String> headerGetter(CloseableHttpResponse closeableHttpResponse) {
725 return s -> {
726 Header header = closeableHttpResponse.getFirstHeader(s);
727 return header != null ? header.getValue() : null;
728 };
729 }
730
731 private class PutTaskEntity extends AbstractHttpEntity {
732
733 private final PutTask task;
734
735 PutTaskEntity(PutTask task) {
736 this.task = task;
737 }
738
739 @Override
740 public boolean isRepeatable() {
741 return true;
742 }
743
744 @Override
745 public boolean isStreaming() {
746 return false;
747 }
748
749 @Override
750 public long getContentLength() {
751 return task.getDataLength();
752 }
753
754 @Override
755 public InputStream getContent() throws IOException {
756 return task.newInputStream();
757 }
758
759 @Override
760 public void writeTo(OutputStream os) throws IOException {
761 try {
762 utilPut(task, os, false);
763 } catch (TransferCancelledException e) {
764 throw (IOException) new InterruptedIOException().initCause(e);
765 }
766 }
767 }
768
769 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
770 private final int retryCount;
771
772 private final long retryInterval;
773
774 private final long retryIntervalMax;
775
776 private final Set<Integer> serviceUnavailableHttpCodes;
777
778
779
780
781
782
783 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
784
785 private ResolverServiceUnavailableRetryStrategy(
786 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
787 if (retryCount < 0) {
788 throw new IllegalArgumentException("retryCount must be >= 0");
789 }
790 if (retryInterval < 0L) {
791 throw new IllegalArgumentException("retryInterval must be >= 0");
792 }
793 if (retryIntervalMax < 0L) {
794 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
795 }
796 this.retryCount = retryCount;
797 this.retryInterval = retryInterval;
798 this.retryIntervalMax = retryIntervalMax;
799 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
800 }
801
802 @Override
803 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
804 final boolean retry = executionCount <= retryCount
805 && (serviceUnavailableHttpCodes.contains(
806 response.getStatusLine().getStatusCode()));
807 if (retry) {
808 Long retryInterval = retryInterval(response, executionCount, context);
809 if (retryInterval != null) {
810 RETRY_INTERVAL_HOLDER.set(retryInterval);
811 return true;
812 }
813 }
814 RETRY_INTERVAL_HOLDER.remove();
815 return false;
816 }
817
818
819
820
821
822
823
824
825 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
826 Long result = null;
827 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
828 if (header != null && header.getValue() != null) {
829 String headerValue = header.getValue();
830 if (headerValue.contains(":")) {
831 Date when = DateUtils.parseDate(headerValue);
832 if (when != null) {
833 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
834 }
835 } else {
836 try {
837 result = Long.parseLong(headerValue) * 1000L;
838 } catch (NumberFormatException e) {
839
840 }
841 }
842 }
843 if (result == null) {
844 result = executionCount * this.retryInterval;
845 }
846 if (result > retryIntervalMax) {
847 return null;
848 }
849 return result;
850 }
851
852 @Override
853 public long getRetryInterval() {
854 Long ri = RETRY_INTERVAL_HOLDER.get();
855 if (ri == null) {
856 return 0L;
857 }
858 RETRY_INTERVAL_HOLDER.remove();
859 return ri;
860 }
861 }
862 }