1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.apache;
20
21 import javax.net.ssl.SSLSession;
22
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.InterruptedIOException;
26 import java.io.OutputStream;
27 import java.io.UncheckedIOException;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Date;
35 import java.util.List;
36 import java.util.Map;
37 import java.util.Set;
38 import java.util.concurrent.ConcurrentHashMap;
39 import java.util.function.Function;
40 import java.util.regex.Matcher;
41
42 import org.apache.http.Header;
43 import org.apache.http.HttpClientConnection;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpException;
47 import org.apache.http.HttpHeaders;
48 import org.apache.http.HttpHost;
49 import org.apache.http.HttpRequest;
50 import org.apache.http.HttpResponse;
51 import org.apache.http.HttpStatus;
52 import org.apache.http.ProtocolVersion;
53 import org.apache.http.auth.AuthScheme;
54 import org.apache.http.auth.AuthSchemeProvider;
55 import org.apache.http.auth.AuthScope;
56 import org.apache.http.client.AuthCache;
57 import org.apache.http.client.CredentialsProvider;
58 import org.apache.http.client.HttpRequestRetryHandler;
59 import org.apache.http.client.HttpResponseException;
60 import org.apache.http.client.ServiceUnavailableRetryStrategy;
61 import org.apache.http.client.config.AuthSchemes;
62 import org.apache.http.client.config.CookieSpecs;
63 import org.apache.http.client.config.RequestConfig;
64 import org.apache.http.client.methods.CloseableHttpResponse;
65 import org.apache.http.client.methods.HttpGet;
66 import org.apache.http.client.methods.HttpHead;
67 import org.apache.http.client.methods.HttpOptions;
68 import org.apache.http.client.methods.HttpPut;
69 import org.apache.http.client.methods.HttpUriRequest;
70 import org.apache.http.client.utils.DateUtils;
71 import org.apache.http.client.utils.URIUtils;
72 import org.apache.http.config.Registry;
73 import org.apache.http.config.RegistryBuilder;
74 import org.apache.http.config.SocketConfig;
75 import org.apache.http.conn.ManagedHttpClientConnection;
76 import org.apache.http.entity.AbstractHttpEntity;
77 import org.apache.http.entity.ByteArrayEntity;
78 import org.apache.http.impl.NoConnectionReuseStrategy;
79 import org.apache.http.impl.auth.BasicScheme;
80 import org.apache.http.impl.auth.BasicSchemeFactory;
81 import org.apache.http.impl.auth.DigestSchemeFactory;
82 import org.apache.http.impl.auth.KerberosSchemeFactory;
83 import org.apache.http.impl.auth.NTLMSchemeFactory;
84 import org.apache.http.impl.auth.SPNegoSchemeFactory;
85 import org.apache.http.impl.client.CloseableHttpClient;
86 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
87 import org.apache.http.impl.client.HttpClientBuilder;
88 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
89 import org.apache.http.protocol.HttpContext;
90 import org.apache.http.protocol.HttpCoreContext;
91 import org.apache.http.protocol.HttpRequestExecutor;
92 import org.apache.http.util.EntityUtils;
93 import org.eclipse.aether.Keys;
94 import org.eclipse.aether.RepositorySystemSession;
95 import org.eclipse.aether.repository.AuthenticationContext;
96 import org.eclipse.aether.repository.Proxy;
97 import org.eclipse.aether.repository.RemoteRepository;
98 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
99 import org.eclipse.aether.spi.connector.transport.GetTask;
100 import org.eclipse.aether.spi.connector.transport.PeekTask;
101 import org.eclipse.aether.spi.connector.transport.PutTask;
102 import org.eclipse.aether.spi.connector.transport.TransportListener;
103 import org.eclipse.aether.spi.connector.transport.TransportTask;
104 import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
105 import org.eclipse.aether.spi.connector.transport.http.HttpTransportPropertiesBuilder;
106 import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
107 import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
108 import org.eclipse.aether.spi.io.PathProcessor;
109 import org.eclipse.aether.transfer.HttpTransportProperty.HttpVersion;
110 import org.eclipse.aether.transfer.NoTransporterException;
111 import org.eclipse.aether.transfer.TransferCancelledException;
112 import org.eclipse.aether.transfer.TransferEvent;
113 import org.eclipse.aether.util.ConfigUtils;
114 import org.eclipse.aether.util.StringDigestUtil;
115 import org.eclipse.aether.util.connector.transport.http.HttpTransporterUtils;
116 import org.slf4j.Logger;
117 import org.slf4j.LoggerFactory;
118
119 import static java.util.Objects.requireNonNull;
120 import static org.eclipse.aether.spi.connector.transport.http.HttpConstants.CONTENT_RANGE_PATTERN;
121 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_FOLLOW_INSECURE_REDIRECTS;
122 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_FOLLOW_REDIRECTS;
123 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
124 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
125 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_MAX_REDIRECTS;
126 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
127 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_FOLLOW_INSECURE_REDIRECTS;
128 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_FOLLOW_REDIRECTS;
129 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
130 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_MAX_REDIRECTS;
131 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
132 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
133 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
134
135
136
137
138 final class ApacheTransporter extends AbstractTransporter implements HttpTransporter {
139
140
141
142 private static final String CONTEXT_ATTRIBUTE_NAME_SSL_SESSION = "ssl.session";
143
144 private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
145
146 private final ChecksumExtractor checksumExtractor;
147
148 private final PathProcessor pathProcessor;
149
150 private final AuthenticationContext repoAuthContext;
151
152 private final AuthenticationContext proxyAuthContext;
153
154 private final URI baseUri;
155
156 private final HttpHost server;
157
158 private final HttpHost proxy;
159
160 private final CloseableHttpClient client;
161
162 private final Map<?, ?> headers;
163
164 private final LocalState state;
165
166 private final boolean preemptiveAuth;
167
168 private final boolean preemptivePutAuth;
169
170 private final boolean supportWebDav;
171
172 private final boolean sendRfc9457Accept;
173
174 private final AuthCache authCache;
175
176 @SuppressWarnings("checkstyle:methodlength")
177 ApacheTransporter(
178 RemoteRepository repository,
179 RepositorySystemSession session,
180 ChecksumExtractor checksumExtractor,
181 PathProcessor pathProcessor)
182 throws NoTransporterException {
183 this.checksumExtractor = checksumExtractor;
184 this.pathProcessor = pathProcessor;
185 try {
186 this.baseUri = HttpTransporterUtils.getBaseUri(repository);
187 this.server = URIUtils.extractHost(baseUri);
188 if (server == null) {
189 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
190 }
191 } catch (URISyntaxException e) {
192 throw new NoTransporterException(repository, e.getMessage(), e);
193 }
194 this.proxy = toHost(repository.getProxy());
195
196 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
197 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
198
199 String httpsSecurityMode = HttpTransporterUtils.getHttpsSecurityMode(session, repository);
200 final int connectionMaxTtlSeconds = HttpTransporterUtils.getHttpConnectionMaxTtlSeconds(session, repository);
201 final int maxConnectionsPerRoute = HttpTransporterUtils.getHttpMaxConnectionsPerRoute(session, repository);
202 this.state = new LocalState(
203 session,
204 repository,
205 new ConnMgrConfig(
206 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
207
208 this.headers = HttpTransporterUtils.getHttpHeaders(session, repository);
209 this.preemptiveAuth = HttpTransporterUtils.isHttpPreemptiveAuth(session, repository);
210 this.preemptivePutAuth = HttpTransporterUtils.isHttpPreemptivePutAuth(session, repository);
211 this.supportWebDav = HttpTransporterUtils.isHttpSupportWebDav(session, repository);
212 this.sendRfc9457Accept = HttpTransporterUtils.isHttpSendRfc9457Accept(session, repository);
213 int connectTimeout = HttpTransporterUtils.getHttpConnectTimeout(session, repository);
214 int requestTimeout = HttpTransporterUtils.getHttpRequestTimeout(session, repository);
215 int retryCount = HttpTransporterUtils.getHttpRetryHandlerCount(session, repository);
216 long retryInterval = HttpTransporterUtils.getHttpRetryHandlerInterval(session, repository);
217 long retryIntervalMax = HttpTransporterUtils.getHttpRetryHandlerIntervalMax(session, repository);
218 String retryHandlerName = ConfigUtils.getString(
219 session,
220 HTTP_RETRY_HANDLER_NAME_STANDARD,
221 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
222 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
223 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
224 session,
225 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
226 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
227 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
228 int maxRedirects = ConfigUtils.getInteger(
229 session,
230 DEFAULT_MAX_REDIRECTS,
231 CONFIG_PROP_MAX_REDIRECTS + "." + repository.getId(),
232 CONFIG_PROP_MAX_REDIRECTS);
233 boolean followRedirects = ConfigUtils.getBoolean(
234 session,
235 DEFAULT_FOLLOW_REDIRECTS,
236 CONFIG_PROP_FOLLOW_REDIRECTS + "." + repository.getId(),
237 CONFIG_PROP_FOLLOW_REDIRECTS);
238 boolean followInsecureRedirects = ConfigUtils.getBoolean(
239 session,
240 DEFAULT_FOLLOW_INSECURE_REDIRECTS,
241 CONFIG_PROP_FOLLOW_INSECURE_REDIRECTS + "." + repository.getId(),
242 CONFIG_PROP_FOLLOW_INSECURE_REDIRECTS);
243 String userAgent = HttpTransporterUtils.getUserAgent(session, repository);
244
245 Charset credentialsCharset = HttpTransporterUtils.getHttpCredentialsEncoding(session, repository);
246 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
247 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
248 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
249 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
250 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
251 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
252 .build();
253 SocketConfig socketConfig =
254
255 SocketConfig.custom().setSoTimeout(requestTimeout).build();
256 RequestConfig requestConfig = RequestConfig.custom()
257 .setMaxRedirects(maxRedirects)
258 .setRedirectsEnabled(followRedirects)
259 .setRelativeRedirectsAllowed(followRedirects)
260
261 .setSocketTimeout(requestTimeout)
262
263 .setConnectTimeout(connectTimeout)
264
265 .setConnectionRequestTimeout(connectTimeout)
266 .setLocalAddress(HttpTransporterUtils.getHttpLocalAddress(session, repository)
267 .orElse(null))
268 .setCookieSpec(CookieSpecs.STANDARD)
269 .build();
270
271 HttpRequestRetryHandler retryHandler;
272 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
273 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
274 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
275 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
276 } else {
277 throw new IllegalArgumentException(
278 "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
279 }
280 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
281 retryCount,
282 retryInterval,
283 retryIntervalMax,
284 HttpTransporterUtils.getHttpServiceUnavailableCodes(session, repository));
285
286 HttpClientBuilder builder = HttpClientBuilder.create()
287 .setUserAgent(userAgent)
288 .setRedirectStrategy(new ResolverRedirectStrategy(followInsecureRedirects))
289 .setDefaultSocketConfig(socketConfig)
290 .setDefaultRequestConfig(requestConfig)
291 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
292 .setRetryHandler(retryHandler)
293 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
294 .setConnectionManager(state.getConnectionManager())
295 .setConnectionManagerShared(true)
296 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
297 .setProxy(proxy);
298 if (ConfigUtils.getBoolean(
299 session,
300 ApacheTransporterConfigurationKeys.DEFAULT_ORIGIN_SCOPED_HEADERS,
301 ApacheTransporterConfigurationKeys.CONFIG_PROP_ORIGIN_SCOPED_HEADERS + "." + repository.getId(),
302 ApacheTransporterConfigurationKeys.CONFIG_PROP_ORIGIN_SCOPED_HEADERS)) {
303
304
305
306 builder.addInterceptorLast(new OriginScopedHeadersInterceptor(server, this.headers.keySet()));
307 }
308 final boolean useSystemProperties = ConfigUtils.getBoolean(
309 session,
310 DEFAULT_USE_SYSTEM_PROPERTIES,
311 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
312 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
313 if (useSystemProperties) {
314 LOGGER.warn(
315 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
316 LOGGER.warn("Please use documented means to configure resolver transport.");
317 builder.useSystemProperties();
318 SystemProxyAuthenticationStrategy systemProxyAuth = new SystemProxyAuthenticationStrategy();
319 builder.setProxyAuthenticationStrategy(systemProxyAuth);
320 builder.addInterceptorFirst(systemProxyAuth);
321 }
322
323
324 builder.setRequestExecutor(new HttpRequestExecutor() {
325
326 @Override
327 public HttpResponse execute(HttpRequest request, HttpClientConnection conn, HttpContext context)
328 throws IOException, HttpException {
329 if (conn instanceof ManagedHttpClientConnection) {
330 context.setAttribute(
331 CONTEXT_ATTRIBUTE_NAME_SSL_SESSION, ((ManagedHttpClientConnection) conn).getSSLSession());
332 }
333 return super.execute(request, conn, context);
334 }
335 });
336
337 HttpTransporterUtils.getHttpExpectContinue(session, repository).ifPresent(state::setExpectContinue);
338 if (!HttpTransporterUtils.isHttpReuseConnections(session, repository)) {
339 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
340 }
341
342 if (session.getCache() != null) {
343 this.authCache = (AuthCache) session.getCache()
344 .computeIfAbsent(
345 session,
346 Keys.of(
347 getClass(),
348 repository.getId() + "-" + StringDigestUtil.sha1(repository.toString())),
349 ConcurrentAuthCache::new);
350 } else {
351 this.authCache = new ConcurrentAuthCache();
352 }
353 this.client = builder.build();
354 }
355
356 private static HttpHost toHost(Proxy proxy) {
357 HttpHost host = null;
358 if (proxy != null) {
359
360
361
362
363 host = new HttpHost(proxy.getHost(), proxy.getPort());
364 }
365 return host;
366 }
367
368 private static CredentialsProvider toCredentialsProvider(
369 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
370 CredentialsProvider provider =
371 toCredentialsProvider(server.getHostName(), effectivePort(server), serverAuthCtx);
372 if (proxy != null) {
373 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
374 provider = new DemuxCredentialsProvider(provider, p, proxy);
375 }
376 return provider;
377 }
378
379
380
381
382
383
384
385
386 static int effectivePort(HttpHost host) {
387 if (host.getPort() >= 0) {
388 return host.getPort();
389 }
390 return "https".equalsIgnoreCase(host.getSchemeName()) ? 443 : 80;
391 }
392
393 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
394 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
395 if (ctx != null) {
396 AuthScope basicScope = new AuthScope(host, port);
397 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
398
399 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
400 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
401 }
402 return provider;
403 }
404
405 LocalState getState() {
406 return state;
407 }
408
409 private URI resolve(TransportTask task) {
410 return UriUtils.resolve(baseUri, task.getLocation());
411 }
412
413 @Override
414 protected void implPeek(PeekTask task) throws Exception {
415 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
416 try {
417 execute(request, null, task.getListener());
418 } catch (HttpResponseException e) {
419 throw new HttpTransporterException(e.getStatusCode());
420 }
421 }
422
423 @Override
424 protected void implGet(GetTask task) throws Exception {
425 boolean resume = true;
426
427 EntityGetter getter = new EntityGetter(task);
428 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
429 if (sendRfc9457Accept) {
430 ApacheRFC9457Reporter.INSTANCE.prepareRequest(request);
431 }
432 while (true) {
433 try {
434 if (resume) {
435 resume(request, task);
436 }
437 execute(request, getter, task.getListener());
438 break;
439 } catch (HttpResponseException e) {
440 if (resume
441 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
442 && request.containsHeader(HttpHeaders.RANGE)) {
443 request = commonHeaders(new HttpGet(resolve(task)));
444 resume = false;
445 continue;
446 }
447 throw new HttpTransporterException(e.getStatusCode());
448 }
449 }
450 }
451
452 @Override
453 protected void implPut(PutTask task) throws Exception {
454 PutTaskEntity entity = new PutTaskEntity(task);
455 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
456 if (sendRfc9457Accept) {
457 ApacheRFC9457Reporter.INSTANCE.prepareRequest(request);
458 }
459 try {
460 execute(request, null, task.getListener());
461 } catch (HttpResponseException e) {
462 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
463 state.setExpectContinue(false);
464 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
465 execute(request, null, task.getListener());
466 return;
467 }
468 throw new HttpTransporterException(e.getStatusCode());
469 }
470 }
471
472 private void execute(HttpUriRequest request, EntityGetter getter, TransportListener listener) throws Exception {
473 try {
474 SharingHttpContext context = new SharingHttpContext(state);
475 context.setAuthCache(authCache);
476 prepare(request, context);
477 try (CloseableHttpResponse response = client.execute(server, request, context)) {
478 try {
479 Map<TransferEvent.TransportPropertyKey, Object> transportProperties =
480 createTransportProperties(response, context);
481 listener.transportPropertiesAvailable(transportProperties);
482 handleStatus(response);
483 if (getter != null) {
484 getter.handle(response);
485 }
486 } finally {
487 EntityUtils.consumeQuietly(response.getEntity());
488 }
489 }
490 } catch (IOException e) {
491 if (e.getCause() instanceof TransferCancelledException) {
492 throw (Exception) e.getCause();
493 }
494 throw e;
495 }
496 }
497
498 private void prepare(HttpUriRequest request, SharingHttpContext context) throws Exception {
499 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
500 if (preemptiveAuth || (preemptivePutAuth && put)) {
501 context.getAuthCache().put(server, new BasicScheme());
502 }
503 if (supportWebDav) {
504 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
505 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
506 try (CloseableHttpResponse response = client.execute(server, req, context)) {
507 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
508 EntityUtils.consumeQuietly(response.getEntity());
509 } catch (IOException e) {
510 LOGGER.debug("Failed to prepare HTTP context", e);
511 }
512 }
513 if (put && Boolean.TRUE.equals(state.getWebDav())) {
514 mkdirs(request.getURI(), context);
515 }
516 }
517 }
518
519 private void mkdirs(URI uri, SharingHttpContext context) throws Exception {
520 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
521 int index = 0;
522 for (; index < dirs.size(); index++) {
523 try (CloseableHttpResponse response =
524 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
525 try {
526 int status = response.getStatusLine().getStatusCode();
527 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
528 break;
529 } else if (status == HttpStatus.SC_CONFLICT) {
530 continue;
531 }
532 handleStatus(response);
533 } finally {
534 EntityUtils.consumeQuietly(response.getEntity());
535 }
536 } catch (IOException e) {
537 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
538 return;
539 }
540 }
541 for (index--; index >= 0; index--) {
542 try (CloseableHttpResponse response =
543 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
544 try {
545 handleStatus(response);
546 } finally {
547 EntityUtils.consumeQuietly(response.getEntity());
548 }
549 } catch (IOException e) {
550 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
551 return;
552 }
553 }
554 }
555
556 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
557 request.setEntity(entity);
558 return request;
559 }
560
561 private boolean isPayloadPresent(HttpUriRequest request) {
562 if (request instanceof HttpEntityEnclosingRequest) {
563 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
564 return entity != null && entity.getContentLength() != 0;
565 }
566 return false;
567 }
568
569 private <T extends HttpUriRequest> T commonHeaders(T request) {
570 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
571 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
572
573 if (state.isExpectContinue() && isPayloadPresent(request)) {
574 request.setHeader(HttpHeaders.EXPECT, "100-continue");
575 }
576
577 for (Map.Entry<?, ?> entry : headers.entrySet()) {
578 if (!(entry.getKey() instanceof String)) {
579 continue;
580 }
581 if (entry.getValue() instanceof String) {
582 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
583 } else {
584 request.removeHeaders(entry.getKey().toString());
585 }
586 }
587
588 if (!state.isExpectContinue()) {
589 request.removeHeaders(HttpHeaders.EXPECT);
590 }
591 return request;
592 }
593
594 private <T extends HttpUriRequest> void resume(T request, GetTask task) throws IOException {
595 long resumeOffset = task.getResumeOffset();
596 if (resumeOffset > 0L && task.getDataPath() != null) {
597 long lastModified = Files.getLastModifiedTime(task.getDataPath()).toMillis();
598 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
599 request.setHeader(
600 HttpHeaders.IF_UNMODIFIED_SINCE, DateUtils.formatDate(new Date(lastModified - 60L * 1000L)));
601 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
602 }
603 }
604
605 private void handleStatus(CloseableHttpResponse response) throws Exception {
606 int status = response.getStatusLine().getStatusCode();
607 if (status >= 300) {
608 ApacheRFC9457Reporter.INSTANCE.generateException(response, (statusCode, reasonPhrase) -> {
609 throw new HttpResponseException(statusCode, reasonPhrase + " (" + statusCode + ")");
610 });
611 }
612 }
613
614 @Override
615 protected void implClose() {
616 try {
617 client.close();
618 } catch (IOException e) {
619 throw new UncheckedIOException(e);
620 }
621 AuthenticationContext.close(repoAuthContext);
622 AuthenticationContext.close(proxyAuthContext);
623 state.close();
624 }
625
626 private class EntityGetter {
627
628 private final GetTask task;
629
630 EntityGetter(GetTask task) {
631 this.task = task;
632 }
633
634 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
635 HttpEntity entity = response.getEntity();
636 if (entity == null) {
637 entity = new ByteArrayEntity(new byte[0]);
638 }
639
640 long offset = 0L, length = entity.getContentLength();
641 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
642 String range = rangeHeader != null ? rangeHeader.getValue() : null;
643 if (range != null) {
644 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
645 if (!m.matches()) {
646 throw new IOException("Invalid Content-Range header for partial download: " + range);
647 }
648 offset = Long.parseLong(m.group(1));
649 length = Long.parseLong(m.group(2)) + 1L;
650 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
651 throw new IOException("Invalid Content-Range header for partial download from offset "
652 + task.getResumeOffset() + ": " + range);
653 }
654 }
655
656 final boolean resume = offset > 0L;
657 final Path dataFile = task.getDataPath();
658 if (dataFile == null) {
659 try (InputStream is = entity.getContent()) {
660 utilGet(task, is, true, length, resume);
661 extractChecksums(response);
662 }
663 } else {
664 try (PathProcessor.CollocatedTempFile tempFile = pathProcessor.newTempFile(dataFile)) {
665 task.setDataPath(tempFile.getPath(), resume);
666 if (resume && Files.isRegularFile(dataFile)) {
667 try (InputStream inputStream = Files.newInputStream(dataFile)) {
668 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
669 }
670 }
671 try (InputStream is = entity.getContent()) {
672 utilGet(task, is, true, length, resume);
673 }
674 tempFile.move();
675 } finally {
676 task.setDataPath(dataFile);
677 }
678 }
679 if (task.getDataPath() != null) {
680 Header lastModifiedHeader =
681 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
682 if (lastModifiedHeader != null) {
683 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
684 if (lastModified != null) {
685 pathProcessor.setLastModified(
686 task.getDataPath(),
687 HttpTransporterUtils.clampRemoteLastModified(lastModified.getTime()));
688 }
689 }
690 }
691 extractChecksums(response);
692 }
693
694 private void extractChecksums(CloseableHttpResponse response) {
695 Map<String, String> checksums = checksumExtractor.extractChecksums(headerGetter(response));
696 if (checksums != null && !checksums.isEmpty()) {
697 checksums.forEach(task::setChecksum);
698 }
699 }
700 }
701
702 private static Map<TransferEvent.TransportPropertyKey, Object> createTransportProperties(
703 CloseableHttpResponse response, HttpCoreContext context) {
704 HttpTransportPropertiesBuilder builder =
705 new HttpTransportPropertiesBuilder(toHttpVersion(response.getProtocolVersion()));
706 SSLSession sslSession = context.getAttribute(CONTEXT_ATTRIBUTE_NAME_SSL_SESSION, SSLSession.class);
707 if (sslSession != null) {
708 builder.withSslProtocol(sslSession.getProtocol());
709 builder.withSslCipherSuite(sslSession.getCipherSuite());
710 }
711
712 return builder.build();
713 }
714
715 static HttpVersion toHttpVersion(ProtocolVersion version) {
716 switch (version.getMajor()) {
717 case 1:
718 if (version.getMinor() == 0) {
719 return HttpVersion.HTTP_1_0;
720 } else {
721 return HttpVersion.HTTP_1_1;
722 }
723 case 2:
724 return HttpVersion.HTTP_2;
725 case 3:
726 return HttpVersion.HTTP_3;
727 default:
728 throw new IllegalArgumentException("Unknown version " + version.toString());
729 }
730 }
731
732 private static Function<String, String> headerGetter(CloseableHttpResponse closeableHttpResponse) {
733 return s -> {
734 Header header = closeableHttpResponse.getFirstHeader(s);
735 return header != null ? header.getValue() : null;
736 };
737 }
738
739 private class PutTaskEntity extends AbstractHttpEntity {
740
741 private final PutTask task;
742
743 PutTaskEntity(PutTask task) {
744 this.task = task;
745 }
746
747 @Override
748 public boolean isRepeatable() {
749 return true;
750 }
751
752 @Override
753 public boolean isStreaming() {
754 return false;
755 }
756
757 @Override
758 public long getContentLength() {
759 return task.getDataLength();
760 }
761
762 @Override
763 public InputStream getContent() throws IOException {
764 return task.newInputStream();
765 }
766
767 @Override
768 public void writeTo(OutputStream os) throws IOException {
769 try {
770 utilPut(task, os, false);
771 } catch (TransferCancelledException e) {
772 throw (IOException) new InterruptedIOException().initCause(e);
773 }
774 }
775 }
776
777 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
778 private final int retryCount;
779
780 private final long retryInterval;
781
782 private final long retryIntervalMax;
783
784 private final Set<Integer> serviceUnavailableHttpCodes;
785
786
787
788
789
790
791 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
792
793 private ResolverServiceUnavailableRetryStrategy(
794 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
795 if (retryCount < 0) {
796 throw new IllegalArgumentException("retryCount must be >= 0");
797 }
798 if (retryInterval < 0L) {
799 throw new IllegalArgumentException("retryInterval must be >= 0");
800 }
801 if (retryIntervalMax < 0L) {
802 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
803 }
804 this.retryCount = retryCount;
805 this.retryInterval = retryInterval;
806 this.retryIntervalMax = retryIntervalMax;
807 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
808 }
809
810 @Override
811 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
812 final boolean retry = executionCount <= retryCount
813 && (serviceUnavailableHttpCodes.contains(
814 response.getStatusLine().getStatusCode()));
815 if (retry) {
816 Long retryInterval = retryInterval(response, executionCount, context);
817 if (retryInterval != null) {
818 RETRY_INTERVAL_HOLDER.set(retryInterval);
819 return true;
820 }
821 }
822 RETRY_INTERVAL_HOLDER.remove();
823 return false;
824 }
825
826
827
828
829
830
831
832
833 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
834 Long result = null;
835 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
836 if (header != null && header.getValue() != null) {
837 String headerValue = header.getValue();
838 if (headerValue.contains(":")) {
839 Date when = DateUtils.parseDate(headerValue);
840 if (when != null) {
841 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
842 }
843 } else {
844 try {
845 result = Long.parseLong(headerValue) * 1000L;
846 } catch (NumberFormatException e) {
847
848 }
849 }
850 }
851 if (result == null) {
852 result = executionCount * this.retryInterval;
853 }
854 if (result > retryIntervalMax) {
855 return null;
856 }
857 return result;
858 }
859
860 @Override
861 public long getRetryInterval() {
862 Long ri = RETRY_INTERVAL_HOLDER.get();
863 if (ri == null) {
864 return 0L;
865 }
866 RETRY_INTERVAL_HOLDER.remove();
867 return ri;
868 }
869 }
870
871 static class ConcurrentAuthCache implements AuthCache {
872 private final ConcurrentHashMap<HttpHost, AuthScheme> map = new ConcurrentHashMap<>();
873
874 @Override
875 public void put(HttpHost host, AuthScheme authScheme) {
876 if (host != null && authScheme != null) {
877 map.put(host, authScheme);
878 }
879 }
880
881 @Override
882 public AuthScheme get(HttpHost host) {
883 if (host == null) {
884 return null;
885 }
886 return map.get(host);
887 }
888
889 @Override
890 public void remove(HttpHost host) {
891 if (host != null) {
892 map.remove(host);
893 }
894 }
895
896 @Override
897 public void clear() {
898 map.clear();
899 }
900 }
901 }