1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.apache;
20
21 import javax.net.ssl.SSLSession;
22
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.InterruptedIOException;
26 import java.io.OutputStream;
27 import java.io.UncheckedIOException;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.nio.charset.Charset;
31 import java.nio.file.Files;
32 import java.nio.file.Path;
33 import java.nio.file.StandardCopyOption;
34 import java.util.Date;
35 import java.util.List;
36 import java.util.Map;
37 import java.util.Set;
38 import java.util.concurrent.ConcurrentHashMap;
39 import java.util.function.Function;
40 import java.util.regex.Matcher;
41
42 import org.apache.http.Header;
43 import org.apache.http.HttpClientConnection;
44 import org.apache.http.HttpEntity;
45 import org.apache.http.HttpEntityEnclosingRequest;
46 import org.apache.http.HttpException;
47 import org.apache.http.HttpHeaders;
48 import org.apache.http.HttpHost;
49 import org.apache.http.HttpRequest;
50 import org.apache.http.HttpResponse;
51 import org.apache.http.HttpStatus;
52 import org.apache.http.ProtocolVersion;
53 import org.apache.http.auth.AuthScheme;
54 import org.apache.http.auth.AuthSchemeProvider;
55 import org.apache.http.auth.AuthScope;
56 import org.apache.http.client.AuthCache;
57 import org.apache.http.client.CredentialsProvider;
58 import org.apache.http.client.HttpRequestRetryHandler;
59 import org.apache.http.client.HttpResponseException;
60 import org.apache.http.client.ServiceUnavailableRetryStrategy;
61 import org.apache.http.client.config.AuthSchemes;
62 import org.apache.http.client.config.CookieSpecs;
63 import org.apache.http.client.config.RequestConfig;
64 import org.apache.http.client.methods.CloseableHttpResponse;
65 import org.apache.http.client.methods.HttpGet;
66 import org.apache.http.client.methods.HttpHead;
67 import org.apache.http.client.methods.HttpOptions;
68 import org.apache.http.client.methods.HttpPut;
69 import org.apache.http.client.methods.HttpUriRequest;
70 import org.apache.http.client.utils.DateUtils;
71 import org.apache.http.client.utils.URIUtils;
72 import org.apache.http.config.Registry;
73 import org.apache.http.config.RegistryBuilder;
74 import org.apache.http.config.SocketConfig;
75 import org.apache.http.conn.ManagedHttpClientConnection;
76 import org.apache.http.entity.AbstractHttpEntity;
77 import org.apache.http.entity.ByteArrayEntity;
78 import org.apache.http.impl.NoConnectionReuseStrategy;
79 import org.apache.http.impl.auth.BasicScheme;
80 import org.apache.http.impl.auth.BasicSchemeFactory;
81 import org.apache.http.impl.auth.DigestSchemeFactory;
82 import org.apache.http.impl.auth.KerberosSchemeFactory;
83 import org.apache.http.impl.auth.NTLMSchemeFactory;
84 import org.apache.http.impl.auth.SPNegoSchemeFactory;
85 import org.apache.http.impl.client.CloseableHttpClient;
86 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
87 import org.apache.http.impl.client.HttpClientBuilder;
88 import org.apache.http.impl.client.LaxRedirectStrategy;
89 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
90 import org.apache.http.protocol.HttpContext;
91 import org.apache.http.protocol.HttpCoreContext;
92 import org.apache.http.protocol.HttpRequestExecutor;
93 import org.apache.http.util.EntityUtils;
94 import org.eclipse.aether.Keys;
95 import org.eclipse.aether.RepositorySystemSession;
96 import org.eclipse.aether.repository.AuthenticationContext;
97 import org.eclipse.aether.repository.Proxy;
98 import org.eclipse.aether.repository.RemoteRepository;
99 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
100 import org.eclipse.aether.spi.connector.transport.GetTask;
101 import org.eclipse.aether.spi.connector.transport.PeekTask;
102 import org.eclipse.aether.spi.connector.transport.PutTask;
103 import org.eclipse.aether.spi.connector.transport.TransportListener;
104 import org.eclipse.aether.spi.connector.transport.TransportTask;
105 import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
106 import org.eclipse.aether.spi.connector.transport.http.HttpTransportPropertiesBuilder;
107 import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
108 import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
109 import org.eclipse.aether.spi.io.PathProcessor;
110 import org.eclipse.aether.transfer.HttpTransportProperty.HttpVersion;
111 import org.eclipse.aether.transfer.NoTransporterException;
112 import org.eclipse.aether.transfer.TransferCancelledException;
113 import org.eclipse.aether.transfer.TransferEvent;
114 import org.eclipse.aether.util.ConfigUtils;
115 import org.eclipse.aether.util.StringDigestUtil;
116 import org.eclipse.aether.util.connector.transport.http.HttpTransporterUtils;
117 import org.slf4j.Logger;
118 import org.slf4j.LoggerFactory;
119
120 import static java.util.Objects.requireNonNull;
121 import static org.eclipse.aether.spi.connector.transport.http.HttpConstants.CONTENT_RANGE_PATTERN;
122 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_FOLLOW_REDIRECTS;
123 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
124 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
125 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_MAX_REDIRECTS;
126 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
127 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_FOLLOW_REDIRECTS;
128 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
129 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_MAX_REDIRECTS;
130 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
131 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
132 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
133
134
135
136
137 final class ApacheTransporter extends AbstractTransporter implements HttpTransporter {
138
139
140
141 private static final String CONTEXT_ATTRIBUTE_NAME_SSL_SESSION = "ssl.session";
142
143 private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
144
145 private final ChecksumExtractor checksumExtractor;
146
147 private final PathProcessor pathProcessor;
148
149 private final AuthenticationContext repoAuthContext;
150
151 private final AuthenticationContext proxyAuthContext;
152
153 private final URI baseUri;
154
155 private final HttpHost server;
156
157 private final HttpHost proxy;
158
159 private final CloseableHttpClient client;
160
161 private final Map<?, ?> headers;
162
163 private final LocalState state;
164
165 private final boolean preemptiveAuth;
166
167 private final boolean preemptivePutAuth;
168
169 private final boolean supportWebDav;
170
171 private final boolean sendRfc9457Accept;
172
173 private final AuthCache authCache;
174
175 @SuppressWarnings("checkstyle:methodlength")
176 ApacheTransporter(
177 RemoteRepository repository,
178 RepositorySystemSession session,
179 ChecksumExtractor checksumExtractor,
180 PathProcessor pathProcessor)
181 throws NoTransporterException {
182 this.checksumExtractor = checksumExtractor;
183 this.pathProcessor = pathProcessor;
184 try {
185 this.baseUri = HttpTransporterUtils.getBaseUri(repository);
186 this.server = URIUtils.extractHost(baseUri);
187 if (server == null) {
188 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
189 }
190 } catch (URISyntaxException e) {
191 throw new NoTransporterException(repository, e.getMessage(), e);
192 }
193 this.proxy = toHost(repository.getProxy());
194
195 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
196 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
197
198 String httpsSecurityMode = HttpTransporterUtils.getHttpsSecurityMode(session, repository);
199 final int connectionMaxTtlSeconds = HttpTransporterUtils.getHttpConnectionMaxTtlSeconds(session, repository);
200 final int maxConnectionsPerRoute = HttpTransporterUtils.getHttpMaxConnectionsPerRoute(session, repository);
201 this.state = new LocalState(
202 session,
203 repository,
204 new ConnMgrConfig(
205 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
206
207 this.headers = HttpTransporterUtils.getHttpHeaders(session, repository);
208 this.preemptiveAuth = HttpTransporterUtils.isHttpPreemptiveAuth(session, repository);
209 this.preemptivePutAuth = HttpTransporterUtils.isHttpPreemptivePutAuth(session, repository);
210 this.supportWebDav = HttpTransporterUtils.isHttpSupportWebDav(session, repository);
211 this.sendRfc9457Accept = HttpTransporterUtils.isHttpSendRfc9457Accept(session, repository);
212 int connectTimeout = HttpTransporterUtils.getHttpConnectTimeout(session, repository);
213 int requestTimeout = HttpTransporterUtils.getHttpRequestTimeout(session, repository);
214 int retryCount = HttpTransporterUtils.getHttpRetryHandlerCount(session, repository);
215 long retryInterval = HttpTransporterUtils.getHttpRetryHandlerInterval(session, repository);
216 long retryIntervalMax = HttpTransporterUtils.getHttpRetryHandlerIntervalMax(session, repository);
217 String retryHandlerName = ConfigUtils.getString(
218 session,
219 HTTP_RETRY_HANDLER_NAME_STANDARD,
220 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
221 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
222 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
223 session,
224 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
225 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
226 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
227 int maxRedirects = ConfigUtils.getInteger(
228 session,
229 DEFAULT_MAX_REDIRECTS,
230 CONFIG_PROP_MAX_REDIRECTS + "." + repository.getId(),
231 CONFIG_PROP_MAX_REDIRECTS);
232 boolean followRedirects = ConfigUtils.getBoolean(
233 session,
234 DEFAULT_FOLLOW_REDIRECTS,
235 CONFIG_PROP_FOLLOW_REDIRECTS + "." + repository.getId(),
236 CONFIG_PROP_FOLLOW_REDIRECTS);
237 String userAgent = HttpTransporterUtils.getUserAgent(session, repository);
238
239 Charset credentialsCharset = HttpTransporterUtils.getHttpCredentialsEncoding(session, repository);
240 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
241 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
242 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
243 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
244 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
245 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
246 .build();
247 SocketConfig socketConfig =
248
249 SocketConfig.custom().setSoTimeout(requestTimeout).build();
250 RequestConfig requestConfig = RequestConfig.custom()
251 .setMaxRedirects(maxRedirects)
252 .setRedirectsEnabled(followRedirects)
253 .setRelativeRedirectsAllowed(followRedirects)
254
255 .setSocketTimeout(requestTimeout)
256
257 .setConnectTimeout(connectTimeout)
258
259 .setConnectionRequestTimeout(connectTimeout)
260 .setLocalAddress(HttpTransporterUtils.getHttpLocalAddress(session, repository)
261 .orElse(null))
262 .setCookieSpec(CookieSpecs.STANDARD)
263 .build();
264
265 HttpRequestRetryHandler retryHandler;
266 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
267 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
268 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
269 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
270 } else {
271 throw new IllegalArgumentException(
272 "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
273 }
274 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
275 retryCount,
276 retryInterval,
277 retryIntervalMax,
278 HttpTransporterUtils.getHttpServiceUnavailableCodes(session, repository));
279
280 HttpClientBuilder builder = HttpClientBuilder.create()
281 .setUserAgent(userAgent)
282 .setRedirectStrategy(LaxRedirectStrategy.INSTANCE)
283 .setDefaultSocketConfig(socketConfig)
284 .setDefaultRequestConfig(requestConfig)
285 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
286 .setRetryHandler(retryHandler)
287 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
288 .setConnectionManager(state.getConnectionManager())
289 .setConnectionManagerShared(true)
290 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
291 .setProxy(proxy);
292 final boolean useSystemProperties = ConfigUtils.getBoolean(
293 session,
294 DEFAULT_USE_SYSTEM_PROPERTIES,
295 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
296 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
297 if (useSystemProperties) {
298 LOGGER.warn(
299 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
300 LOGGER.warn("Please use documented means to configure resolver transport.");
301 builder.useSystemProperties();
302 }
303
304
305 builder.setRequestExecutor(new HttpRequestExecutor() {
306
307 @Override
308 public HttpResponse execute(HttpRequest request, HttpClientConnection conn, HttpContext context)
309 throws IOException, HttpException {
310 if (conn instanceof ManagedHttpClientConnection) {
311 context.setAttribute(
312 CONTEXT_ATTRIBUTE_NAME_SSL_SESSION, ((ManagedHttpClientConnection) conn).getSSLSession());
313 }
314 return super.execute(request, conn, context);
315 }
316 });
317
318 HttpTransporterUtils.getHttpExpectContinue(session, repository).ifPresent(state::setExpectContinue);
319 if (!HttpTransporterUtils.isHttpReuseConnections(session, repository)) {
320 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
321 }
322
323 if (session.getCache() != null) {
324 this.authCache = (AuthCache) session.getCache()
325 .computeIfAbsent(
326 session,
327 Keys.of(
328 getClass(),
329 repository.getId() + "-" + StringDigestUtil.sha1(repository.toString())),
330 ConcurrentAuthCache::new);
331 } else {
332 this.authCache = new ConcurrentAuthCache();
333 }
334 this.client = builder.build();
335 }
336
337 private static HttpHost toHost(Proxy proxy) {
338 HttpHost host = null;
339 if (proxy != null) {
340
341
342
343
344 host = new HttpHost(proxy.getHost(), proxy.getPort());
345 }
346 return host;
347 }
348
349 private static CredentialsProvider toCredentialsProvider(
350 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
351 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
352 if (proxy != null) {
353 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
354 provider = new DemuxCredentialsProvider(provider, p, proxy);
355 }
356 return provider;
357 }
358
359 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
360 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
361 if (ctx != null) {
362 AuthScope basicScope = new AuthScope(host, port);
363 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
364
365 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
366 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
367 }
368 return provider;
369 }
370
371 LocalState getState() {
372 return state;
373 }
374
375 private URI resolve(TransportTask task) {
376 return UriUtils.resolve(baseUri, task.getLocation());
377 }
378
379 @Override
380 protected void implPeek(PeekTask task) throws Exception {
381 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
382 try {
383 execute(request, null, task.getListener());
384 } catch (HttpResponseException e) {
385 throw new HttpTransporterException(e.getStatusCode());
386 }
387 }
388
389 @Override
390 protected void implGet(GetTask task) throws Exception {
391 boolean resume = true;
392
393 EntityGetter getter = new EntityGetter(task);
394 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
395 if (sendRfc9457Accept) {
396 ApacheRFC9457Reporter.INSTANCE.prepareRequest(request);
397 }
398 while (true) {
399 try {
400 if (resume) {
401 resume(request, task);
402 }
403 execute(request, getter, task.getListener());
404 break;
405 } catch (HttpResponseException e) {
406 if (resume
407 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
408 && request.containsHeader(HttpHeaders.RANGE)) {
409 request = commonHeaders(new HttpGet(resolve(task)));
410 resume = false;
411 continue;
412 }
413 throw new HttpTransporterException(e.getStatusCode());
414 }
415 }
416 }
417
418 @Override
419 protected void implPut(PutTask task) throws Exception {
420 PutTaskEntity entity = new PutTaskEntity(task);
421 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
422 if (sendRfc9457Accept) {
423 ApacheRFC9457Reporter.INSTANCE.prepareRequest(request);
424 }
425 try {
426 execute(request, null, task.getListener());
427 } catch (HttpResponseException e) {
428 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
429 state.setExpectContinue(false);
430 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
431 execute(request, null, task.getListener());
432 return;
433 }
434 throw new HttpTransporterException(e.getStatusCode());
435 }
436 }
437
438 private void execute(HttpUriRequest request, EntityGetter getter, TransportListener listener) throws Exception {
439 try {
440 SharingHttpContext context = new SharingHttpContext(state);
441 context.setAuthCache(authCache);
442 prepare(request, context);
443 try (CloseableHttpResponse response = client.execute(server, request, context)) {
444 try {
445 Map<TransferEvent.TransportPropertyKey, Object> transportProperties =
446 createTransportProperties(response, context);
447 listener.transportPropertiesAvailable(transportProperties);
448 handleStatus(response);
449 if (getter != null) {
450 getter.handle(response);
451 }
452 } finally {
453 EntityUtils.consumeQuietly(response.getEntity());
454 }
455 }
456 } catch (IOException e) {
457 if (e.getCause() instanceof TransferCancelledException) {
458 throw (Exception) e.getCause();
459 }
460 throw e;
461 }
462 }
463
464 private void prepare(HttpUriRequest request, SharingHttpContext context) throws Exception {
465 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
466 if (preemptiveAuth || (preemptivePutAuth && put)) {
467 context.getAuthCache().put(server, new BasicScheme());
468 }
469 if (supportWebDav) {
470 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
471 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
472 try (CloseableHttpResponse response = client.execute(server, req, context)) {
473 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
474 EntityUtils.consumeQuietly(response.getEntity());
475 } catch (IOException e) {
476 LOGGER.debug("Failed to prepare HTTP context", e);
477 }
478 }
479 if (put && Boolean.TRUE.equals(state.getWebDav())) {
480 mkdirs(request.getURI(), context);
481 }
482 }
483 }
484
485 private void mkdirs(URI uri, SharingHttpContext context) throws Exception {
486 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
487 int index = 0;
488 for (; index < dirs.size(); index++) {
489 try (CloseableHttpResponse response =
490 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
491 try {
492 int status = response.getStatusLine().getStatusCode();
493 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
494 break;
495 } else if (status == HttpStatus.SC_CONFLICT) {
496 continue;
497 }
498 handleStatus(response);
499 } finally {
500 EntityUtils.consumeQuietly(response.getEntity());
501 }
502 } catch (IOException e) {
503 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
504 return;
505 }
506 }
507 for (index--; index >= 0; index--) {
508 try (CloseableHttpResponse response =
509 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
510 try {
511 handleStatus(response);
512 } finally {
513 EntityUtils.consumeQuietly(response.getEntity());
514 }
515 } catch (IOException e) {
516 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
517 return;
518 }
519 }
520 }
521
522 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
523 request.setEntity(entity);
524 return request;
525 }
526
527 private boolean isPayloadPresent(HttpUriRequest request) {
528 if (request instanceof HttpEntityEnclosingRequest) {
529 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
530 return entity != null && entity.getContentLength() != 0;
531 }
532 return false;
533 }
534
535 private <T extends HttpUriRequest> T commonHeaders(T request) {
536 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
537 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
538
539 if (state.isExpectContinue() && isPayloadPresent(request)) {
540 request.setHeader(HttpHeaders.EXPECT, "100-continue");
541 }
542
543 for (Map.Entry<?, ?> entry : headers.entrySet()) {
544 if (!(entry.getKey() instanceof String)) {
545 continue;
546 }
547 if (entry.getValue() instanceof String) {
548 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
549 } else {
550 request.removeHeaders(entry.getKey().toString());
551 }
552 }
553
554 if (!state.isExpectContinue()) {
555 request.removeHeaders(HttpHeaders.EXPECT);
556 }
557 return request;
558 }
559
560 private <T extends HttpUriRequest> void resume(T request, GetTask task) throws IOException {
561 long resumeOffset = task.getResumeOffset();
562 if (resumeOffset > 0L && task.getDataPath() != null) {
563 long lastModified = Files.getLastModifiedTime(task.getDataPath()).toMillis();
564 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
565 request.setHeader(
566 HttpHeaders.IF_UNMODIFIED_SINCE, DateUtils.formatDate(new Date(lastModified - 60L * 1000L)));
567 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
568 }
569 }
570
571 private void handleStatus(CloseableHttpResponse response) throws Exception {
572 int status = response.getStatusLine().getStatusCode();
573 if (status >= 300) {
574 ApacheRFC9457Reporter.INSTANCE.generateException(response, (statusCode, reasonPhrase) -> {
575 throw new HttpResponseException(statusCode, reasonPhrase + " (" + statusCode + ")");
576 });
577 }
578 }
579
580 @Override
581 protected void implClose() {
582 try {
583 client.close();
584 } catch (IOException e) {
585 throw new UncheckedIOException(e);
586 }
587 AuthenticationContext.close(repoAuthContext);
588 AuthenticationContext.close(proxyAuthContext);
589 state.close();
590 }
591
592 private class EntityGetter {
593
594 private final GetTask task;
595
596 EntityGetter(GetTask task) {
597 this.task = task;
598 }
599
600 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
601 HttpEntity entity = response.getEntity();
602 if (entity == null) {
603 entity = new ByteArrayEntity(new byte[0]);
604 }
605
606 long offset = 0L, length = entity.getContentLength();
607 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
608 String range = rangeHeader != null ? rangeHeader.getValue() : null;
609 if (range != null) {
610 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
611 if (!m.matches()) {
612 throw new IOException("Invalid Content-Range header for partial download: " + range);
613 }
614 offset = Long.parseLong(m.group(1));
615 length = Long.parseLong(m.group(2)) + 1L;
616 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
617 throw new IOException("Invalid Content-Range header for partial download from offset "
618 + task.getResumeOffset() + ": " + range);
619 }
620 }
621
622 final boolean resume = offset > 0L;
623 final Path dataFile = task.getDataPath();
624 if (dataFile == null) {
625 try (InputStream is = entity.getContent()) {
626 utilGet(task, is, true, length, resume);
627 extractChecksums(response);
628 }
629 } else {
630 try (PathProcessor.CollocatedTempFile tempFile = pathProcessor.newTempFile(dataFile)) {
631 task.setDataPath(tempFile.getPath(), resume);
632 if (resume && Files.isRegularFile(dataFile)) {
633 try (InputStream inputStream = Files.newInputStream(dataFile)) {
634 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
635 }
636 }
637 try (InputStream is = entity.getContent()) {
638 utilGet(task, is, true, length, resume);
639 }
640 tempFile.move();
641 } finally {
642 task.setDataPath(dataFile);
643 }
644 }
645 if (task.getDataPath() != null) {
646 Header lastModifiedHeader =
647 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
648 if (lastModifiedHeader != null) {
649 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
650 if (lastModified != null) {
651 pathProcessor.setLastModified(task.getDataPath(), lastModified.getTime());
652 }
653 }
654 }
655 extractChecksums(response);
656 }
657
658 private void extractChecksums(CloseableHttpResponse response) {
659 Map<String, String> checksums = checksumExtractor.extractChecksums(headerGetter(response));
660 if (checksums != null && !checksums.isEmpty()) {
661 checksums.forEach(task::setChecksum);
662 }
663 }
664 }
665
666 private static Map<TransferEvent.TransportPropertyKey, Object> createTransportProperties(
667 CloseableHttpResponse response, HttpCoreContext context) {
668 HttpTransportPropertiesBuilder builder =
669 new HttpTransportPropertiesBuilder(toHttpVersion(response.getProtocolVersion()));
670 SSLSession sslSession = context.getAttribute(CONTEXT_ATTRIBUTE_NAME_SSL_SESSION, SSLSession.class);
671 if (sslSession != null) {
672 builder.withSslProtocol(sslSession.getProtocol());
673 builder.withSslCipherSuite(sslSession.getCipherSuite());
674 }
675
676 return builder.build();
677 }
678
679 static HttpVersion toHttpVersion(ProtocolVersion version) {
680 switch (version.getMajor()) {
681 case 1:
682 if (version.getMinor() == 0) {
683 return HttpVersion.HTTP_1_0;
684 } else {
685 return HttpVersion.HTTP_1_1;
686 }
687 case 2:
688 return HttpVersion.HTTP_2;
689 case 3:
690 return HttpVersion.HTTP_3;
691 default:
692 throw new IllegalArgumentException("Unknown version " + version.toString());
693 }
694 }
695
696 private static Function<String, String> headerGetter(CloseableHttpResponse closeableHttpResponse) {
697 return s -> {
698 Header header = closeableHttpResponse.getFirstHeader(s);
699 return header != null ? header.getValue() : null;
700 };
701 }
702
703 private class PutTaskEntity extends AbstractHttpEntity {
704
705 private final PutTask task;
706
707 PutTaskEntity(PutTask task) {
708 this.task = task;
709 }
710
711 @Override
712 public boolean isRepeatable() {
713 return true;
714 }
715
716 @Override
717 public boolean isStreaming() {
718 return false;
719 }
720
721 @Override
722 public long getContentLength() {
723 return task.getDataLength();
724 }
725
726 @Override
727 public InputStream getContent() throws IOException {
728 return task.newInputStream();
729 }
730
731 @Override
732 public void writeTo(OutputStream os) throws IOException {
733 try {
734 utilPut(task, os, false);
735 } catch (TransferCancelledException e) {
736 throw (IOException) new InterruptedIOException().initCause(e);
737 }
738 }
739 }
740
741 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
742 private final int retryCount;
743
744 private final long retryInterval;
745
746 private final long retryIntervalMax;
747
748 private final Set<Integer> serviceUnavailableHttpCodes;
749
750
751
752
753
754
755 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
756
757 private ResolverServiceUnavailableRetryStrategy(
758 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
759 if (retryCount < 0) {
760 throw new IllegalArgumentException("retryCount must be >= 0");
761 }
762 if (retryInterval < 0L) {
763 throw new IllegalArgumentException("retryInterval must be >= 0");
764 }
765 if (retryIntervalMax < 0L) {
766 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
767 }
768 this.retryCount = retryCount;
769 this.retryInterval = retryInterval;
770 this.retryIntervalMax = retryIntervalMax;
771 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
772 }
773
774 @Override
775 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
776 final boolean retry = executionCount <= retryCount
777 && (serviceUnavailableHttpCodes.contains(
778 response.getStatusLine().getStatusCode()));
779 if (retry) {
780 Long retryInterval = retryInterval(response, executionCount, context);
781 if (retryInterval != null) {
782 RETRY_INTERVAL_HOLDER.set(retryInterval);
783 return true;
784 }
785 }
786 RETRY_INTERVAL_HOLDER.remove();
787 return false;
788 }
789
790
791
792
793
794
795
796
797 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
798 Long result = null;
799 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
800 if (header != null && header.getValue() != null) {
801 String headerValue = header.getValue();
802 if (headerValue.contains(":")) {
803 Date when = DateUtils.parseDate(headerValue);
804 if (when != null) {
805 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
806 }
807 } else {
808 try {
809 result = Long.parseLong(headerValue) * 1000L;
810 } catch (NumberFormatException e) {
811
812 }
813 }
814 }
815 if (result == null) {
816 result = executionCount * this.retryInterval;
817 }
818 if (result > retryIntervalMax) {
819 return null;
820 }
821 return result;
822 }
823
824 @Override
825 public long getRetryInterval() {
826 Long ri = RETRY_INTERVAL_HOLDER.get();
827 if (ri == null) {
828 return 0L;
829 }
830 RETRY_INTERVAL_HOLDER.remove();
831 return ri;
832 }
833 }
834
835 static class ConcurrentAuthCache implements AuthCache {
836 private final ConcurrentHashMap<HttpHost, AuthScheme> map = new ConcurrentHashMap<>();
837
838 @Override
839 public void put(HttpHost host, AuthScheme authScheme) {
840 if (host != null && authScheme != null) {
841 map.put(host, authScheme);
842 }
843 }
844
845 @Override
846 public AuthScheme get(HttpHost host) {
847 if (host == null) {
848 return null;
849 }
850 return map.get(host);
851 }
852
853 @Override
854 public void remove(HttpHost host) {
855 if (host != null) {
856 map.remove(host);
857 }
858 }
859
860 @Override
861 public void clear() {
862 map.clear();
863 }
864 }
865 }