View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.eclipse.aether.transport.apache;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.InterruptedIOException;
24  import java.io.OutputStream;
25  import java.io.UncheckedIOException;
26  import java.net.InetAddress;
27  import java.net.URI;
28  import java.net.URISyntaxException;
29  import java.net.UnknownHostException;
30  import java.nio.charset.Charset;
31  import java.nio.file.Files;
32  import java.nio.file.Path;
33  import java.nio.file.StandardCopyOption;
34  import java.util.Collections;
35  import java.util.Date;
36  import java.util.HashSet;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.Set;
40  import java.util.function.Function;
41  import java.util.regex.Matcher;
42  
43  import org.apache.http.Header;
44  import org.apache.http.HttpEntity;
45  import org.apache.http.HttpEntityEnclosingRequest;
46  import org.apache.http.HttpHeaders;
47  import org.apache.http.HttpHost;
48  import org.apache.http.HttpResponse;
49  import org.apache.http.HttpStatus;
50  import org.apache.http.auth.AuthSchemeProvider;
51  import org.apache.http.auth.AuthScope;
52  import org.apache.http.client.CredentialsProvider;
53  import org.apache.http.client.HttpRequestRetryHandler;
54  import org.apache.http.client.HttpResponseException;
55  import org.apache.http.client.ServiceUnavailableRetryStrategy;
56  import org.apache.http.client.config.AuthSchemes;
57  import org.apache.http.client.config.CookieSpecs;
58  import org.apache.http.client.config.RequestConfig;
59  import org.apache.http.client.methods.CloseableHttpResponse;
60  import org.apache.http.client.methods.HttpGet;
61  import org.apache.http.client.methods.HttpHead;
62  import org.apache.http.client.methods.HttpOptions;
63  import org.apache.http.client.methods.HttpPut;
64  import org.apache.http.client.methods.HttpUriRequest;
65  import org.apache.http.client.utils.DateUtils;
66  import org.apache.http.client.utils.URIUtils;
67  import org.apache.http.config.Registry;
68  import org.apache.http.config.RegistryBuilder;
69  import org.apache.http.config.SocketConfig;
70  import org.apache.http.entity.AbstractHttpEntity;
71  import org.apache.http.entity.ByteArrayEntity;
72  import org.apache.http.impl.NoConnectionReuseStrategy;
73  import org.apache.http.impl.auth.BasicScheme;
74  import org.apache.http.impl.auth.BasicSchemeFactory;
75  import org.apache.http.impl.auth.DigestSchemeFactory;
76  import org.apache.http.impl.auth.KerberosSchemeFactory;
77  import org.apache.http.impl.auth.NTLMSchemeFactory;
78  import org.apache.http.impl.auth.SPNegoSchemeFactory;
79  import org.apache.http.impl.client.CloseableHttpClient;
80  import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
81  import org.apache.http.impl.client.HttpClientBuilder;
82  import org.apache.http.impl.client.LaxRedirectStrategy;
83  import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
84  import org.apache.http.protocol.HttpContext;
85  import org.apache.http.util.EntityUtils;
86  import org.eclipse.aether.ConfigurationProperties;
87  import org.eclipse.aether.RepositorySystemSession;
88  import org.eclipse.aether.repository.AuthenticationContext;
89  import org.eclipse.aether.repository.Proxy;
90  import org.eclipse.aether.repository.RemoteRepository;
91  import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
92  import org.eclipse.aether.spi.connector.transport.GetTask;
93  import org.eclipse.aether.spi.connector.transport.PeekTask;
94  import org.eclipse.aether.spi.connector.transport.PutTask;
95  import org.eclipse.aether.spi.connector.transport.TransportTask;
96  import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
97  import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
98  import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
99  import org.eclipse.aether.spi.io.PathProcessor;
100 import org.eclipse.aether.transfer.NoTransporterException;
101 import org.eclipse.aether.transfer.TransferCancelledException;
102 import org.eclipse.aether.util.ConfigUtils;
103 import org.eclipse.aether.util.FileUtils;
104 import org.slf4j.Logger;
105 import org.slf4j.LoggerFactory;
106 
107 import static java.util.Objects.requireNonNull;
108 import static org.eclipse.aether.spi.connector.transport.http.HttpConstants.CONTENT_RANGE_PATTERN;
109 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_FOLLOW_REDIRECTS;
110 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
111 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
112 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_MAX_REDIRECTS;
113 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
114 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_FOLLOW_REDIRECTS;
115 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
116 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_MAX_REDIRECTS;
117 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
118 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
119 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
120 
121 /**
122  * A transporter for HTTP/HTTPS.
123  */
124 final class ApacheTransporter extends AbstractTransporter implements HttpTransporter {
125     private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
126 
127     private final ChecksumExtractor checksumExtractor;
128 
129     private final PathProcessor pathProcessor;
130 
131     private final AuthenticationContext repoAuthContext;
132 
133     private final AuthenticationContext proxyAuthContext;
134 
135     private final URI baseUri;
136 
137     private final HttpHost server;
138 
139     private final HttpHost proxy;
140 
141     private final CloseableHttpClient client;
142 
143     private final Map<?, ?> headers;
144 
145     private final LocalState state;
146 
147     private final boolean preemptiveAuth;
148 
149     private final boolean preemptivePutAuth;
150 
151     private final boolean supportWebDav;
152 
153     @SuppressWarnings("checkstyle:methodlength")
154     ApacheTransporter(
155             RemoteRepository repository,
156             RepositorySystemSession session,
157             ChecksumExtractor checksumExtractor,
158             PathProcessor pathProcessor)
159             throws NoTransporterException {
160         this.checksumExtractor = checksumExtractor;
161         this.pathProcessor = pathProcessor;
162         try {
163             this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
164             if (baseUri.isOpaque()) {
165                 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
166             }
167             this.server = URIUtils.extractHost(baseUri);
168             if (server == null) {
169                 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
170             }
171         } catch (URISyntaxException e) {
172             throw new NoTransporterException(repository, e.getMessage(), e);
173         }
174         this.proxy = toHost(repository.getProxy());
175 
176         this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
177         this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
178 
179         String httpsSecurityMode = ConfigUtils.getString(
180                 session,
181                 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
182                 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
183                 ConfigurationProperties.HTTPS_SECURITY_MODE);
184         final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
185                 session,
186                 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
187                 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
188                 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
189         final int maxConnectionsPerRoute = ConfigUtils.getInteger(
190                 session,
191                 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
192                 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
193                 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
194         this.state = new LocalState(
195                 session,
196                 repository,
197                 new ConnMgrConfig(
198                         session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
199 
200         this.headers = ConfigUtils.getMap(
201                 session,
202                 Collections.emptyMap(),
203                 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
204                 ConfigurationProperties.HTTP_HEADERS);
205 
206         this.preemptiveAuth = ConfigUtils.getBoolean(
207                 session,
208                 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
209                 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
210                 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
211         this.preemptivePutAuth = ConfigUtils.getBoolean(
212                 session,
213                 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_PUT_AUTH,
214                 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH + "." + repository.getId(),
215                 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH);
216         this.supportWebDav = ConfigUtils.getBoolean(
217                 session,
218                 ConfigurationProperties.DEFAULT_HTTP_SUPPORT_WEBDAV,
219                 ConfigurationProperties.HTTP_SUPPORT_WEBDAV + "." + repository.getId(),
220                 ConfigurationProperties.HTTP_SUPPORT_WEBDAV);
221         String credentialEncoding = ConfigUtils.getString(
222                 session,
223                 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
224                 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
225                 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
226         int connectTimeout = ConfigUtils.getInteger(
227                 session,
228                 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
229                 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
230                 ConfigurationProperties.CONNECT_TIMEOUT);
231         int requestTimeout = ConfigUtils.getInteger(
232                 session,
233                 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
234                 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
235                 ConfigurationProperties.REQUEST_TIMEOUT);
236         int retryCount = ConfigUtils.getInteger(
237                 session,
238                 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
239                 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
240                 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
241         long retryInterval = ConfigUtils.getLong(
242                 session,
243                 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
244                 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
245                 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
246         long retryIntervalMax = ConfigUtils.getLong(
247                 session,
248                 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
249                 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
250                 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
251         String serviceUnavailableCodesString = ConfigUtils.getString(
252                 session,
253                 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
254                 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
255                 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
256         String retryHandlerName = ConfigUtils.getString(
257                 session,
258                 HTTP_RETRY_HANDLER_NAME_STANDARD,
259                 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
260                 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
261         boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
262                 session,
263                 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
264                 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
265                 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
266         int maxRedirects = ConfigUtils.getInteger(
267                 session,
268                 DEFAULT_MAX_REDIRECTS,
269                 CONFIG_PROP_MAX_REDIRECTS + "." + repository.getId(),
270                 CONFIG_PROP_MAX_REDIRECTS);
271         boolean followRedirects = ConfigUtils.getBoolean(
272                 session,
273                 DEFAULT_FOLLOW_REDIRECTS,
274                 CONFIG_PROP_FOLLOW_REDIRECTS + "." + repository.getId(),
275                 CONFIG_PROP_FOLLOW_REDIRECTS);
276         String userAgent = ConfigUtils.getString(
277                 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
278 
279         Charset credentialsCharset = Charset.forName(credentialEncoding);
280         Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
281                 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
282                 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
283                 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
284                 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
285                 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
286                 .build();
287         SocketConfig socketConfig =
288                 // the time to establish connection (low level)
289                 SocketConfig.custom().setSoTimeout(requestTimeout).build();
290         RequestConfig requestConfig = RequestConfig.custom()
291                 .setMaxRedirects(maxRedirects)
292                 .setRedirectsEnabled(followRedirects)
293                 .setRelativeRedirectsAllowed(followRedirects)
294                 // the time waiting for data; max time between two data packets
295                 .setSocketTimeout(requestTimeout)
296                 // the time to establish the connection (high level)
297                 .setConnectTimeout(connectTimeout)
298                 // the time to wait for a connection from the connection manager/pool
299                 .setConnectionRequestTimeout(connectTimeout)
300                 .setLocalAddress(getHttpLocalAddress(session, repository))
301                 .setCookieSpec(CookieSpecs.STANDARD)
302                 .build();
303 
304         HttpRequestRetryHandler retryHandler;
305         if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
306             retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
307         } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
308             retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
309         } else {
310             throw new IllegalArgumentException(
311                     "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
312         }
313         Set<Integer> serviceUnavailableCodes = new HashSet<>();
314         try {
315             for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
316                 serviceUnavailableCodes.add(Integer.parseInt(code));
317             }
318         } catch (NumberFormatException e) {
319             throw new IllegalArgumentException(
320                     "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
321                             + " (list of integers): " + serviceUnavailableCodesString);
322         }
323         ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
324                 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
325 
326         HttpClientBuilder builder = HttpClientBuilder.create()
327                 .setUserAgent(userAgent)
328                 .setRedirectStrategy(LaxRedirectStrategy.INSTANCE)
329                 .setDefaultSocketConfig(socketConfig)
330                 .setDefaultRequestConfig(requestConfig)
331                 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
332                 .setRetryHandler(retryHandler)
333                 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
334                 .setConnectionManager(state.getConnectionManager())
335                 .setConnectionManagerShared(true)
336                 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
337                 .setProxy(proxy);
338         final boolean useSystemProperties = ConfigUtils.getBoolean(
339                 session,
340                 DEFAULT_USE_SYSTEM_PROPERTIES,
341                 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
342                 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
343         if (useSystemProperties) {
344             LOGGER.warn(
345                     "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
346             LOGGER.warn("Please use documented means to configure resolver transport.");
347             builder.useSystemProperties();
348         }
349 
350         final String expectContinue = ConfigUtils.getString(
351                 session,
352                 null,
353                 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
354                 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
355         if (expectContinue != null) {
356             state.setExpectContinue(Boolean.parseBoolean(expectContinue));
357         }
358 
359         final boolean reuseConnections = ConfigUtils.getBoolean(
360                 session,
361                 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
362                 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
363                 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
364         if (!reuseConnections) {
365             builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
366         }
367 
368         this.client = builder.build();
369     }
370 
371     /**
372      * Returns non-null {@link InetAddress} if set in configuration, {@code null} otherwise.
373      */
374     private InetAddress getHttpLocalAddress(RepositorySystemSession session, RemoteRepository repository) {
375         String bindAddress = ConfigUtils.getString(
376                 session,
377                 null,
378                 ConfigurationProperties.HTTP_LOCAL_ADDRESS + "." + repository.getId(),
379                 ConfigurationProperties.HTTP_LOCAL_ADDRESS);
380         if (bindAddress == null) {
381             return null;
382         }
383         try {
384             return InetAddress.getByName(bindAddress);
385         } catch (UnknownHostException uhe) {
386             throw new IllegalArgumentException(
387                     "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
388                     uhe);
389         }
390     }
391 
392     private static HttpHost toHost(Proxy proxy) {
393         HttpHost host = null;
394         if (proxy != null) {
395             // in Maven, the proxy.protocol is used for proxy matching against remote repository protocol; no TLS proxy
396             // support
397             // https://github.com/apache/maven/issues/2519
398             // https://github.com/apache/maven-resolver/issues/745
399             host = new HttpHost(proxy.getHost(), proxy.getPort());
400         }
401         return host;
402     }
403 
404     private static CredentialsProvider toCredentialsProvider(
405             HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
406         CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
407         if (proxy != null) {
408             CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
409             provider = new DemuxCredentialsProvider(provider, p, proxy);
410         }
411         return provider;
412     }
413 
414     private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
415         DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
416         if (ctx != null) {
417             AuthScope basicScope = new AuthScope(host, port);
418             provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
419 
420             AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
421             provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
422         }
423         return provider;
424     }
425 
426     LocalState getState() {
427         return state;
428     }
429 
430     private URI resolve(TransportTask task) {
431         return UriUtils.resolve(baseUri, task.getLocation());
432     }
433 
434     @Override
435     public int classify(Throwable error) {
436         if (error instanceof HttpTransporterException
437                 && ((HttpTransporterException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
438             return ERROR_NOT_FOUND;
439         }
440         return ERROR_OTHER;
441     }
442 
443     @Override
444     protected void implPeek(PeekTask task) throws Exception {
445         HttpHead request = commonHeaders(new HttpHead(resolve(task)));
446         try {
447             execute(request, null);
448         } catch (HttpResponseException e) {
449             throw new HttpTransporterException(e.getStatusCode());
450         }
451     }
452 
453     @Override
454     protected void implGet(GetTask task) throws Exception {
455         boolean resume = true;
456 
457         EntityGetter getter = new EntityGetter(task);
458         HttpGet request = commonHeaders(new HttpGet(resolve(task)));
459         while (true) {
460             try {
461                 if (resume) {
462                     resume(request, task);
463                 }
464                 execute(request, getter);
465                 break;
466             } catch (HttpResponseException e) {
467                 if (resume
468                         && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
469                         && request.containsHeader(HttpHeaders.RANGE)) {
470                     request = commonHeaders(new HttpGet(resolve(task)));
471                     resume = false;
472                     continue;
473                 }
474                 throw new HttpTransporterException(e.getStatusCode());
475             }
476         }
477     }
478 
479     @Override
480     protected void implPut(PutTask task) throws Exception {
481         PutTaskEntity entity = new PutTaskEntity(task);
482         HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
483         try {
484             execute(request, null);
485         } catch (HttpResponseException e) {
486             if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
487                 state.setExpectContinue(false);
488                 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
489                 execute(request, null);
490                 return;
491             }
492             throw new HttpTransporterException(e.getStatusCode());
493         }
494     }
495 
496     private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
497         try {
498             SharingHttpContext context = new SharingHttpContext(state);
499             prepare(request, context);
500             try (CloseableHttpResponse response = client.execute(server, request, context)) {
501                 try {
502                     context.close();
503                     handleStatus(response);
504                     if (getter != null) {
505                         getter.handle(response);
506                     }
507                 } finally {
508                     EntityUtils.consumeQuietly(response.getEntity());
509                 }
510             }
511         } catch (IOException e) {
512             if (e.getCause() instanceof TransferCancelledException) {
513                 throw (Exception) e.getCause();
514             }
515             throw e;
516         }
517     }
518 
519     private void prepare(HttpUriRequest request, SharingHttpContext context) throws Exception {
520         final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
521         if (preemptiveAuth || (preemptivePutAuth && put)) {
522             context.getAuthCache().put(server, new BasicScheme());
523         }
524         if (supportWebDav) {
525             if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
526                 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
527                 try (CloseableHttpResponse response = client.execute(server, req, context)) {
528                     state.setWebDav(response.containsHeader(HttpHeaders.DAV));
529                     EntityUtils.consumeQuietly(response.getEntity());
530                 } catch (IOException e) {
531                     LOGGER.debug("Failed to prepare HTTP context", e);
532                 }
533             }
534             if (put && Boolean.TRUE.equals(state.getWebDav())) {
535                 mkdirs(request.getURI(), context);
536             }
537         }
538     }
539 
540     private void mkdirs(URI uri, SharingHttpContext context) throws Exception {
541         List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
542         int index = 0;
543         for (; index < dirs.size(); index++) {
544             try (CloseableHttpResponse response =
545                     client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
546                 try {
547                     int status = response.getStatusLine().getStatusCode();
548                     if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
549                         break;
550                     } else if (status == HttpStatus.SC_CONFLICT) {
551                         continue;
552                     }
553                     handleStatus(response);
554                 } finally {
555                     EntityUtils.consumeQuietly(response.getEntity());
556                 }
557             } catch (IOException e) {
558                 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
559                 return;
560             }
561         }
562         for (index--; index >= 0; index--) {
563             try (CloseableHttpResponse response =
564                     client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
565                 try {
566                     handleStatus(response);
567                 } finally {
568                     EntityUtils.consumeQuietly(response.getEntity());
569                 }
570             } catch (IOException e) {
571                 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
572                 return;
573             }
574         }
575     }
576 
577     private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
578         request.setEntity(entity);
579         return request;
580     }
581 
582     private boolean isPayloadPresent(HttpUriRequest request) {
583         if (request instanceof HttpEntityEnclosingRequest) {
584             HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
585             return entity != null && entity.getContentLength() != 0;
586         }
587         return false;
588     }
589 
590     private <T extends HttpUriRequest> T commonHeaders(T request) {
591         request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
592         request.setHeader(HttpHeaders.PRAGMA, "no-cache");
593 
594         if (state.isExpectContinue() && isPayloadPresent(request)) {
595             request.setHeader(HttpHeaders.EXPECT, "100-continue");
596         }
597 
598         for (Map.Entry<?, ?> entry : headers.entrySet()) {
599             if (!(entry.getKey() instanceof String)) {
600                 continue;
601             }
602             if (entry.getValue() instanceof String) {
603                 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
604             } else {
605                 request.removeHeaders(entry.getKey().toString());
606             }
607         }
608 
609         if (!state.isExpectContinue()) {
610             request.removeHeaders(HttpHeaders.EXPECT);
611         }
612 
613         return request;
614     }
615 
616     private <T extends HttpUriRequest> void resume(T request, GetTask task) throws IOException {
617         long resumeOffset = task.getResumeOffset();
618         if (resumeOffset > 0L && task.getDataPath() != null) {
619             long lastModified = Files.getLastModifiedTime(task.getDataPath()).toMillis();
620             request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
621             request.setHeader(
622                     HttpHeaders.IF_UNMODIFIED_SINCE, DateUtils.formatDate(new Date(lastModified - 60L * 1000L)));
623             request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
624         }
625     }
626 
627     private void handleStatus(CloseableHttpResponse response) throws Exception {
628         int status = response.getStatusLine().getStatusCode();
629         if (status >= 300) {
630             ApacheRFC9457Reporter.INSTANCE.generateException(response, (statusCode, reasonPhrase) -> {
631                 throw new HttpResponseException(statusCode, reasonPhrase + " (" + statusCode + ")");
632             });
633         }
634     }
635 
636     @Override
637     protected void implClose() {
638         try {
639             client.close();
640         } catch (IOException e) {
641             throw new UncheckedIOException(e);
642         }
643         AuthenticationContext.close(repoAuthContext);
644         AuthenticationContext.close(proxyAuthContext);
645         state.close();
646     }
647 
648     private class EntityGetter {
649 
650         private final GetTask task;
651 
652         EntityGetter(GetTask task) {
653             this.task = task;
654         }
655 
656         public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
657             HttpEntity entity = response.getEntity();
658             if (entity == null) {
659                 entity = new ByteArrayEntity(new byte[0]);
660             }
661 
662             long offset = 0L, length = entity.getContentLength();
663             Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
664             String range = rangeHeader != null ? rangeHeader.getValue() : null;
665             if (range != null) {
666                 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
667                 if (!m.matches()) {
668                     throw new IOException("Invalid Content-Range header for partial download: " + range);
669                 }
670                 offset = Long.parseLong(m.group(1));
671                 length = Long.parseLong(m.group(2)) + 1L;
672                 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
673                     throw new IOException("Invalid Content-Range header for partial download from offset "
674                             + task.getResumeOffset() + ": " + range);
675                 }
676             }
677 
678             final boolean resume = offset > 0L;
679             final Path dataFile = task.getDataPath();
680             if (dataFile == null) {
681                 try (InputStream is = entity.getContent()) {
682                     utilGet(task, is, true, length, resume);
683                     extractChecksums(response);
684                 }
685             } else {
686                 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile)) {
687                     task.setDataPath(tempFile.getPath(), resume);
688                     if (resume && Files.isRegularFile(dataFile)) {
689                         try (InputStream inputStream = Files.newInputStream(dataFile)) {
690                             Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
691                         }
692                     }
693                     try (InputStream is = entity.getContent()) {
694                         utilGet(task, is, true, length, resume);
695                     }
696                     tempFile.move();
697                 } finally {
698                     task.setDataPath(dataFile);
699                 }
700             }
701             if (task.getDataPath() != null) {
702                 Header lastModifiedHeader =
703                         response.getFirstHeader(HttpHeaders.LAST_MODIFIED); // note: Wagon also does first not last
704                 if (lastModifiedHeader != null) {
705                     Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
706                     if (lastModified != null) {
707                         pathProcessor.setLastModified(task.getDataPath(), lastModified.getTime());
708                     }
709                 }
710             }
711             extractChecksums(response);
712         }
713 
714         private void extractChecksums(CloseableHttpResponse response) {
715             Map<String, String> checksums = checksumExtractor.extractChecksums(headerGetter(response));
716             if (checksums != null && !checksums.isEmpty()) {
717                 checksums.forEach(task::setChecksum);
718             }
719         }
720     }
721 
722     private static Function<String, String> headerGetter(CloseableHttpResponse closeableHttpResponse) {
723         return s -> {
724             Header header = closeableHttpResponse.getFirstHeader(s);
725             return header != null ? header.getValue() : null;
726         };
727     }
728 
729     private class PutTaskEntity extends AbstractHttpEntity {
730 
731         private final PutTask task;
732 
733         PutTaskEntity(PutTask task) {
734             this.task = task;
735         }
736 
737         @Override
738         public boolean isRepeatable() {
739             return true;
740         }
741 
742         @Override
743         public boolean isStreaming() {
744             return false;
745         }
746 
747         @Override
748         public long getContentLength() {
749             return task.getDataLength();
750         }
751 
752         @Override
753         public InputStream getContent() throws IOException {
754             return task.newInputStream();
755         }
756 
757         @Override
758         public void writeTo(OutputStream os) throws IOException {
759             try {
760                 utilPut(task, os, false);
761             } catch (TransferCancelledException e) {
762                 throw (IOException) new InterruptedIOException().initCause(e);
763             }
764         }
765     }
766 
767     private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
768         private final int retryCount;
769 
770         private final long retryInterval;
771 
772         private final long retryIntervalMax;
773 
774         private final Set<Integer> serviceUnavailableHttpCodes;
775 
776         /**
777          * Ugly, but forced by HttpClient API {@link ServiceUnavailableRetryStrategy}: the calls for
778          * {@link #retryRequest(HttpResponse, int, HttpContext)} and {@link #getRetryInterval()} are done by same
779          * thread and are actually done from spot that are very close to each other (almost subsequent calls).
780          */
781         private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
782 
783         private ResolverServiceUnavailableRetryStrategy(
784                 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
785             if (retryCount < 0) {
786                 throw new IllegalArgumentException("retryCount must be >= 0");
787             }
788             if (retryInterval < 0L) {
789                 throw new IllegalArgumentException("retryInterval must be >= 0");
790             }
791             if (retryIntervalMax < 0L) {
792                 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
793             }
794             this.retryCount = retryCount;
795             this.retryInterval = retryInterval;
796             this.retryIntervalMax = retryIntervalMax;
797             this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
798         }
799 
800         @Override
801         public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
802             final boolean retry = executionCount <= retryCount
803                     && (serviceUnavailableHttpCodes.contains(
804                             response.getStatusLine().getStatusCode()));
805             if (retry) {
806                 Long retryInterval = retryInterval(response, executionCount, context);
807                 if (retryInterval != null) {
808                     RETRY_INTERVAL_HOLDER.set(retryInterval);
809                     return true;
810                 }
811             }
812             RETRY_INTERVAL_HOLDER.remove();
813             return false;
814         }
815 
816         /**
817          * Calculates retry interval in milliseconds. If {@link HttpHeaders#RETRY_AFTER} header present, it obeys it.
818          * Otherwise, it returns {@link this#retryInterval} long value multiplied with {@code executionCount} (starts
819          * from 1 and goes 2, 3,...).
820          *
821          * @return Long representing the retry interval as millis, or {@code null} if the request should be failed.
822          */
823         private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
824             Long result = null;
825             Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
826             if (header != null && header.getValue() != null) {
827                 String headerValue = header.getValue();
828                 if (headerValue.contains(":")) { // is date when to retry
829                     Date when = DateUtils.parseDate(headerValue); // presumably future
830                     if (when != null) {
831                         result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
832                     }
833                 } else {
834                     try {
835                         result = Long.parseLong(headerValue) * 1000L; // is in seconds
836                     } catch (NumberFormatException e) {
837                         // fall through
838                     }
839                 }
840             }
841             if (result == null) {
842                 result = executionCount * this.retryInterval;
843             }
844             if (result > retryIntervalMax) {
845                 return null;
846             }
847             return result;
848         }
849 
850         @Override
851         public long getRetryInterval() {
852             Long ri = RETRY_INTERVAL_HOLDER.get();
853             if (ri == null) {
854                 return 0L;
855             }
856             RETRY_INTERVAL_HOLDER.remove();
857             return ri;
858         }
859     }
860 }