1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.apache;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.UnknownHostException;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.nio.file.StandardCopyOption;
34 import java.nio.file.attribute.FileTime;
35 import java.util.Collections;
36 import java.util.Date;
37 import java.util.HashMap;
38 import java.util.HashSet;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.Set;
42 import java.util.regex.Matcher;
43 import java.util.regex.Pattern;
44
45 import org.apache.http.Header;
46 import org.apache.http.HttpEntity;
47 import org.apache.http.HttpEntityEnclosingRequest;
48 import org.apache.http.HttpHeaders;
49 import org.apache.http.HttpHost;
50 import org.apache.http.HttpResponse;
51 import org.apache.http.HttpStatus;
52 import org.apache.http.auth.AuthSchemeProvider;
53 import org.apache.http.auth.AuthScope;
54 import org.apache.http.client.CredentialsProvider;
55 import org.apache.http.client.HttpRequestRetryHandler;
56 import org.apache.http.client.HttpResponseException;
57 import org.apache.http.client.ServiceUnavailableRetryStrategy;
58 import org.apache.http.client.config.AuthSchemes;
59 import org.apache.http.client.config.RequestConfig;
60 import org.apache.http.client.methods.CloseableHttpResponse;
61 import org.apache.http.client.methods.HttpGet;
62 import org.apache.http.client.methods.HttpHead;
63 import org.apache.http.client.methods.HttpOptions;
64 import org.apache.http.client.methods.HttpPut;
65 import org.apache.http.client.methods.HttpUriRequest;
66 import org.apache.http.client.utils.DateUtils;
67 import org.apache.http.client.utils.URIUtils;
68 import org.apache.http.config.Registry;
69 import org.apache.http.config.RegistryBuilder;
70 import org.apache.http.config.SocketConfig;
71 import org.apache.http.entity.AbstractHttpEntity;
72 import org.apache.http.entity.ByteArrayEntity;
73 import org.apache.http.impl.NoConnectionReuseStrategy;
74 import org.apache.http.impl.auth.BasicScheme;
75 import org.apache.http.impl.auth.BasicSchemeFactory;
76 import org.apache.http.impl.auth.DigestSchemeFactory;
77 import org.apache.http.impl.auth.KerberosSchemeFactory;
78 import org.apache.http.impl.auth.NTLMSchemeFactory;
79 import org.apache.http.impl.auth.SPNegoSchemeFactory;
80 import org.apache.http.impl.client.CloseableHttpClient;
81 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
82 import org.apache.http.impl.client.HttpClientBuilder;
83 import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
84 import org.apache.http.protocol.HttpContext;
85 import org.apache.http.util.EntityUtils;
86 import org.eclipse.aether.ConfigurationProperties;
87 import org.eclipse.aether.RepositorySystemSession;
88 import org.eclipse.aether.repository.AuthenticationContext;
89 import org.eclipse.aether.repository.Proxy;
90 import org.eclipse.aether.repository.RemoteRepository;
91 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
92 import org.eclipse.aether.spi.connector.transport.GetTask;
93 import org.eclipse.aether.spi.connector.transport.PeekTask;
94 import org.eclipse.aether.spi.connector.transport.PutTask;
95 import org.eclipse.aether.spi.connector.transport.TransportTask;
96 import org.eclipse.aether.transfer.NoTransporterException;
97 import org.eclipse.aether.transfer.TransferCancelledException;
98 import org.eclipse.aether.util.ConfigUtils;
99 import org.eclipse.aether.util.FileUtils;
100 import org.slf4j.Logger;
101 import org.slf4j.LoggerFactory;
102
103 import static java.util.Objects.requireNonNull;
104 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_NAME;
105 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
106 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.CONFIG_PROP_USE_SYSTEM_PROPERTIES;
107 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED;
108 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.DEFAULT_USE_SYSTEM_PROPERTIES;
109 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_DEFAULT;
110 import static org.eclipse.aether.transport.apache.ApacheTransporterConfigurationKeys.HTTP_RETRY_HANDLER_NAME_STANDARD;
111
112
113
114
115 final class ApacheTransporter extends AbstractTransporter {
116 private static final Pattern CONTENT_RANGE_PATTERN =
117 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
118
119 private static final Logger LOGGER = LoggerFactory.getLogger(ApacheTransporter.class);
120
121 private final AuthenticationContext repoAuthContext;
122
123 private final AuthenticationContext proxyAuthContext;
124
125 private final URI baseUri;
126
127 private final HttpHost server;
128
129 private final HttpHost proxy;
130
131 private final CloseableHttpClient client;
132
133 private final Map<?, ?> headers;
134
135 private final LocalState state;
136
137 private final boolean preemptiveAuth;
138
139 private final boolean preemptivePutAuth;
140
141 private final boolean supportWebDav;
142
143 @SuppressWarnings("checkstyle:methodlength")
144 ApacheTransporter(RemoteRepository repository, RepositorySystemSession session) throws NoTransporterException {
145 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
146 throw new NoTransporterException(repository);
147 }
148 try {
149 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
150 if (baseUri.isOpaque()) {
151 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
152 }
153 this.server = URIUtils.extractHost(baseUri);
154 if (server == null) {
155 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
156 }
157 } catch (URISyntaxException e) {
158 throw new NoTransporterException(repository, e.getMessage(), e);
159 }
160 this.proxy = toHost(repository.getProxy());
161
162 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
163 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
164
165 String httpsSecurityMode = ConfigUtils.getString(
166 session,
167 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
168 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
169 ConfigurationProperties.HTTPS_SECURITY_MODE);
170 final int connectionMaxTtlSeconds = ConfigUtils.getInteger(
171 session,
172 ConfigurationProperties.DEFAULT_HTTP_CONNECTION_MAX_TTL,
173 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL + "." + repository.getId(),
174 ConfigurationProperties.HTTP_CONNECTION_MAX_TTL);
175 final int maxConnectionsPerRoute = ConfigUtils.getInteger(
176 session,
177 ConfigurationProperties.DEFAULT_HTTP_MAX_CONNECTIONS_PER_ROUTE,
178 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE + "." + repository.getId(),
179 ConfigurationProperties.HTTP_MAX_CONNECTIONS_PER_ROUTE);
180 this.state = new LocalState(
181 session,
182 repository,
183 new ConnMgrConfig(
184 session, repoAuthContext, httpsSecurityMode, connectionMaxTtlSeconds, maxConnectionsPerRoute));
185
186 this.headers = ConfigUtils.getMap(
187 session,
188 Collections.emptyMap(),
189 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
190 ConfigurationProperties.HTTP_HEADERS);
191
192 this.preemptiveAuth = ConfigUtils.getBoolean(
193 session,
194 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
195 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
196 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
197 this.preemptivePutAuth = ConfigUtils.getBoolean(
198 session,
199 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_PUT_AUTH,
200 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH + "." + repository.getId(),
201 ConfigurationProperties.HTTP_PREEMPTIVE_PUT_AUTH);
202 this.supportWebDav = ConfigUtils.getBoolean(
203 session,
204 ConfigurationProperties.DEFAULT_HTTP_SUPPORT_WEBDAV,
205 ConfigurationProperties.HTTP_SUPPORT_WEBDAV + "." + repository.getId(),
206 ConfigurationProperties.HTTP_SUPPORT_WEBDAV);
207 String credentialEncoding = ConfigUtils.getString(
208 session,
209 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
210 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
211 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
212 int connectTimeout = ConfigUtils.getInteger(
213 session,
214 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
215 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
216 ConfigurationProperties.CONNECT_TIMEOUT);
217 int requestTimeout = ConfigUtils.getInteger(
218 session,
219 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
220 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
221 ConfigurationProperties.REQUEST_TIMEOUT);
222 int retryCount = ConfigUtils.getInteger(
223 session,
224 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
225 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
226 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
227 long retryInterval = ConfigUtils.getLong(
228 session,
229 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL,
230 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL + "." + repository.getId(),
231 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL);
232 long retryIntervalMax = ConfigUtils.getLong(
233 session,
234 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_INTERVAL_MAX,
235 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX + "." + repository.getId(),
236 ConfigurationProperties.HTTP_RETRY_HANDLER_INTERVAL_MAX);
237 String serviceUnavailableCodesString = ConfigUtils.getString(
238 session,
239 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE,
240 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE + "." + repository.getId(),
241 ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE);
242 String retryHandlerName = ConfigUtils.getString(
243 session,
244 HTTP_RETRY_HANDLER_NAME_STANDARD,
245 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + "." + repository.getId(),
246 CONFIG_PROP_HTTP_RETRY_HANDLER_NAME);
247 boolean retryHandlerRequestSentEnabled = ConfigUtils.getBoolean(
248 session,
249 DEFAULT_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED,
250 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED + "." + repository.getId(),
251 CONFIG_PROP_HTTP_RETRY_HANDLER_REQUEST_SENT_ENABLED);
252 String userAgent = ConfigUtils.getString(
253 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
254
255 Charset credentialsCharset = Charset.forName(credentialEncoding);
256 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
257 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
258 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
259 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
260 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
261 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
262 .build();
263 SocketConfig socketConfig =
264 SocketConfig.custom().setSoTimeout(requestTimeout).build();
265 RequestConfig requestConfig = RequestConfig.custom()
266 .setConnectTimeout(connectTimeout)
267 .setConnectionRequestTimeout(connectTimeout)
268 .setLocalAddress(getHttpLocalAddress(session, repository))
269 .setSocketTimeout(requestTimeout)
270 .build();
271
272 HttpRequestRetryHandler retryHandler;
273 if (HTTP_RETRY_HANDLER_NAME_STANDARD.equals(retryHandlerName)) {
274 retryHandler = new StandardHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
275 } else if (HTTP_RETRY_HANDLER_NAME_DEFAULT.equals(retryHandlerName)) {
276 retryHandler = new DefaultHttpRequestRetryHandler(retryCount, retryHandlerRequestSentEnabled);
277 } else {
278 throw new IllegalArgumentException(
279 "Unsupported parameter " + CONFIG_PROP_HTTP_RETRY_HANDLER_NAME + " value: " + retryHandlerName);
280 }
281 Set<Integer> serviceUnavailableCodes = new HashSet<>();
282 try {
283 for (String code : ConfigUtils.parseCommaSeparatedUniqueNames(serviceUnavailableCodesString)) {
284 serviceUnavailableCodes.add(Integer.parseInt(code));
285 }
286 } catch (NumberFormatException e) {
287 throw new IllegalArgumentException(
288 "Illegal HTTP codes for " + ConfigurationProperties.HTTP_RETRY_HANDLER_SERVICE_UNAVAILABLE
289 + " (list of integers): " + serviceUnavailableCodesString);
290 }
291 ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ResolverServiceUnavailableRetryStrategy(
292 retryCount, retryInterval, retryIntervalMax, serviceUnavailableCodes);
293
294 HttpClientBuilder builder = HttpClientBuilder.create()
295 .setUserAgent(userAgent)
296 .setDefaultSocketConfig(socketConfig)
297 .setDefaultRequestConfig(requestConfig)
298 .setServiceUnavailableRetryStrategy(serviceUnavailableRetryStrategy)
299 .setRetryHandler(retryHandler)
300 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
301 .setConnectionManager(state.getConnectionManager())
302 .setConnectionManagerShared(true)
303 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
304 .setProxy(proxy);
305 final boolean useSystemProperties = ConfigUtils.getBoolean(
306 session,
307 DEFAULT_USE_SYSTEM_PROPERTIES,
308 CONFIG_PROP_USE_SYSTEM_PROPERTIES + "." + repository.getId(),
309 CONFIG_PROP_USE_SYSTEM_PROPERTIES);
310 if (useSystemProperties) {
311 LOGGER.warn(
312 "Transport used Apache HttpClient is instructed to use system properties: this may yield in unwanted side-effects!");
313 LOGGER.warn("Please use documented means to configure resolver transport.");
314 builder.useSystemProperties();
315 }
316
317 final String expectContinue = ConfigUtils.getString(
318 session,
319 null,
320 ConfigurationProperties.HTTP_EXPECT_CONTINUE + "." + repository.getId(),
321 ConfigurationProperties.HTTP_EXPECT_CONTINUE);
322 if (expectContinue != null) {
323 state.setExpectContinue(Boolean.parseBoolean(expectContinue));
324 }
325
326 final boolean reuseConnections = ConfigUtils.getBoolean(
327 session,
328 ConfigurationProperties.DEFAULT_HTTP_REUSE_CONNECTIONS,
329 ConfigurationProperties.HTTP_REUSE_CONNECTIONS + "." + repository.getId(),
330 ConfigurationProperties.HTTP_REUSE_CONNECTIONS);
331 if (!reuseConnections) {
332 builder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
333 }
334
335 this.client = builder.build();
336 }
337
338
339
340
341 private InetAddress getHttpLocalAddress(RepositorySystemSession session, RemoteRepository repository) {
342 String bindAddress = ConfigUtils.getString(
343 session,
344 null,
345 ConfigurationProperties.HTTP_LOCAL_ADDRESS + "." + repository.getId(),
346 ConfigurationProperties.HTTP_LOCAL_ADDRESS);
347 if (bindAddress == null) {
348 return null;
349 }
350 try {
351 return InetAddress.getByName(bindAddress);
352 } catch (UnknownHostException uhe) {
353 throw new IllegalArgumentException(
354 "Given bind address (" + bindAddress + ") cannot be resolved for remote repository " + repository,
355 uhe);
356 }
357 }
358
359 private static HttpHost toHost(Proxy proxy) {
360 HttpHost host = null;
361 if (proxy != null) {
362 host = new HttpHost(proxy.getHost(), proxy.getPort());
363 }
364 return host;
365 }
366
367 private static CredentialsProvider toCredentialsProvider(
368 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
369 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
370 if (proxy != null) {
371 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
372 provider = new DemuxCredentialsProvider(provider, p, proxy);
373 }
374 return provider;
375 }
376
377 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
378 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
379 if (ctx != null) {
380 AuthScope basicScope = new AuthScope(host, port);
381 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
382
383 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
384 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
385 }
386 return provider;
387 }
388
389 LocalState getState() {
390 return state;
391 }
392
393 private URI resolve(TransportTask task) {
394 return UriUtils.resolve(baseUri, task.getLocation());
395 }
396
397 @Override
398 public int classify(Throwable error) {
399 if (error instanceof HttpResponseException
400 && ((HttpResponseException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
401 return ERROR_NOT_FOUND;
402 }
403 return ERROR_OTHER;
404 }
405
406 @Override
407 protected void implPeek(PeekTask task) throws Exception {
408 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
409 execute(request, null);
410 }
411
412 @Override
413 protected void implGet(GetTask task) throws Exception {
414 boolean resume = true;
415
416 EntityGetter getter = new EntityGetter(task);
417 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
418 while (true) {
419 try {
420 if (resume) {
421 resume(request, task);
422 }
423 execute(request, getter);
424 break;
425 } catch (HttpResponseException e) {
426 if (resume
427 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
428 && request.containsHeader(HttpHeaders.RANGE)) {
429 request = commonHeaders(new HttpGet(resolve(task)));
430 resume = false;
431 continue;
432 }
433 throw e;
434 }
435 }
436 }
437
438 @Override
439 protected void implPut(PutTask task) throws Exception {
440 PutTaskEntity entity = new PutTaskEntity(task);
441 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
442 try {
443 execute(request, null);
444 } catch (HttpResponseException e) {
445 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
446 state.setExpectContinue(false);
447 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
448 execute(request, null);
449 return;
450 }
451 throw e;
452 }
453 }
454
455 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
456 try {
457 SharingHttpContext context = new SharingHttpContext(state);
458 prepare(request, context);
459 try (CloseableHttpResponse response = client.execute(server, request, context)) {
460 try {
461 context.close();
462 handleStatus(response);
463 if (getter != null) {
464 getter.handle(response);
465 }
466 } finally {
467 EntityUtils.consumeQuietly(response.getEntity());
468 }
469 }
470 } catch (IOException e) {
471 if (e.getCause() instanceof TransferCancelledException) {
472 throw (Exception) e.getCause();
473 }
474 throw e;
475 }
476 }
477
478 private void prepare(HttpUriRequest request, SharingHttpContext context) {
479 final boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
480 if (preemptiveAuth || (preemptivePutAuth && put)) {
481 context.getAuthCache().put(server, new BasicScheme());
482 }
483 if (supportWebDav) {
484 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
485 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
486 try (CloseableHttpResponse response = client.execute(server, req, context)) {
487 state.setWebDav(response.containsHeader(HttpHeaders.DAV));
488 EntityUtils.consumeQuietly(response.getEntity());
489 } catch (IOException e) {
490 LOGGER.debug("Failed to prepare HTTP context", e);
491 }
492 }
493 if (put && Boolean.TRUE.equals(state.getWebDav())) {
494 mkdirs(request.getURI(), context);
495 }
496 }
497 }
498
499 @SuppressWarnings("checkstyle:magicnumber")
500 private void mkdirs(URI uri, SharingHttpContext context) {
501 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
502 int index = 0;
503 for (; index < dirs.size(); index++) {
504 try (CloseableHttpResponse response =
505 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
506 try {
507 int status = response.getStatusLine().getStatusCode();
508 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
509 break;
510 } else if (status == HttpStatus.SC_CONFLICT) {
511 continue;
512 }
513 handleStatus(response);
514 } finally {
515 EntityUtils.consumeQuietly(response.getEntity());
516 }
517 } catch (IOException e) {
518 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
519 return;
520 }
521 }
522 for (index--; index >= 0; index--) {
523 try (CloseableHttpResponse response =
524 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
525 try {
526 handleStatus(response);
527 } finally {
528 EntityUtils.consumeQuietly(response.getEntity());
529 }
530 } catch (IOException e) {
531 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
532 return;
533 }
534 }
535 }
536
537 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
538 request.setEntity(entity);
539 return request;
540 }
541
542 private boolean isPayloadPresent(HttpUriRequest request) {
543 if (request instanceof HttpEntityEnclosingRequest) {
544 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
545 return entity != null && entity.getContentLength() != 0;
546 }
547 return false;
548 }
549
550 private <T extends HttpUriRequest> T commonHeaders(T request) {
551 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
552 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
553
554 if (state.isExpectContinue() && isPayloadPresent(request)) {
555 request.setHeader(HttpHeaders.EXPECT, "100-continue");
556 }
557
558 for (Map.Entry<?, ?> entry : headers.entrySet()) {
559 if (!(entry.getKey() instanceof String)) {
560 continue;
561 }
562 if (entry.getValue() instanceof String) {
563 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
564 } else {
565 request.removeHeaders(entry.getKey().toString());
566 }
567 }
568
569 if (!state.isExpectContinue()) {
570 request.removeHeaders(HttpHeaders.EXPECT);
571 }
572
573 return request;
574 }
575
576 @SuppressWarnings("checkstyle:magicnumber")
577 private <T extends HttpUriRequest> void resume(T request, GetTask task) {
578 long resumeOffset = task.getResumeOffset();
579 if (resumeOffset > 0L && task.getDataFile() != null) {
580 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
581 request.setHeader(
582 HttpHeaders.IF_UNMODIFIED_SINCE,
583 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
584 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
585 }
586 }
587
588 @SuppressWarnings("checkstyle:magicnumber")
589 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
590 int status = response.getStatusLine().getStatusCode();
591 if (status >= 300) {
592 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
593 }
594 }
595
596 @Override
597 protected void implClose() {
598 try {
599 client.close();
600 } catch (IOException e) {
601 throw new UncheckedIOException(e);
602 }
603 AuthenticationContext.close(repoAuthContext);
604 AuthenticationContext.close(proxyAuthContext);
605 state.close();
606 }
607
608 private class EntityGetter {
609
610 private final GetTask task;
611
612 EntityGetter(GetTask task) {
613 this.task = task;
614 }
615
616 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
617 HttpEntity entity = response.getEntity();
618 if (entity == null) {
619 entity = new ByteArrayEntity(new byte[0]);
620 }
621
622 long offset = 0L, length = entity.getContentLength();
623 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
624 String range = rangeHeader != null ? rangeHeader.getValue() : null;
625 if (range != null) {
626 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
627 if (!m.matches()) {
628 throw new IOException("Invalid Content-Range header for partial download: " + range);
629 }
630 offset = Long.parseLong(m.group(1));
631 length = Long.parseLong(m.group(2)) + 1L;
632 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
633 throw new IOException("Invalid Content-Range header for partial download from offset "
634 + task.getResumeOffset() + ": " + range);
635 }
636 }
637
638 final boolean resume = offset > 0L;
639 final File dataFile = task.getDataFile();
640 if (dataFile == null) {
641 try (InputStream is = entity.getContent()) {
642 utilGet(task, is, true, length, resume);
643 extractChecksums(response);
644 }
645 } else {
646 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
647 task.setDataFile(tempFile.getPath().toFile(), resume);
648 if (resume && Files.isRegularFile(dataFile.toPath())) {
649 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
650 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
651 }
652 }
653 try (InputStream is = entity.getContent()) {
654 utilGet(task, is, true, length, resume);
655 }
656 tempFile.move();
657 } finally {
658 task.setDataFile(dataFile);
659 }
660 }
661 if (task.getDataFile() != null) {
662 Header lastModifiedHeader =
663 response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
664 if (lastModifiedHeader != null) {
665 Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
666 if (lastModified != null) {
667 Files.setLastModifiedTime(
668 task.getDataFile().toPath(), FileTime.fromMillis(lastModified.getTime()));
669 }
670 }
671 }
672 extractChecksums(response);
673 }
674
675 private void extractChecksums(CloseableHttpResponse response) {
676 Map<String, String> checksums = extractXChecksums(response);
677 if (checksums == null) {
678 checksums = extractNx2Checksums(response);
679 }
680 if (checksums != null) {
681 checksums.forEach(task::setChecksum);
682 }
683 }
684 }
685
686 private static Map<String, String> extractNx2Checksums(HttpResponse response) {
687
688 Header header = response.getFirstHeader(HttpHeaders.ETAG);
689 String etag = header != null ? header.getValue() : null;
690 if (etag != null) {
691 int start = etag.indexOf("SHA1{"), end = etag.indexOf("}", start + 5);
692 if (start >= 0 && end > start) {
693 return Collections.singletonMap("SHA-1", etag.substring(start + 5, end));
694 }
695 }
696 return null;
697 }
698
699 private static Map<String, String> extractXChecksums(HttpResponse response) {
700 String value;
701 HashMap<String, String> result = new HashMap<>();
702
703 value = extractXChecksum(response, "x-checksum-sha1");
704 if (value != null) {
705 result.put("SHA-1", value);
706 }
707
708 value = extractXChecksum(response, "x-checksum-md5");
709 if (value != null) {
710 result.put("MD5", value);
711 }
712 if (!result.isEmpty()) {
713 return result;
714 }
715
716 value = extractXChecksum(response, "x-goog-meta-checksum-sha1");
717 if (value != null) {
718 result.put("SHA-1", value);
719 }
720
721 value = extractXChecksum(response, "x-goog-meta-checksum-md5");
722 if (value != null) {
723 result.put("MD5", value);
724 }
725
726 return result.isEmpty() ? null : result;
727 }
728
729 private static String extractXChecksum(HttpResponse response, String name) {
730 Header header = response.getFirstHeader(name);
731 return header != null ? header.getValue() : null;
732 }
733
734 private class PutTaskEntity extends AbstractHttpEntity {
735
736 private final PutTask task;
737
738 PutTaskEntity(PutTask task) {
739 this.task = task;
740 }
741
742 @Override
743 public boolean isRepeatable() {
744 return true;
745 }
746
747 @Override
748 public boolean isStreaming() {
749 return false;
750 }
751
752 @Override
753 public long getContentLength() {
754 return task.getDataLength();
755 }
756
757 @Override
758 public InputStream getContent() throws IOException {
759 return task.newInputStream();
760 }
761
762 @Override
763 public void writeTo(OutputStream os) throws IOException {
764 try {
765 utilPut(task, os, false);
766 } catch (TransferCancelledException e) {
767 throw (IOException) new InterruptedIOException().initCause(e);
768 }
769 }
770 }
771
772 private static class ResolverServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
773 private final int retryCount;
774
775 private final long retryInterval;
776
777 private final long retryIntervalMax;
778
779 private final Set<Integer> serviceUnavailableHttpCodes;
780
781
782
783
784
785
786 private static final ThreadLocal<Long> RETRY_INTERVAL_HOLDER = new ThreadLocal<>();
787
788 private ResolverServiceUnavailableRetryStrategy(
789 int retryCount, long retryInterval, long retryIntervalMax, Set<Integer> serviceUnavailableHttpCodes) {
790 if (retryCount < 0) {
791 throw new IllegalArgumentException("retryCount must be >= 0");
792 }
793 if (retryInterval < 0L) {
794 throw new IllegalArgumentException("retryInterval must be >= 0");
795 }
796 if (retryIntervalMax < 0L) {
797 throw new IllegalArgumentException("retryIntervalMax must be >= 0");
798 }
799 this.retryCount = retryCount;
800 this.retryInterval = retryInterval;
801 this.retryIntervalMax = retryIntervalMax;
802 this.serviceUnavailableHttpCodes = requireNonNull(serviceUnavailableHttpCodes);
803 }
804
805 @Override
806 public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) {
807 final boolean retry = executionCount <= retryCount
808 && (serviceUnavailableHttpCodes.contains(
809 response.getStatusLine().getStatusCode()));
810 if (retry) {
811 Long retryInterval = retryInterval(response, executionCount, context);
812 if (retryInterval != null) {
813 RETRY_INTERVAL_HOLDER.set(retryInterval);
814 return true;
815 }
816 }
817 RETRY_INTERVAL_HOLDER.remove();
818 return false;
819 }
820
821
822
823
824
825
826
827
828 private Long retryInterval(HttpResponse httpResponse, int executionCount, HttpContext httpContext) {
829 Long result = null;
830 Header header = httpResponse.getFirstHeader(HttpHeaders.RETRY_AFTER);
831 if (header != null && header.getValue() != null) {
832 String headerValue = header.getValue();
833 if (headerValue.contains(":")) {
834 Date when = DateUtils.parseDate(headerValue);
835 if (when != null) {
836 result = Math.max(when.getTime() - System.currentTimeMillis(), 0L);
837 }
838 } else {
839 try {
840 result = Long.parseLong(headerValue) * 1000L;
841 } catch (NumberFormatException e) {
842
843 }
844 }
845 }
846 if (result == null) {
847 result = executionCount * this.retryInterval;
848 }
849 if (result > retryIntervalMax) {
850 return null;
851 }
852 return result;
853 }
854
855 @Override
856 public long getRetryInterval() {
857 Long ri = RETRY_INTERVAL_HOLDER.get();
858 if (ri == null) {
859 return 0L;
860 }
861 RETRY_INTERVAL_HOLDER.remove();
862 return ri;
863 }
864 }
865 }