1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.aether.transport.http;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InterruptedIOException;
25 import java.io.OutputStream;
26 import java.io.UncheckedIOException;
27 import java.net.URI;
28 import java.net.URISyntaxException;
29 import java.nio.charset.Charset;
30 import java.nio.file.Files;
31 import java.nio.file.StandardCopyOption;
32 import java.util.Collections;
33 import java.util.Date;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.regex.Matcher;
37 import java.util.regex.Pattern;
38
39 import org.apache.http.Header;
40 import org.apache.http.HttpEntity;
41 import org.apache.http.HttpEntityEnclosingRequest;
42 import org.apache.http.HttpHeaders;
43 import org.apache.http.HttpHost;
44 import org.apache.http.HttpStatus;
45 import org.apache.http.auth.AuthSchemeProvider;
46 import org.apache.http.auth.AuthScope;
47 import org.apache.http.client.CredentialsProvider;
48 import org.apache.http.client.HttpResponseException;
49 import org.apache.http.client.config.AuthSchemes;
50 import org.apache.http.client.config.RequestConfig;
51 import org.apache.http.client.methods.CloseableHttpResponse;
52 import org.apache.http.client.methods.HttpGet;
53 import org.apache.http.client.methods.HttpHead;
54 import org.apache.http.client.methods.HttpOptions;
55 import org.apache.http.client.methods.HttpPut;
56 import org.apache.http.client.methods.HttpUriRequest;
57 import org.apache.http.client.utils.DateUtils;
58 import org.apache.http.client.utils.URIUtils;
59 import org.apache.http.config.Registry;
60 import org.apache.http.config.RegistryBuilder;
61 import org.apache.http.config.SocketConfig;
62 import org.apache.http.entity.AbstractHttpEntity;
63 import org.apache.http.entity.ByteArrayEntity;
64 import org.apache.http.impl.auth.BasicScheme;
65 import org.apache.http.impl.auth.BasicSchemeFactory;
66 import org.apache.http.impl.auth.DigestSchemeFactory;
67 import org.apache.http.impl.auth.KerberosSchemeFactory;
68 import org.apache.http.impl.auth.NTLMSchemeFactory;
69 import org.apache.http.impl.auth.SPNegoSchemeFactory;
70 import org.apache.http.impl.client.CloseableHttpClient;
71 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
72 import org.apache.http.impl.client.HttpClientBuilder;
73 import org.apache.http.util.EntityUtils;
74 import org.eclipse.aether.ConfigurationProperties;
75 import org.eclipse.aether.RepositorySystemSession;
76 import org.eclipse.aether.repository.AuthenticationContext;
77 import org.eclipse.aether.repository.Proxy;
78 import org.eclipse.aether.repository.RemoteRepository;
79 import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
80 import org.eclipse.aether.spi.connector.transport.GetTask;
81 import org.eclipse.aether.spi.connector.transport.PeekTask;
82 import org.eclipse.aether.spi.connector.transport.PutTask;
83 import org.eclipse.aether.spi.connector.transport.TransportTask;
84 import org.eclipse.aether.transfer.NoTransporterException;
85 import org.eclipse.aether.transfer.TransferCancelledException;
86 import org.eclipse.aether.util.ConfigUtils;
87 import org.eclipse.aether.util.FileUtils;
88 import org.slf4j.Logger;
89 import org.slf4j.LoggerFactory;
90
91 import static java.util.Objects.requireNonNull;
92
93
94
95
96 final class HttpTransporter extends AbstractTransporter {
97
98 private static final Pattern CONTENT_RANGE_PATTERN =
99 Pattern.compile("\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*");
100
101 private static final Logger LOGGER = LoggerFactory.getLogger(HttpTransporter.class);
102
103 private final Map<String, ChecksumExtractor> checksumExtractors;
104
105 private final AuthenticationContext repoAuthContext;
106
107 private final AuthenticationContext proxyAuthContext;
108
109 private final URI baseUri;
110
111 private final HttpHost server;
112
113 private final HttpHost proxy;
114
115 private final CloseableHttpClient client;
116
117 private final Map<?, ?> headers;
118
119 private final LocalState state;
120
121 private final boolean preemptiveAuth;
122
123 HttpTransporter(
124 Map<String, ChecksumExtractor> checksumExtractors,
125 RemoteRepository repository,
126 RepositorySystemSession session)
127 throws NoTransporterException {
128 if (!"http".equalsIgnoreCase(repository.getProtocol()) && !"https".equalsIgnoreCase(repository.getProtocol())) {
129 throw new NoTransporterException(repository);
130 }
131 this.checksumExtractors = requireNonNull(checksumExtractors, "checksum extractors must not be null");
132 try {
133 this.baseUri = new URI(repository.getUrl()).parseServerAuthority();
134 if (baseUri.isOpaque()) {
135 throw new URISyntaxException(repository.getUrl(), "URL must not be opaque");
136 }
137 this.server = URIUtils.extractHost(baseUri);
138 if (server == null) {
139 throw new URISyntaxException(repository.getUrl(), "URL lacks host name");
140 }
141 } catch (URISyntaxException e) {
142 throw new NoTransporterException(repository, e.getMessage(), e);
143 }
144 this.proxy = toHost(repository.getProxy());
145
146 this.repoAuthContext = AuthenticationContext.forRepository(session, repository);
147 this.proxyAuthContext = AuthenticationContext.forProxy(session, repository);
148
149 String httpsSecurityMode = ConfigUtils.getString(
150 session,
151 ConfigurationProperties.HTTPS_SECURITY_MODE_DEFAULT,
152 ConfigurationProperties.HTTPS_SECURITY_MODE + "." + repository.getId(),
153 ConfigurationProperties.HTTPS_SECURITY_MODE);
154 this.state = new LocalState(session, repository, new SslConfig(session, repoAuthContext, httpsSecurityMode));
155
156 this.headers = ConfigUtils.getMap(
157 session,
158 Collections.emptyMap(),
159 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
160 ConfigurationProperties.HTTP_HEADERS);
161
162 this.preemptiveAuth = ConfigUtils.getBoolean(
163 session,
164 ConfigurationProperties.DEFAULT_HTTP_PREEMPTIVE_AUTH,
165 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH + "." + repository.getId(),
166 ConfigurationProperties.HTTP_PREEMPTIVE_AUTH);
167 String credentialEncoding = ConfigUtils.getString(
168 session,
169 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
170 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
171 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING);
172 int connectTimeout = ConfigUtils.getInteger(
173 session,
174 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
175 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
176 ConfigurationProperties.CONNECT_TIMEOUT);
177 int requestTimeout = ConfigUtils.getInteger(
178 session,
179 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
180 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
181 ConfigurationProperties.REQUEST_TIMEOUT);
182 int retryCount = ConfigUtils.getInteger(
183 session,
184 ConfigurationProperties.DEFAULT_HTTP_RETRY_HANDLER_COUNT,
185 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT + "." + repository.getId(),
186 ConfigurationProperties.HTTP_RETRY_HANDLER_COUNT);
187 String userAgent = ConfigUtils.getString(
188 session, ConfigurationProperties.DEFAULT_USER_AGENT, ConfigurationProperties.USER_AGENT);
189
190 Charset credentialsCharset = Charset.forName(credentialEncoding);
191
192 Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
193 .register(AuthSchemes.BASIC, new BasicSchemeFactory(credentialsCharset))
194 .register(AuthSchemes.DIGEST, new DigestSchemeFactory(credentialsCharset))
195 .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
196 .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
197 .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
198 .build();
199
200 SocketConfig socketConfig =
201 SocketConfig.custom().setSoTimeout(requestTimeout).build();
202
203 RequestConfig requestConfig = RequestConfig.custom()
204 .setConnectTimeout(connectTimeout)
205 .setConnectionRequestTimeout(connectTimeout)
206 .setSocketTimeout(requestTimeout)
207 .build();
208
209 DefaultHttpRequestRetryHandler retryHandler = new DefaultHttpRequestRetryHandler(retryCount, false);
210
211 this.client = HttpClientBuilder.create()
212 .setUserAgent(userAgent)
213 .setDefaultSocketConfig(socketConfig)
214 .setDefaultRequestConfig(requestConfig)
215 .setRetryHandler(retryHandler)
216 .setDefaultAuthSchemeRegistry(authSchemeRegistry)
217 .setConnectionManager(state.getConnectionManager())
218 .setConnectionManagerShared(true)
219 .setDefaultCredentialsProvider(toCredentialsProvider(server, repoAuthContext, proxy, proxyAuthContext))
220 .setProxy(proxy)
221 .build();
222 }
223
224 private static HttpHost toHost(Proxy proxy) {
225 HttpHost host = null;
226 if (proxy != null) {
227 host = new HttpHost(proxy.getHost(), proxy.getPort());
228 }
229 return host;
230 }
231
232 private static CredentialsProvider toCredentialsProvider(
233 HttpHost server, AuthenticationContext serverAuthCtx, HttpHost proxy, AuthenticationContext proxyAuthCtx) {
234 CredentialsProvider provider = toCredentialsProvider(server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx);
235 if (proxy != null) {
236 CredentialsProvider p = toCredentialsProvider(proxy.getHostName(), proxy.getPort(), proxyAuthCtx);
237 provider = new DemuxCredentialsProvider(provider, p, proxy);
238 }
239 return provider;
240 }
241
242 private static CredentialsProvider toCredentialsProvider(String host, int port, AuthenticationContext ctx) {
243 DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
244 if (ctx != null) {
245 AuthScope basicScope = new AuthScope(host, port);
246 provider.setCredentials(basicScope, new DeferredCredentialsProvider.BasicFactory(ctx));
247
248 AuthScope ntlmScope = new AuthScope(host, port, AuthScope.ANY_REALM, "ntlm");
249 provider.setCredentials(ntlmScope, new DeferredCredentialsProvider.NtlmFactory(ctx));
250 }
251 return provider;
252 }
253
254 LocalState getState() {
255 return state;
256 }
257
258 private URI resolve(TransportTask task) {
259 return UriUtils.resolve(baseUri, task.getLocation());
260 }
261
262 @Override
263 public int classify(Throwable error) {
264 if (error instanceof HttpResponseException
265 && ((HttpResponseException) error).getStatusCode() == HttpStatus.SC_NOT_FOUND) {
266 return ERROR_NOT_FOUND;
267 }
268 return ERROR_OTHER;
269 }
270
271 @Override
272 protected void implPeek(PeekTask task) throws Exception {
273 HttpHead request = commonHeaders(new HttpHead(resolve(task)));
274 execute(request, null);
275 }
276
277 @Override
278 protected void implGet(GetTask task) throws Exception {
279 boolean resume = true;
280 boolean applyChecksumExtractors = true;
281
282 EntityGetter getter = new EntityGetter(task);
283 HttpGet request = commonHeaders(new HttpGet(resolve(task)));
284 while (true) {
285 try {
286 if (resume) {
287 resume(request, task);
288 }
289 if (applyChecksumExtractors) {
290 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
291 checksumExtractor.prepareRequest(request);
292 }
293 }
294 execute(request, getter);
295 break;
296 } catch (HttpResponseException e) {
297 if (resume
298 && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
299 && request.containsHeader(HttpHeaders.RANGE)) {
300 request = commonHeaders(new HttpGet(resolve(task)));
301 resume = false;
302 continue;
303 }
304 if (applyChecksumExtractors) {
305 boolean retryWithoutExtractors = false;
306 for (ChecksumExtractor checksumExtractor : checksumExtractors.values()) {
307 if (checksumExtractor.retryWithoutExtractor(e)) {
308 retryWithoutExtractors = true;
309 break;
310 }
311 }
312 if (retryWithoutExtractors) {
313 request = commonHeaders(new HttpGet(resolve(task)));
314 applyChecksumExtractors = false;
315 continue;
316 }
317 }
318 throw e;
319 }
320 }
321 }
322
323 @Override
324 protected void implPut(PutTask task) throws Exception {
325 PutTaskEntity entity = new PutTaskEntity(task);
326 HttpPut request = commonHeaders(entity(new HttpPut(resolve(task)), entity));
327 try {
328 execute(request, null);
329 } catch (HttpResponseException e) {
330 if (e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader(HttpHeaders.EXPECT)) {
331 state.setExpectContinue(false);
332 request = commonHeaders(entity(new HttpPut(request.getURI()), entity));
333 execute(request, null);
334 return;
335 }
336 throw e;
337 }
338 }
339
340 private void execute(HttpUriRequest request, EntityGetter getter) throws Exception {
341 try {
342 SharingHttpContext context = new SharingHttpContext(state);
343 prepare(request, context);
344 try (CloseableHttpResponse response = client.execute(server, request, context)) {
345 try {
346 context.close();
347 handleStatus(response);
348 if (getter != null) {
349 getter.handle(response);
350 }
351 } finally {
352 EntityUtils.consumeQuietly(response.getEntity());
353 }
354 }
355 } catch (IOException e) {
356 if (e.getCause() instanceof TransferCancelledException) {
357 throw (Exception) e.getCause();
358 }
359 throw e;
360 }
361 }
362
363 private void prepare(HttpUriRequest request, SharingHttpContext context) {
364 if (preemptiveAuth) {
365 state.setAuthScheme(server, new BasicScheme());
366 }
367 boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase(request.getMethod());
368 if (state.getWebDav() == null && (put || isPayloadPresent(request))) {
369 HttpOptions req = commonHeaders(new HttpOptions(request.getURI()));
370 try (CloseableHttpResponse response = client.execute(server, req, context)) {
371 state.setWebDav(isWebDav(response));
372 EntityUtils.consumeQuietly(response.getEntity());
373 } catch (IOException e) {
374 LOGGER.debug("Failed to prepare HTTP context", e);
375 }
376 }
377 if (put && Boolean.TRUE.equals(state.getWebDav())) {
378 mkdirs(request.getURI(), context);
379 }
380 }
381
382 private boolean isWebDav(CloseableHttpResponse response) {
383 return response.containsHeader(HttpHeaders.DAV);
384 }
385
386 @SuppressWarnings("checkstyle:magicnumber")
387 private void mkdirs(URI uri, SharingHttpContext context) {
388 List<URI> dirs = UriUtils.getDirectories(baseUri, uri);
389 int index = 0;
390 for (; index < dirs.size(); index++) {
391 try (CloseableHttpResponse response =
392 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
393 try {
394 int status = response.getStatusLine().getStatusCode();
395 if (status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED) {
396 break;
397 } else if (status == HttpStatus.SC_CONFLICT) {
398 continue;
399 }
400 handleStatus(response);
401 } finally {
402 EntityUtils.consumeQuietly(response.getEntity());
403 }
404 } catch (IOException e) {
405 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
406 return;
407 }
408 }
409 for (index--; index >= 0; index--) {
410 try (CloseableHttpResponse response =
411 client.execute(server, commonHeaders(new HttpMkCol(dirs.get(index))), context)) {
412 try {
413 handleStatus(response);
414 } finally {
415 EntityUtils.consumeQuietly(response.getEntity());
416 }
417 } catch (IOException e) {
418 LOGGER.debug("Failed to create parent directory {}", dirs.get(index), e);
419 return;
420 }
421 }
422 }
423
424 private <T extends HttpEntityEnclosingRequest> T entity(T request, HttpEntity entity) {
425 request.setEntity(entity);
426 return request;
427 }
428
429 private boolean isPayloadPresent(HttpUriRequest request) {
430 if (request instanceof HttpEntityEnclosingRequest) {
431 HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
432 return entity != null && entity.getContentLength() != 0;
433 }
434 return false;
435 }
436
437 private <T extends HttpUriRequest> T commonHeaders(T request) {
438 request.setHeader(HttpHeaders.CACHE_CONTROL, "no-cache, no-store");
439 request.setHeader(HttpHeaders.PRAGMA, "no-cache");
440
441 if (state.isExpectContinue() && isPayloadPresent(request)) {
442 request.setHeader(HttpHeaders.EXPECT, "100-continue");
443 }
444
445 for (Map.Entry<?, ?> entry : headers.entrySet()) {
446 if (!(entry.getKey() instanceof String)) {
447 continue;
448 }
449 if (entry.getValue() instanceof String) {
450 request.setHeader(entry.getKey().toString(), entry.getValue().toString());
451 } else {
452 request.removeHeaders(entry.getKey().toString());
453 }
454 }
455
456 if (!state.isExpectContinue()) {
457 request.removeHeaders(HttpHeaders.EXPECT);
458 }
459
460 return request;
461 }
462
463 @SuppressWarnings("checkstyle:magicnumber")
464 private <T extends HttpUriRequest> T resume(T request, GetTask task) {
465 long resumeOffset = task.getResumeOffset();
466 if (resumeOffset > 0L && task.getDataFile() != null) {
467 request.setHeader(HttpHeaders.RANGE, "bytes=" + resumeOffset + '-');
468 request.setHeader(
469 HttpHeaders.IF_UNMODIFIED_SINCE,
470 DateUtils.formatDate(new Date(task.getDataFile().lastModified() - 60L * 1000L)));
471 request.setHeader(HttpHeaders.ACCEPT_ENCODING, "identity");
472 }
473 return request;
474 }
475
476 @SuppressWarnings("checkstyle:magicnumber")
477 private void handleStatus(CloseableHttpResponse response) throws HttpResponseException {
478 int status = response.getStatusLine().getStatusCode();
479 if (status >= 300) {
480 throw new HttpResponseException(status, response.getStatusLine().getReasonPhrase() + " (" + status + ")");
481 }
482 }
483
484 @Override
485 protected void implClose() {
486 try {
487 client.close();
488 } catch (IOException e) {
489 throw new UncheckedIOException(e);
490 }
491 AuthenticationContext.close(repoAuthContext);
492 AuthenticationContext.close(proxyAuthContext);
493 state.close();
494 }
495
496 private class EntityGetter {
497
498 private final GetTask task;
499
500 EntityGetter(GetTask task) {
501 this.task = task;
502 }
503
504 public void handle(CloseableHttpResponse response) throws IOException, TransferCancelledException {
505 HttpEntity entity = response.getEntity();
506 if (entity == null) {
507 entity = new ByteArrayEntity(new byte[0]);
508 }
509
510 long offset = 0L, length = entity.getContentLength();
511 Header rangeHeader = response.getFirstHeader(HttpHeaders.CONTENT_RANGE);
512 String range = rangeHeader != null ? rangeHeader.getValue() : null;
513 if (range != null) {
514 Matcher m = CONTENT_RANGE_PATTERN.matcher(range);
515 if (!m.matches()) {
516 throw new IOException("Invalid Content-Range header for partial download: " + range);
517 }
518 offset = Long.parseLong(m.group(1));
519 length = Long.parseLong(m.group(2)) + 1L;
520 if (offset < 0L || offset >= length || (offset > 0L && offset != task.getResumeOffset())) {
521 throw new IOException("Invalid Content-Range header for partial download from offset "
522 + task.getResumeOffset() + ": " + range);
523 }
524 }
525
526 final boolean resume = offset > 0L;
527 final File dataFile = task.getDataFile();
528 if (dataFile == null) {
529 try (InputStream is = entity.getContent()) {
530 utilGet(task, is, true, length, resume);
531 extractChecksums(response);
532 }
533 } else {
534 try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile.toPath())) {
535 task.setDataFile(tempFile.getPath().toFile(), resume);
536 if (resume && Files.isRegularFile(dataFile.toPath())) {
537 try (InputStream inputStream = Files.newInputStream(dataFile.toPath())) {
538 Files.copy(inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING);
539 }
540 }
541 try (InputStream is = entity.getContent()) {
542 utilGet(task, is, true, length, resume);
543 }
544 tempFile.move();
545 } finally {
546 task.setDataFile(dataFile);
547 }
548 }
549 extractChecksums(response);
550 }
551
552 private void extractChecksums(CloseableHttpResponse response) {
553 for (Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet()) {
554 Map<String, String> checksums = extractorEntry.getValue().extractChecksums(response);
555 if (checksums != null) {
556 checksums.forEach(task::setChecksum);
557 return;
558 }
559 }
560 }
561 }
562
563 private class PutTaskEntity extends AbstractHttpEntity {
564
565 private final PutTask task;
566
567 PutTaskEntity(PutTask task) {
568 this.task = task;
569 }
570
571 @Override
572 public boolean isRepeatable() {
573 return true;
574 }
575
576 @Override
577 public boolean isStreaming() {
578 return false;
579 }
580
581 @Override
582 public long getContentLength() {
583 return task.getDataLength();
584 }
585
586 @Override
587 public InputStream getContent() throws IOException {
588 return task.newInputStream();
589 }
590
591 @Override
592 public void writeTo(OutputStream os) throws IOException {
593 try {
594 utilPut(task, os, false);
595 } catch (TransferCancelledException e) {
596 throw (IOException) new InterruptedIOException().initCause(e);
597 }
598 }
599 }
600 }