View Javadoc
1   package org.eclipse.aether.transport.http;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   * 
12   *  http://www.apache.org/licenses/LICENSE-2.0
13   * 
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.http.Header;
23  import org.apache.http.HttpEntity;
24  import org.apache.http.HttpEntityEnclosingRequest;
25  import org.apache.http.HttpHeaders;
26  import org.apache.http.HttpHost;
27  import org.apache.http.HttpStatus;
28  import org.apache.http.auth.AuthSchemeProvider;
29  import org.apache.http.auth.AuthScope;
30  import org.apache.http.client.CredentialsProvider;
31  import org.apache.http.client.HttpResponseException;
32  import org.apache.http.client.config.AuthSchemes;
33  import org.apache.http.client.config.RequestConfig;
34  import org.apache.http.client.methods.CloseableHttpResponse;
35  import org.apache.http.client.methods.HttpGet;
36  import org.apache.http.client.methods.HttpHead;
37  import org.apache.http.client.methods.HttpOptions;
38  import org.apache.http.client.methods.HttpPut;
39  import org.apache.http.client.methods.HttpUriRequest;
40  import org.apache.http.client.utils.DateUtils;
41  import org.apache.http.client.utils.URIUtils;
42  import org.apache.http.config.Registry;
43  import org.apache.http.config.RegistryBuilder;
44  import org.apache.http.config.SocketConfig;
45  import org.apache.http.entity.AbstractHttpEntity;
46  import org.apache.http.entity.ByteArrayEntity;
47  import org.apache.http.impl.auth.BasicSchemeFactory;
48  import org.apache.http.impl.auth.DigestSchemeFactory;
49  import org.apache.http.impl.auth.KerberosSchemeFactory;
50  import org.apache.http.impl.auth.NTLMSchemeFactory;
51  import org.apache.http.impl.auth.SPNegoSchemeFactory;
52  import org.apache.http.impl.client.CloseableHttpClient;
53  import org.apache.http.impl.client.HttpClientBuilder;
54  import org.apache.http.util.EntityUtils;
55  import org.eclipse.aether.ConfigurationProperties;
56  import org.eclipse.aether.RepositorySystemSession;
57  import org.eclipse.aether.repository.AuthenticationContext;
58  import org.eclipse.aether.repository.Proxy;
59  import org.eclipse.aether.repository.RemoteRepository;
60  import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
61  import org.eclipse.aether.spi.connector.transport.GetTask;
62  import org.eclipse.aether.spi.connector.transport.PeekTask;
63  import org.eclipse.aether.spi.connector.transport.PutTask;
64  import org.eclipse.aether.spi.connector.transport.TransportTask;
65  import org.eclipse.aether.transfer.NoTransporterException;
66  import org.eclipse.aether.transfer.TransferCancelledException;
67  import org.eclipse.aether.util.ConfigUtils;
68  import org.eclipse.aether.util.FileUtils;
69  import org.slf4j.Logger;
70  import org.slf4j.LoggerFactory;
71  
72  import java.io.File;
73  import java.io.IOException;
74  import java.io.InputStream;
75  import java.io.InterruptedIOException;
76  import java.io.OutputStream;
77  import java.io.UncheckedIOException;
78  import java.net.URI;
79  import java.net.URISyntaxException;
80  import java.nio.charset.Charset;
81  import java.nio.file.Files;
82  import java.nio.file.StandardCopyOption;
83  import java.util.Collections;
84  import java.util.Date;
85  import java.util.List;
86  import java.util.Map;
87  import java.util.regex.Matcher;
88  import java.util.regex.Pattern;
89  
90  import static java.util.Objects.requireNonNull;
91  
92  /**
93   * A transporter for HTTP/HTTPS.
94   */
95  final class HttpTransporter
96      extends AbstractTransporter
97  {
98  
99      private static final Pattern CONTENT_RANGE_PATTERN =
100         Pattern.compile( "\\s*bytes\\s+([0-9]+)\\s*-\\s*([0-9]+)\\s*/.*" );
101 
102     private static final Logger LOGGER = LoggerFactory.getLogger( HttpTransporter.class );
103 
104     private final Map<String, ChecksumExtractor> checksumExtractors;
105 
106     private final AuthenticationContext repoAuthContext;
107 
108     private final AuthenticationContext proxyAuthContext;
109 
110     private final URI baseUri;
111 
112     private final HttpHost server;
113 
114     private final HttpHost proxy;
115 
116     private final CloseableHttpClient client;
117 
118     private final Map<?, ?> headers;
119 
120     private final LocalState state;
121 
122     HttpTransporter( Map<String, ChecksumExtractor> checksumExtractors,
123                      RemoteRepository repository,
124                      RepositorySystemSession session )
125         throws NoTransporterException
126     {
127         if ( !"http".equalsIgnoreCase( repository.getProtocol() )
128             && !"https".equalsIgnoreCase( repository.getProtocol() ) )
129         {
130             throw new NoTransporterException( repository );
131         }
132         this.checksumExtractors = requireNonNull( checksumExtractors, "checksum extractors must not be null" );
133         try
134         {
135             this.baseUri = new URI( repository.getUrl() ).parseServerAuthority();
136             if ( baseUri.isOpaque() )
137             {
138                 throw new URISyntaxException( repository.getUrl(), "URL must not be opaque" );
139             }
140             this.server = URIUtils.extractHost( baseUri );
141             if ( server == null )
142             {
143                 throw new URISyntaxException( repository.getUrl(), "URL lacks host name" );
144             }
145         }
146         catch ( URISyntaxException e )
147         {
148             throw new NoTransporterException( repository, e.getMessage(), e );
149         }
150         this.proxy = toHost( repository.getProxy() );
151 
152         this.repoAuthContext = AuthenticationContext.forRepository( session, repository );
153         this.proxyAuthContext = AuthenticationContext.forProxy( session, repository );
154 
155         this.state = new LocalState( session, repository, new SslConfig( session, repoAuthContext ) );
156 
157         this.headers = ConfigUtils.getMap( session, Collections.emptyMap(),
158                 ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(),
159                 ConfigurationProperties.HTTP_HEADERS );
160 
161         String credentialEncoding = ConfigUtils.getString( session,
162                 ConfigurationProperties.DEFAULT_HTTP_CREDENTIAL_ENCODING,
163                 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING + "." + repository.getId(),
164                 ConfigurationProperties.HTTP_CREDENTIAL_ENCODING );
165         int connectTimeout = ConfigUtils.getInteger( session,
166                 ConfigurationProperties.DEFAULT_CONNECT_TIMEOUT,
167                 ConfigurationProperties.CONNECT_TIMEOUT + "." + repository.getId(),
168                 ConfigurationProperties.CONNECT_TIMEOUT );
169         int requestTimeout = ConfigUtils.getInteger( session,
170                 ConfigurationProperties.DEFAULT_REQUEST_TIMEOUT,
171                 ConfigurationProperties.REQUEST_TIMEOUT + "." + repository.getId(),
172                 ConfigurationProperties.REQUEST_TIMEOUT );
173         String userAgent = ConfigUtils.getString( session,
174                 ConfigurationProperties.DEFAULT_USER_AGENT,
175                 ConfigurationProperties.USER_AGENT );
176 
177         Charset credentialsCharset = Charset.forName( credentialEncoding );
178 
179         Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
180                 .register( AuthSchemes.BASIC, new BasicSchemeFactory( credentialsCharset ) )
181                 .register( AuthSchemes.DIGEST, new DigestSchemeFactory( credentialsCharset ) )
182                 .register( AuthSchemes.NTLM, new NTLMSchemeFactory() )
183                 .register( AuthSchemes.SPNEGO, new SPNegoSchemeFactory() )
184                 .register( AuthSchemes.KERBEROS, new KerberosSchemeFactory() )
185                 .build();
186 
187         SocketConfig socketConfig = SocketConfig.custom()
188                  .setSoTimeout( requestTimeout ).build();
189 
190         RequestConfig requestConfig = RequestConfig.custom()
191                 .setConnectTimeout( connectTimeout )
192                 .setConnectionRequestTimeout( connectTimeout )
193                 .setSocketTimeout( requestTimeout ).build();
194 
195         this.client = HttpClientBuilder.create()
196                 .setUserAgent( userAgent )
197                 .setDefaultSocketConfig( socketConfig )
198                 .setDefaultRequestConfig( requestConfig )
199                 .setDefaultAuthSchemeRegistry( authSchemeRegistry )
200                 .setConnectionManager( state.getConnectionManager() )
201                 .setConnectionManagerShared( true )
202                 .setDefaultCredentialsProvider(
203                        toCredentialsProvider( server, repoAuthContext, proxy, proxyAuthContext )
204                 )
205                 .setProxy( proxy )
206                 .build();
207     }
208 
209     private static HttpHost toHost( Proxy proxy )
210     {
211         HttpHost host = null;
212         if ( proxy != null )
213         {
214             host = new HttpHost( proxy.getHost(), proxy.getPort() );
215         }
216         return host;
217     }
218 
219     private static CredentialsProvider toCredentialsProvider( HttpHost server, AuthenticationContext serverAuthCtx,
220                                                               HttpHost proxy, AuthenticationContext proxyAuthCtx )
221     {
222         CredentialsProvider provider = toCredentialsProvider( server.getHostName(), AuthScope.ANY_PORT, serverAuthCtx );
223         if ( proxy != null )
224         {
225             CredentialsProvider p = toCredentialsProvider( proxy.getHostName(), proxy.getPort(), proxyAuthCtx );
226             provider = new DemuxCredentialsProvider( provider, p, proxy );
227         }
228         return provider;
229     }
230 
231     private static CredentialsProvider toCredentialsProvider( String host, int port, AuthenticationContext ctx )
232     {
233         DeferredCredentialsProvider provider = new DeferredCredentialsProvider();
234         if ( ctx != null )
235         {
236             AuthScope basicScope = new AuthScope( host, port );
237             provider.setCredentials( basicScope, new DeferredCredentialsProvider.BasicFactory( ctx ) );
238 
239             AuthScope ntlmScope = new AuthScope( host, port, AuthScope.ANY_REALM, "ntlm" );
240             provider.setCredentials( ntlmScope, new DeferredCredentialsProvider.NtlmFactory( ctx ) );
241         }
242         return provider;
243     }
244 
245     LocalState getState()
246     {
247         return state;
248     }
249 
250     private URI resolve( TransportTask task )
251     {
252         return UriUtils.resolve( baseUri, task.getLocation() );
253     }
254 
255     @Override
256     public int classify( Throwable error )
257     {
258         if ( error instanceof HttpResponseException
259             && ( (HttpResponseException) error ).getStatusCode() == HttpStatus.SC_NOT_FOUND )
260         {
261             return ERROR_NOT_FOUND;
262         }
263         return ERROR_OTHER;
264     }
265 
266     @Override
267     protected void implPeek( PeekTask task )
268         throws Exception
269     {
270         HttpHead request = commonHeaders( new HttpHead( resolve( task ) ) );
271         execute( request, null );
272     }
273 
274     @Override
275     protected void implGet( GetTask task )
276         throws Exception
277     {
278         boolean resume = true;
279         boolean applyChecksumExtractors = true;
280 
281         EntityGetter getter = new EntityGetter( task );
282         HttpGet request = commonHeaders( new HttpGet( resolve( task ) ) );
283         while ( true )
284         {
285             try
286             {
287                 if ( resume )
288                 {
289                     resume( request, task );
290                 }
291                 if ( applyChecksumExtractors )
292                 {
293                     for ( ChecksumExtractor checksumExtractor : checksumExtractors.values() )
294                     {
295                         checksumExtractor.prepareRequest( request );
296                     }
297                 }
298                 execute( request, getter );
299                 break;
300             }
301             catch ( HttpResponseException e )
302             {
303                 if ( resume && e.getStatusCode() == HttpStatus.SC_PRECONDITION_FAILED
304                         && request.containsHeader( HttpHeaders.RANGE ) )
305                 {
306                     request = commonHeaders( new HttpGet( resolve( task ) ) );
307                     resume = false;
308                     continue;
309                 }
310                 if ( applyChecksumExtractors )
311                 {
312                     boolean retryWithoutExtractors = false;
313                     for ( ChecksumExtractor checksumExtractor : checksumExtractors.values() )
314                     {
315                         if ( checksumExtractor.retryWithoutExtractor( e ) )
316                         {
317                             retryWithoutExtractors = true;
318                             break;
319                         }
320                     }
321                     if ( retryWithoutExtractors )
322                     {
323                         request = commonHeaders( new HttpGet( resolve( task ) ) );
324                         applyChecksumExtractors = false;
325                         continue;
326                     }
327                 }
328                 throw e;
329             }
330         }
331     }
332 
333     @Override
334     protected void implPut( PutTask task )
335         throws Exception
336     {
337         PutTaskEntity entity = new PutTaskEntity( task );
338         HttpPut request = commonHeaders( entity( new HttpPut( resolve( task ) ), entity ) );
339         try
340         {
341             execute( request, null );
342         }
343         catch ( HttpResponseException e )
344         {
345             if ( e.getStatusCode() == HttpStatus.SC_EXPECTATION_FAILED && request.containsHeader( HttpHeaders.EXPECT ) )
346             {
347                 state.setExpectContinue( false );
348                 request = commonHeaders( entity( new HttpPut( request.getURI() ), entity ) );
349                 execute( request, null );
350                 return;
351             }
352             throw e;
353         }
354     }
355 
356     private void execute( HttpUriRequest request, EntityGetter getter )
357         throws Exception
358     {
359         try
360         {
361             SharingHttpContext context = new SharingHttpContext( state );
362             prepare( request, context );
363             try ( CloseableHttpResponse response = client.execute( server, request, context ) )
364             {
365                 try
366                 {
367                     context.close();
368                     handleStatus( response );
369                     if ( getter != null )
370                     {
371                         getter.handle( response );
372                     }
373                 }
374                 finally
375                 {
376                     EntityUtils.consumeQuietly( response.getEntity() );
377                 }
378             }
379         }
380         catch ( IOException e )
381         {
382             if ( e.getCause() instanceof TransferCancelledException )
383             {
384                 throw (Exception) e.getCause();
385             }
386             throw e;
387         }
388     }
389 
390     private void prepare( HttpUriRequest request, SharingHttpContext context )
391     {
392         boolean put = HttpPut.METHOD_NAME.equalsIgnoreCase( request.getMethod() );
393         if ( state.getWebDav() == null && ( put || isPayloadPresent( request ) ) )
394         {
395             HttpOptions req = commonHeaders( new HttpOptions( request.getURI() ) );
396             try ( CloseableHttpResponse response = client.execute( server, req, context ) )
397             {
398                 state.setWebDav( isWebDav( response ) );
399                 EntityUtils.consumeQuietly( response.getEntity() );
400             }
401             catch ( IOException e )
402             {
403                 LOGGER.debug( "Failed to prepare HTTP context", e );
404             }
405         }
406         if ( put && Boolean.TRUE.equals( state.getWebDav() ) )
407         {
408             mkdirs( request.getURI(), context );
409         }
410     }
411 
412     private boolean isWebDav( CloseableHttpResponse response )
413     {
414         return response.containsHeader( HttpHeaders.DAV );
415     }
416 
417     @SuppressWarnings( "checkstyle:magicnumber" )
418     private void mkdirs( URI uri, SharingHttpContext context )
419     {
420         List<URI> dirs = UriUtils.getDirectories( baseUri, uri );
421         int index = 0;
422         for ( ; index < dirs.size(); index++ )
423         {
424             try ( CloseableHttpResponse response =
425                           client.execute( server, commonHeaders( new HttpMkCol( dirs.get( index ) ) ), context ) )
426             {
427                 try
428                 {
429                     int status = response.getStatusLine().getStatusCode();
430                     if ( status < 300 || status == HttpStatus.SC_METHOD_NOT_ALLOWED )
431                     {
432                         break;
433                     }
434                     else if ( status == HttpStatus.SC_CONFLICT )
435                     {
436                         continue;
437                     }
438                     handleStatus( response );
439                 }
440                 finally
441                 {
442                     EntityUtils.consumeQuietly( response.getEntity() );
443                 }
444             }
445             catch ( IOException e )
446             {
447                 LOGGER.debug( "Failed to create parent directory {}", dirs.get( index ), e );
448                 return;
449             }
450         }
451         for ( index--; index >= 0; index-- )
452         {
453             try ( CloseableHttpResponse response =
454                           client.execute( server, commonHeaders( new HttpMkCol( dirs.get( index ) ) ), context ) )
455             {
456                 try
457                 {
458                     handleStatus( response );
459                 }
460                 finally
461                 {
462                     EntityUtils.consumeQuietly( response.getEntity() );
463                 }
464             }
465             catch ( IOException e )
466             {
467                 LOGGER.debug( "Failed to create parent directory {}", dirs.get( index ), e );
468                 return;
469             }
470         }
471     }
472 
473     private <T extends HttpEntityEnclosingRequest> T entity( T request, HttpEntity entity )
474     {
475         request.setEntity( entity );
476         return request;
477     }
478 
479     private boolean isPayloadPresent( HttpUriRequest request )
480     {
481         if ( request instanceof HttpEntityEnclosingRequest )
482         {
483             HttpEntity entity = ( (HttpEntityEnclosingRequest) request ).getEntity();
484             return entity != null && entity.getContentLength() != 0;
485         }
486         return false;
487     }
488 
489     private <T extends HttpUriRequest> T commonHeaders( T request )
490     {
491         request.setHeader( HttpHeaders.CACHE_CONTROL, "no-cache, no-store" );
492         request.setHeader( HttpHeaders.PRAGMA, "no-cache" );
493 
494         if ( state.isExpectContinue() && isPayloadPresent( request ) )
495         {
496             request.setHeader( HttpHeaders.EXPECT, "100-continue" );
497         }
498 
499         for ( Map.Entry<?, ?> entry : headers.entrySet() )
500         {
501             if ( !( entry.getKey() instanceof String ) )
502             {
503                 continue;
504             }
505             if ( entry.getValue() instanceof String )
506             {
507                 request.setHeader( entry.getKey().toString(), entry.getValue().toString() );
508             }
509             else
510             {
511                 request.removeHeaders( entry.getKey().toString() );
512             }
513         }
514 
515         if ( !state.isExpectContinue() )
516         {
517             request.removeHeaders( HttpHeaders.EXPECT );
518         }
519 
520         return request;
521     }
522 
523     @SuppressWarnings( "checkstyle:magicnumber" )
524     private <T extends HttpUriRequest> T resume( T request, GetTask task )
525     {
526         long resumeOffset = task.getResumeOffset();
527         if ( resumeOffset > 0L && task.getDataFile() != null )
528         {
529             request.setHeader( HttpHeaders.RANGE, "bytes=" + resumeOffset + '-' );
530             request.setHeader( HttpHeaders.IF_UNMODIFIED_SINCE,
531                                DateUtils.formatDate( new Date( task.getDataFile().lastModified() - 60L * 1000L ) ) );
532             request.setHeader( HttpHeaders.ACCEPT_ENCODING, "identity" );
533         }
534         return request;
535     }
536 
537     @SuppressWarnings( "checkstyle:magicnumber" )
538     private void handleStatus( CloseableHttpResponse response )
539         throws HttpResponseException
540     {
541         int status = response.getStatusLine().getStatusCode();
542         if ( status >= 300 )
543         {
544             throw new HttpResponseException( status, response.getStatusLine().getReasonPhrase() + " (" + status + ")" );
545         }
546     }
547 
548     @Override
549     protected void implClose()
550     {
551         try
552         {
553             client.close();
554         }
555         catch ( IOException e )
556         {
557             throw new UncheckedIOException( e );
558         }
559         AuthenticationContext.close( repoAuthContext );
560         AuthenticationContext.close( proxyAuthContext );
561         state.close();
562     }
563 
564     private class EntityGetter
565     {
566 
567         private final GetTask task;
568 
569         EntityGetter( GetTask task )
570         {
571             this.task = task;
572         }
573 
574         public void handle( CloseableHttpResponse response )
575             throws IOException, TransferCancelledException
576         {
577             HttpEntity entity = response.getEntity();
578             if ( entity == null )
579             {
580                 entity = new ByteArrayEntity( new byte[0] );
581             }
582 
583             long offset = 0L, length = entity.getContentLength();
584             Header rangeHeader = response.getFirstHeader( HttpHeaders.CONTENT_RANGE );
585             String range = rangeHeader != null ? rangeHeader.getValue() : null;
586             if ( range != null )
587             {
588                 Matcher m = CONTENT_RANGE_PATTERN.matcher( range );
589                 if ( !m.matches() )
590                 {
591                     throw new IOException( "Invalid Content-Range header for partial download: " + range );
592                 }
593                 offset = Long.parseLong( m.group( 1 ) );
594                 length = Long.parseLong( m.group( 2 ) ) + 1L;
595                 if ( offset < 0L || offset >= length || ( offset > 0L && offset != task.getResumeOffset() ) )
596                 {
597                     throw new IOException( "Invalid Content-Range header for partial download from offset "
598                             + task.getResumeOffset() + ": " + range );
599                 }
600             }
601 
602             final boolean resume = offset > 0L;
603             final File dataFile = task.getDataFile();
604             if ( dataFile == null )
605             {
606                 try ( InputStream is = entity.getContent() )
607                 {
608                     utilGet( task, is, true, length, resume );
609                     extractChecksums( response );
610                 }
611             }
612             else
613             {
614                 try ( FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile( dataFile.toPath() ) )
615                 {
616                     task.setDataFile( tempFile.getPath().toFile(), resume );
617                     if ( resume && Files.isRegularFile( dataFile.toPath() ) )
618                     {
619                         try ( InputStream inputStream = Files.newInputStream( dataFile.toPath() ) )
620                         {
621                             Files.copy( inputStream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING );
622                         }
623                     }
624                     try ( InputStream is = entity.getContent() )
625                     {
626                         utilGet( task, is, true, length, resume );
627                     }
628                     tempFile.move();
629                 }
630                 finally
631                 {
632                     task.setDataFile( dataFile );
633                 }
634             }
635             extractChecksums( response );
636         }
637 
638         private void extractChecksums( CloseableHttpResponse response )
639         {
640             for ( Map.Entry<String, ChecksumExtractor> extractorEntry : checksumExtractors.entrySet() )
641             {
642                 Map<String, String> checksums = extractorEntry.getValue().extractChecksums( response );
643                 if ( checksums != null )
644                 {
645                     checksums.forEach( task::setChecksum );
646                     return;
647                 }
648             }
649         }
650     }
651 
652     private class PutTaskEntity
653         extends AbstractHttpEntity
654     {
655 
656         private final PutTask task;
657 
658         PutTaskEntity( PutTask task )
659         {
660             this.task = task;
661         }
662 
663         @Override
664         public boolean isRepeatable()
665         {
666             return true;
667         }
668 
669         @Override
670         public boolean isStreaming()
671         {
672             return false;
673         }
674 
675         @Override
676         public long getContentLength()
677         {
678             return task.getDataLength();
679         }
680 
681         @Override
682         public InputStream getContent()
683             throws IOException
684         {
685             return task.newInputStream();
686         }
687 
688         @Override
689         public void writeTo( OutputStream os )
690             throws IOException
691         {
692             try
693             {
694                 utilPut( task, os, false );
695             }
696             catch ( TransferCancelledException e )
697             {
698                 throw (IOException) new InterruptedIOException().initCause( e );
699             }
700         }
701 
702     }
703 
704 }