001package org.apache.maven.wagon.shared.http;
002
003/*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements.  See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership.  The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License.  You may obtain a copy of the License at
011 *
012 *   http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied.  See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022import org.apache.commons.httpclient.Credentials;
023import org.apache.commons.httpclient.Header;
024import org.apache.commons.httpclient.HostConfiguration;
025import org.apache.commons.httpclient.HttpClient;
026import org.apache.commons.httpclient.HttpConnectionManager;
027import org.apache.commons.httpclient.HttpMethod;
028import org.apache.commons.httpclient.HttpStatus;
029import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
030import org.apache.commons.httpclient.NTCredentials;
031import org.apache.commons.httpclient.UsernamePasswordCredentials;
032import org.apache.commons.httpclient.auth.AuthScope;
033import org.apache.commons.httpclient.cookie.CookiePolicy;
034import org.apache.commons.httpclient.methods.EntityEnclosingMethod;
035import org.apache.commons.httpclient.methods.GetMethod;
036import org.apache.commons.httpclient.methods.HeadMethod;
037import org.apache.commons.httpclient.methods.PutMethod;
038import org.apache.commons.httpclient.methods.RequestEntity;
039import org.apache.commons.httpclient.params.HttpMethodParams;
040import org.apache.commons.httpclient.util.DateParseException;
041import org.apache.commons.httpclient.util.DateUtil;
042import org.apache.commons.io.IOUtils;
043import org.apache.commons.lang.StringUtils;
044import org.apache.maven.wagon.InputData;
045import org.apache.maven.wagon.OutputData;
046import org.apache.maven.wagon.PathUtils;
047import org.apache.maven.wagon.ResourceDoesNotExistException;
048import org.apache.maven.wagon.StreamWagon;
049import org.apache.maven.wagon.TransferFailedException;
050import org.apache.maven.wagon.Wagon;
051import org.apache.maven.wagon.authorization.AuthorizationException;
052import org.apache.maven.wagon.events.TransferEvent;
053import org.apache.maven.wagon.proxy.ProxyInfo;
054import org.apache.maven.wagon.repository.Repository;
055import org.apache.maven.wagon.resource.Resource;
056
057import java.io.ByteArrayInputStream;
058import java.io.File;
059import java.io.FileInputStream;
060import java.io.IOException;
061import java.io.InputStream;
062import java.io.OutputStream;
063import java.net.URLEncoder;
064import java.nio.ByteBuffer;
065import java.text.SimpleDateFormat;
066import java.util.Date;
067import java.util.Iterator;
068import java.util.Locale;
069import java.util.Properties;
070import java.util.TimeZone;
071import java.util.zip.GZIPInputStream;
072
073/**
074 * @author <a href="michal.maczka@dimatics.com">Michal Maczka</a>
075 * @author <a href="mailto:james@atlassian.com">James William Dumay</a>
076 */
077public abstract class AbstractHttpClientWagon
078    extends StreamWagon
079{
080    private final class RequestEntityImplementation
081        implements RequestEntity
082    {
083        private final Resource resource;
084
085        private final Wagon wagon;
086
087        private File source;
088
089        private ByteBuffer byteBuffer;
090
091        private RequestEntityImplementation( final InputStream stream, final Resource resource, final Wagon wagon,
092                                             final File source )
093            throws TransferFailedException
094        {
095            if ( source != null )
096            {
097                this.source = source;
098            }
099            else
100            {
101                try
102                {
103                    byte[] bytes = IOUtils.toByteArray( stream );
104                    this.byteBuffer = ByteBuffer.allocate( bytes.length );
105                    this.byteBuffer.put( bytes );
106                }
107                catch ( IOException e )
108                {
109                    throw new TransferFailedException( e.getMessage(), e );
110                }
111            }
112
113            this.resource = resource;
114            this.wagon = wagon;
115        }
116
117        public long getContentLength()
118        {
119            return resource.getContentLength();
120        }
121
122        public String getContentType()
123        {
124            return null;
125        }
126
127        public boolean isRepeatable()
128        {
129            return true;
130        }
131
132        public void writeRequest( OutputStream output )
133            throws IOException
134        {
135            byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
136
137            TransferEvent transferEvent =
138                new TransferEvent( wagon, resource, TransferEvent.TRANSFER_PROGRESS, TransferEvent.REQUEST_PUT );
139            transferEvent.setTimestamp( System.currentTimeMillis() );
140
141            InputStream fin = null;
142            try
143            {
144                fin = this.source != null
145                    ? new FileInputStream( source )
146                    : new ByteArrayInputStream( this.byteBuffer.array() );
147                int remaining = Integer.MAX_VALUE;
148                while ( remaining > 0 )
149                {
150                    int n = fin.read( buffer, 0, Math.min( buffer.length, remaining ) );
151
152                    if ( n == -1 )
153                    {
154                        break;
155                    }
156
157                    fireTransferProgress( transferEvent, buffer, n );
158
159                    output.write( buffer, 0, n );
160
161                    remaining -= n;
162                }
163            }
164            finally
165            {
166                IOUtils.closeQuietly( fin );
167            }
168
169            output.flush();
170        }
171    }
172
173    protected static final int SC_NULL = -1;
174
175    protected static final TimeZone GMT_TIME_ZONE = TimeZone.getTimeZone( "GMT" );
176
177    private HttpClient client;
178
179    protected HttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
180
181    /**
182     * @deprecated Use httpConfiguration instead.
183     */
184    private Properties httpHeaders;
185
186    /**
187     * @since 1.0-beta-6
188     */
189    private HttpConfiguration httpConfiguration;
190
191    private HttpMethod getMethod;
192
193    public void openConnectionInternal()
194    {
195        repository.setUrl( getURL( repository ) );
196        client = new HttpClient( connectionManager );
197
198        // WAGON-273: default the cookie-policy to browser compatible
199        client.getParams().setCookiePolicy( CookiePolicy.BROWSER_COMPATIBILITY );
200
201
202
203        String username = null;
204        String password = null;
205        String domain = null;
206
207        if ( authenticationInfo != null )
208        {
209            username = authenticationInfo.getUserName();
210
211            if ( StringUtils.contains( username, "\\" ) )
212            {
213                String[] domainAndUsername = username.split( "\\\\" );
214                domain = domainAndUsername[0];
215                username = domainAndUsername[1];
216            }
217
218            password = authenticationInfo.getPassword();
219
220
221        }
222
223        String host = getRepository().getHost();
224
225        if ( StringUtils.isNotEmpty( username ) && StringUtils.isNotEmpty( password ) )
226        {
227            Credentials creds;
228            if ( domain != null )
229            {
230                creds = new NTCredentials( username, password, host, domain );
231            }
232            else
233            {
234                creds = new UsernamePasswordCredentials( username, password );
235            }
236
237            int port = getRepository().getPort() > -1 ? getRepository().getPort() : AuthScope.ANY_PORT;
238
239            AuthScope scope = new AuthScope( host, port );
240            client.getState().setCredentials( scope, creds );
241        }
242
243        HostConfiguration hc = new HostConfiguration();
244
245        ProxyInfo proxyInfo = getProxyInfo( getRepository().getProtocol(), getRepository().getHost() );
246        if ( proxyInfo != null )
247        {
248            String proxyUsername = proxyInfo.getUserName();
249            String proxyPassword = proxyInfo.getPassword();
250            String proxyHost = proxyInfo.getHost();
251            int proxyPort = proxyInfo.getPort();
252            String proxyNtlmHost = proxyInfo.getNtlmHost();
253            String proxyNtlmDomain = proxyInfo.getNtlmDomain();
254            if ( proxyHost != null )
255            {
256                hc.setProxy( proxyHost, proxyPort );
257
258                if ( proxyUsername != null && proxyPassword != null )
259                {
260                    Credentials creds;
261                    if ( proxyNtlmHost != null || proxyNtlmDomain != null )
262                    {
263                        creds = new NTCredentials( proxyUsername, proxyPassword, proxyNtlmHost, proxyNtlmDomain );
264                    }
265                    else
266                    {
267                        creds = new UsernamePasswordCredentials( proxyUsername, proxyPassword );
268                    }
269
270                    int port = proxyInfo.getPort() > -1 ? proxyInfo.getPort() : AuthScope.ANY_PORT;
271
272                    AuthScope scope = new AuthScope( proxyHost, port );
273                    client.getState().setProxyCredentials( scope, creds );
274                }
275            }
276        }
277
278        hc.setHost( host );
279
280        //start a session with the webserver
281        client.setHostConfiguration( hc );
282    }
283
284    public void closeConnection()
285    {
286        if ( connectionManager instanceof MultiThreadedHttpConnectionManager )
287        {
288            ( (MultiThreadedHttpConnectionManager) connectionManager ).shutdown();
289        }
290    }
291
292    public void put( File source, String resourceName )
293        throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
294    {
295        Resource resource = new Resource( resourceName );
296
297        firePutInitiated( resource, source );
298
299        resource.setContentLength( source.length() );
300
301        resource.setLastModified( source.lastModified() );
302
303        put( null, resource, source );
304    }
305
306    public void putFromStream( final InputStream stream, String destination, long contentLength, long lastModified )
307        throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
308    {
309        Resource resource = new Resource( destination );
310
311        firePutInitiated( resource, null );
312
313        resource.setContentLength( contentLength );
314
315        resource.setLastModified( lastModified );
316
317        put( stream, resource, null );
318    }
319
320    private void put( final InputStream stream, Resource resource, File source )
321        throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException
322    {
323        StringBuilder url = new StringBuilder( getRepository().getUrl() );
324        String[] parts = StringUtils.split( resource.getName(), "/" );
325        for ( String part : parts )
326        {
327            // TODO: Fix encoding...
328            if ( !url.toString().endsWith( "/" ) )
329            {
330                url.append( '/' );
331            }
332            url.append( URLEncoder.encode( part ) );
333        }
334        RequestEntityImplementation requestEntityImplementation =
335            new RequestEntityImplementation( stream, resource, this, source );
336        put( resource, source, requestEntityImplementation, url.toString() );
337
338    }
339
340    private void put( Resource resource, File source, RequestEntityImplementation requestEntityImplementation,
341                      String url )
342        throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException
343    {
344
345        // preemptive true for put
346        client.getParams().setAuthenticationPreemptive( true );
347
348        //Parent directories need to be created before posting
349        try
350        {
351            mkdirs( PathUtils.dirname( resource.getName() ) );
352        }
353        catch ( IOException e )
354        {
355            fireTransferError( resource, e, TransferEvent.REQUEST_GET );
356        }
357
358        PutMethod putMethod = new PutMethod( url );
359
360        firePutStarted( resource, source );
361
362        try
363        {
364            putMethod.setRequestEntity( requestEntityImplementation );
365
366            int statusCode;
367            try
368            {
369                statusCode = execute( putMethod );
370
371            }
372            catch ( IOException e )
373            {
374                fireTransferError( resource, e, TransferEvent.REQUEST_PUT );
375
376                throw new TransferFailedException( e.getMessage(), e );
377            }
378
379            fireTransferDebug( url + " - Status code: " + statusCode );
380
381            // Check that we didn't run out of retries.
382            switch ( statusCode )
383            {
384                // Success Codes
385                case HttpStatus.SC_OK: // 200
386                case HttpStatus.SC_CREATED: // 201
387                case HttpStatus.SC_ACCEPTED: // 202
388                case HttpStatus.SC_NO_CONTENT:  // 204
389                    break;
390
391                // handle all redirect even if http specs says " the user agent MUST NOT automatically redirect the request unless it can be confirmed by the user"
392                case HttpStatus.SC_MOVED_PERMANENTLY: // 301
393                case HttpStatus.SC_MOVED_TEMPORARILY: // 302
394                case HttpStatus.SC_SEE_OTHER: // 303
395                    String relocatedUrl = calculateRelocatedUrl( putMethod );
396                    fireTransferDebug( "relocate to " + relocatedUrl );
397                    put( resource, source, requestEntityImplementation, relocatedUrl );
398                    return;
399
400                case SC_NULL:
401                {
402                    TransferFailedException e = new TransferFailedException( "Failed to transfer file: " + url );
403                    fireTransferError( resource, e, TransferEvent.REQUEST_PUT );
404                    throw e;
405                }
406
407                case HttpStatus.SC_FORBIDDEN:
408                    fireSessionConnectionRefused();
409                    throw new AuthorizationException( "Access denied to: " + url );
410
411                case HttpStatus.SC_NOT_FOUND:
412                    throw new ResourceDoesNotExistException( "File: " + url + " does not exist" );
413
414                    //add more entries here
415                default:
416                {
417                    TransferFailedException e = new TransferFailedException(
418                        "Failed to transfer file: " + url + ". Return code is: " + statusCode );
419                    fireTransferError( resource, e, TransferEvent.REQUEST_PUT );
420                    throw e;
421                }
422            }
423
424            firePutCompleted( resource, source );
425        }
426        finally
427        {
428            putMethod.releaseConnection();
429        }
430    }
431
432    protected String calculateRelocatedUrl( EntityEnclosingMethod method )
433    {
434        Header locationHeader = method.getResponseHeader( "Location" );
435        String locationField = locationHeader.getValue();
436        // is it a relative Location or a full ?
437        return locationField.startsWith( "http" ) ? locationField : getURL( getRepository() ) + '/' + locationField;
438    }
439
440    protected void mkdirs( String dirname )
441        throws IOException
442    {
443        // do nothing as default.
444    }
445
446    public boolean resourceExists( String resourceName )
447        throws TransferFailedException, AuthorizationException
448    {
449        StringBuilder url = new StringBuilder( getRepository().getUrl() );
450        if ( !url.toString().endsWith( "/" ) )
451        {
452            url.append( '/' );
453        }
454        url.append( resourceName );
455        HeadMethod headMethod = new HeadMethod( url.toString() );
456
457        int statusCode;
458        try
459        {
460            statusCode = execute( headMethod );
461        }
462        catch ( IOException e )
463        {
464            throw new TransferFailedException( e.getMessage(), e );
465        }
466        try
467        {
468            switch ( statusCode )
469            {
470                case HttpStatus.SC_OK:
471                    return true;
472
473                case HttpStatus.SC_NOT_MODIFIED:
474                    return true;
475
476                case SC_NULL:
477                    throw new TransferFailedException( "Failed to transfer file: " + url );
478
479                case HttpStatus.SC_FORBIDDEN:
480                    throw new AuthorizationException( "Access denied to: " + url );
481
482                case HttpStatus.SC_UNAUTHORIZED:
483                    throw new AuthorizationException( "Not authorized." );
484
485                case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
486                    throw new AuthorizationException( "Not authorized by proxy." );
487
488                case HttpStatus.SC_NOT_FOUND:
489                    return false;
490
491                //add more entries here
492                default:
493                    throw new TransferFailedException(
494                        "Failed to transfer file: " + url + ". Return code is: " + statusCode );
495            }
496        }
497        finally
498        {
499            headMethod.releaseConnection();
500        }
501    }
502
503    protected int execute( HttpMethod httpMethod )
504        throws IOException
505    {
506        int statusCode;
507
508        setParameters( httpMethod );
509        setHeaders( httpMethod );
510
511        statusCode = client.executeMethod( httpMethod );
512        return statusCode;
513    }
514
515    protected void setParameters( HttpMethod method )
516    {
517        HttpMethodConfiguration config =
518            httpConfiguration == null ? null : httpConfiguration.getMethodConfiguration( method );
519        if ( config != null )
520        {
521            HttpMethodParams params = config.asMethodParams( method.getParams() );
522            if ( params != null )
523            {
524                method.setParams( params );
525            }
526        }
527
528        if ( config == null || config.getConnectionTimeout() == HttpMethodConfiguration.DEFAULT_CONNECTION_TIMEOUT )
529        {
530            method.getParams().setSoTimeout( getTimeout() );
531        }
532    }
533
534    protected void setHeaders( HttpMethod method )
535    {
536        HttpMethodConfiguration config =
537            httpConfiguration == null ? null : httpConfiguration.getMethodConfiguration( method );
538        if ( config == null || config.isUseDefaultHeaders() )
539        {
540            // TODO: merge with the other headers and have some better defaults, unify with lightweight headers
541            method.addRequestHeader( "Cache-control", "no-cache" );
542            method.addRequestHeader( "Cache-store", "no-store" );
543            method.addRequestHeader( "Pragma", "no-cache" );
544            method.addRequestHeader( "Expires", "0" );
545            method.addRequestHeader( "Accept-Encoding", "gzip" );
546        }
547
548        if ( httpHeaders != null )
549        {
550            for ( Iterator i = httpHeaders.keySet().iterator(); i.hasNext(); )
551            {
552                String header = (String) i.next();
553                method.addRequestHeader( header, httpHeaders.getProperty( header ) );
554            }
555        }
556
557        Header[] headers = config == null ? null : config.asRequestHeaders();
558        if ( headers != null )
559        {
560            for ( int i = 0; i < headers.length; i++ )
561            {
562                method.addRequestHeader( headers[i] );
563            }
564        }
565    }
566
567    /**
568     * getUrl
569     * Implementors can override this to remove unwanted parts of the url such as role-hints
570     *
571     * @param repository
572     * @return
573     */
574    protected String getURL( Repository repository )
575    {
576        return repository.getUrl();
577    }
578
579    protected HttpClient getClient()
580    {
581        return client;
582    }
583
584    public void setConnectionManager( HttpConnectionManager connectionManager )
585    {
586        this.connectionManager = connectionManager;
587    }
588
589    public Properties getHttpHeaders()
590    {
591        return httpHeaders;
592    }
593
594    public void setHttpHeaders( Properties httpHeaders )
595    {
596        this.httpHeaders = httpHeaders;
597    }
598
599    public HttpConfiguration getHttpConfiguration()
600    {
601        return httpConfiguration;
602    }
603
604    public void setHttpConfiguration( HttpConfiguration httpConfiguration )
605    {
606        this.httpConfiguration = httpConfiguration;
607    }
608
609    public void fillInputData( InputData inputData )
610        throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
611    {
612        Resource resource = inputData.getResource();
613
614        StringBuilder url = new StringBuilder( getRepository().getUrl() );
615        if ( !url.toString().endsWith( "/" ) )
616        {
617            url.append( '/' );
618        }
619        url.append( resource.getName() );
620
621        getMethod = new GetMethod( url.toString() );
622
623        long timestamp = resource.getLastModified();
624        if ( timestamp > 0 )
625        {
626            SimpleDateFormat fmt = new SimpleDateFormat( "EEE, dd-MMM-yy HH:mm:ss zzz", Locale.US );
627            fmt.setTimeZone( GMT_TIME_ZONE );
628            Header hdr = new Header( "If-Modified-Since", fmt.format( new Date( timestamp ) ) );
629            fireTransferDebug( "sending ==> " + hdr + "(" + timestamp + ")" );
630            getMethod.addRequestHeader( hdr );
631        }
632
633        int statusCode;
634        try
635        {
636            statusCode = execute( getMethod );
637        }
638        catch ( IOException e )
639        {
640            fireTransferError( resource, e, TransferEvent.REQUEST_GET );
641
642            throw new TransferFailedException( e.getMessage(), e );
643        }
644
645        fireTransferDebug( url + " - Status code: " + statusCode );
646
647        // TODO [BP]: according to httpclient docs, really should swallow the output on error. verify if that is
648        // required
649        switch ( statusCode )
650        {
651            case HttpStatus.SC_OK:
652                break;
653
654            case HttpStatus.SC_NOT_MODIFIED:
655                // return, leaving last modified set to original value so getIfNewer should return unmodified
656                return;
657
658            case SC_NULL:
659            {
660                TransferFailedException e = new TransferFailedException( "Failed to transfer file: " + url );
661                fireTransferError( resource, e, TransferEvent.REQUEST_GET );
662                throw e;
663            }
664
665            case HttpStatus.SC_FORBIDDEN:
666                fireSessionConnectionRefused();
667                throw new AuthorizationException( "Access denied to: " + url );
668
669            case HttpStatus.SC_UNAUTHORIZED:
670                fireSessionConnectionRefused();
671                throw new AuthorizationException( "Not authorized." );
672
673            case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
674                fireSessionConnectionRefused();
675                throw new AuthorizationException( "Not authorized by proxy." );
676
677            case HttpStatus.SC_NOT_FOUND:
678                throw new ResourceDoesNotExistException( "File: " + url + " does not exist" );
679
680                // add more entries here
681            default:
682            {
683                cleanupGetTransfer( resource );
684                TransferFailedException e = new TransferFailedException(
685                    "Failed to transfer file: " + url + ". Return code is: " + statusCode );
686                fireTransferError( resource, e, TransferEvent.REQUEST_GET );
687                throw e;
688            }
689        }
690
691        InputStream is = null;
692
693        Header contentLengthHeader = getMethod.getResponseHeader( "Content-Length" );
694
695        if ( contentLengthHeader != null )
696        {
697            try
698            {
699                long contentLength = Integer.valueOf( contentLengthHeader.getValue() ).intValue();
700
701                resource.setContentLength( contentLength );
702            }
703            catch ( NumberFormatException e )
704            {
705                fireTransferDebug(
706                    "error parsing content length header '" + contentLengthHeader.getValue() + "' " + e );
707            }
708        }
709
710        Header lastModifiedHeader = getMethod.getResponseHeader( "Last-Modified" );
711
712        long lastModified = 0;
713
714        if ( lastModifiedHeader != null )
715        {
716            try
717            {
718                lastModified = DateUtil.parseDate( lastModifiedHeader.getValue() ).getTime();
719
720                resource.setLastModified( lastModified );
721            }
722            catch ( DateParseException e )
723            {
724                fireTransferDebug( "Unable to parse last modified header" );
725            }
726
727            fireTransferDebug( "last-modified = " + lastModifiedHeader.getValue() + " (" + lastModified + ")" );
728        }
729
730        Header contentEncoding = getMethod.getResponseHeader( "Content-Encoding" );
731        boolean isGZipped = contentEncoding != null && "gzip".equalsIgnoreCase( contentEncoding.getValue() );
732
733        try
734        {
735            is = getMethod.getResponseBodyAsStream();
736            if ( isGZipped )
737            {
738                is = new GZIPInputStream( is );
739            }
740        }
741        catch ( IOException e )
742        {
743            fireTransferError( resource, e, TransferEvent.REQUEST_GET );
744
745            String msg =
746                "Error occurred while retrieving from remote repository:" + getRepository() + ": " + e.getMessage();
747
748            throw new TransferFailedException( msg, e );
749        }
750
751        inputData.setInputStream( is );
752    }
753
754    protected void cleanupGetTransfer( Resource resource )
755    {
756        if ( getMethod != null )
757        {
758            getMethod.releaseConnection();
759        }
760    }
761
762    @Override
763    public void putFromStream( InputStream stream, String destination )
764        throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException
765    {
766        putFromStream( stream, destination, -1, -1 );
767    }
768
769    @Override
770    protected void putFromStream( InputStream stream, Resource resource )
771        throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException
772    {
773        putFromStream( stream, resource.getName(), -1, -1 );
774    }
775
776    @Override
777    public void fillOutputData( OutputData outputData )
778        throws TransferFailedException
779    {
780        // no needed in this implementation but throw an Exception if used
781        throw new IllegalStateException( "this wagon http client must not use fillOutputData" );
782    }
783}