001package org.apache.maven.wagon.shared.http; 002 003/* 004 * Licensed to the Apache Software Foundation (ASF) under one 005 * or more contributor license agreements. See the NOTICE file 006 * distributed with this work for additional information 007 * regarding copyright ownership. The ASF licenses this file 008 * to you under the Apache License, Version 2.0 (the 009 * "License"); you may not use this file except in compliance 010 * with the License. You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, 015 * software distributed under the License is distributed on an 016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 017 * KIND, either express or implied. See the License for the 018 * specific language governing permissions and limitations 019 * under the License. 020 */ 021 022import org.apache.http.Header; 023import org.apache.http.HttpEntity; 024import org.apache.http.HttpException; 025import org.apache.http.HttpHost; 026import org.apache.http.HttpResponse; 027import org.apache.http.HttpStatus; 028import org.apache.http.auth.AuthSchemeProvider; 029import org.apache.http.auth.AuthScope; 030import org.apache.http.auth.ChallengeState; 031import org.apache.http.auth.Credentials; 032import org.apache.http.auth.NTCredentials; 033import org.apache.http.auth.UsernamePasswordCredentials; 034import org.apache.http.client.AuthCache; 035import org.apache.http.client.CredentialsProvider; 036import org.apache.http.client.HttpRequestRetryHandler; 037import org.apache.http.client.config.AuthSchemes; 038import org.apache.http.client.config.CookieSpecs; 039import org.apache.http.client.config.RequestConfig; 040import org.apache.http.client.methods.CloseableHttpResponse; 041import org.apache.http.client.methods.HttpGet; 042import org.apache.http.client.methods.HttpHead; 043import org.apache.http.client.methods.HttpPut; 044import org.apache.http.client.methods.HttpUriRequest; 045import org.apache.http.client.protocol.HttpClientContext; 046import org.apache.http.client.utils.DateUtils; 047import org.apache.http.config.Registry; 048import org.apache.http.config.RegistryBuilder; 049import org.apache.http.conn.HttpClientConnectionManager; 050import org.apache.http.conn.socket.ConnectionSocketFactory; 051import org.apache.http.conn.socket.PlainConnectionSocketFactory; 052import org.apache.http.conn.ssl.SSLConnectionSocketFactory; 053import org.apache.http.conn.ssl.SSLContextBuilder; 054import org.apache.http.conn.ssl.SSLInitializationException; 055import org.apache.http.entity.AbstractHttpEntity; 056import org.apache.http.impl.auth.BasicScheme; 057import org.apache.http.impl.auth.BasicSchemeFactory; 058import org.apache.http.impl.auth.DigestSchemeFactory; 059import org.apache.http.impl.auth.NTLMSchemeFactory; 060import org.apache.http.impl.client.BasicAuthCache; 061import org.apache.http.impl.client.BasicCredentialsProvider; 062import org.apache.http.impl.client.CloseableHttpClient; 063import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; 064import org.apache.http.impl.client.HttpClientBuilder; 065import org.apache.http.impl.client.StandardHttpRequestRetryHandler; 066import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; 067import org.apache.http.message.BasicHeader; 068import org.apache.http.protocol.HTTP; 069import org.apache.http.util.EntityUtils; 070import org.apache.maven.wagon.InputData; 071import org.apache.maven.wagon.OutputData; 072import org.apache.maven.wagon.PathUtils; 073import org.apache.maven.wagon.ResourceDoesNotExistException; 074import org.apache.maven.wagon.StreamWagon; 075import org.apache.maven.wagon.TransferFailedException; 076import org.apache.maven.wagon.Wagon; 077import org.apache.maven.wagon.authorization.AuthorizationException; 078import org.apache.maven.wagon.events.TransferEvent; 079import org.apache.maven.wagon.proxy.ProxyInfo; 080import org.apache.maven.wagon.repository.Repository; 081import org.apache.maven.wagon.resource.Resource; 082import org.codehaus.plexus.util.StringUtils; 083 084import javax.net.ssl.HttpsURLConnection; 085import javax.net.ssl.SSLContext; 086import java.io.Closeable; 087import java.io.File; 088import java.io.FileInputStream; 089import java.io.IOException; 090import java.io.InputStream; 091import java.io.OutputStream; 092import java.io.RandomAccessFile; 093import java.nio.ByteBuffer; 094import java.nio.channels.Channels; 095import java.nio.channels.ReadableByteChannel; 096import java.nio.charset.StandardCharsets; 097import java.text.SimpleDateFormat; 098import java.util.ArrayList; 099import java.util.Collection; 100import java.util.Date; 101import java.util.List; 102import java.util.Locale; 103import java.util.Map; 104import java.util.Properties; 105import java.util.TimeZone; 106import java.util.concurrent.TimeUnit; 107 108/** 109 * @author <a href="michal.maczka@dimatics.com">Michal Maczka</a> 110 * @author <a href="mailto:james@atlassian.com">James William Dumay</a> 111 */ 112public abstract class AbstractHttpClientWagon 113 extends StreamWagon 114{ 115 private final class RequestEntityImplementation 116 extends AbstractHttpEntity 117 { 118 private final Resource resource; 119 120 private final Wagon wagon; 121 122 private InputStream stream; 123 124 private File source; 125 126 private long length = -1; 127 128 private boolean repeatable; 129 130 private RequestEntityImplementation( final InputStream stream, final Resource resource, final Wagon wagon, 131 final File source ) 132 throws TransferFailedException 133 { 134 if ( source != null ) 135 { 136 this.source = source; 137 this.repeatable = true; 138 } 139 else 140 { 141 this.stream = stream; 142 this.repeatable = false; 143 } 144 this.resource = resource; 145 this.length = resource == null ? -1 : resource.getContentLength(); 146 147 this.wagon = wagon; 148 } 149 150 public long getContentLength() 151 { 152 return length; 153 } 154 155 public InputStream getContent() 156 throws IOException, IllegalStateException 157 { 158 if ( this.source != null ) 159 { 160 return new FileInputStream( this.source ); 161 } 162 return stream; 163 } 164 165 public boolean isRepeatable() 166 { 167 return repeatable; 168 } 169 170 public void writeTo( final OutputStream output ) 171 throws IOException 172 { 173 if ( output == null ) 174 { 175 throw new NullPointerException( "output cannot be null" ); 176 } 177 TransferEvent transferEvent = 178 new TransferEvent( wagon, resource, TransferEvent.TRANSFER_PROGRESS, TransferEvent.REQUEST_PUT ); 179 transferEvent.setTimestamp( System.currentTimeMillis() ); 180 181 try ( ReadableByteChannel input = ( this.source != null ) 182 ? new RandomAccessFile( this.source, "r" ).getChannel() 183 : Channels.newChannel( stream ) ) 184 { 185 ByteBuffer buffer = ByteBuffer.allocate( getBufferCapacityForTransfer( this.length ) ); 186 int halfBufferCapacity = buffer.capacity() / 2; 187 188 long remaining = this.length < 0L ? Long.MAX_VALUE : this.length; 189 while ( remaining > 0L ) 190 { 191 int read = input.read( buffer ); 192 if ( read == -1 ) 193 { 194 // EOF, but some data has not been written yet. 195 if ( buffer.position() != 0 ) 196 { 197 buffer.flip(); 198 fireTransferProgress( transferEvent, buffer.array(), buffer.limit() ); 199 output.write( buffer.array(), 0, buffer.limit() ); 200 buffer.clear(); 201 } 202 203 break; 204 } 205 206 // Prevent minichunking / fragmentation: when less than half the buffer is utilized, 207 // read some more bytes before writing and firing progress. 208 if ( buffer.position() < halfBufferCapacity ) 209 { 210 continue; 211 } 212 213 buffer.flip(); 214 fireTransferProgress( transferEvent, buffer.array(), buffer.limit() ); 215 output.write( buffer.array(), 0, buffer.limit() ); 216 remaining -= buffer.limit(); 217 buffer.clear(); 218 219 } 220 output.flush(); 221 } 222 } 223 224 public boolean isStreaming() 225 { 226 return true; 227 } 228 } 229 230 private static final TimeZone GMT_TIME_ZONE = TimeZone.getTimeZone( "GMT" ); 231 232 /** 233 * use http(s) connection pool mechanism. 234 * <b>enabled by default</b> 235 */ 236 private static boolean persistentPool = 237 Boolean.valueOf( System.getProperty( "maven.wagon.http.pool", "true" ) ); 238 239 /** 240 * skip failure on certificate validity checks. 241 * <b>disabled by default</b> 242 */ 243 private static final boolean SSL_INSECURE = 244 Boolean.valueOf( System.getProperty( "maven.wagon.http.ssl.insecure", "false" ) ); 245 246 /** 247 * if using sslInsecure, certificate date issues will be ignored 248 * <b>disabled by default</b> 249 */ 250 private static final boolean IGNORE_SSL_VALIDITY_DATES = 251 Boolean.valueOf( System.getProperty( "maven.wagon.http.ssl.ignore.validity.dates", "false" ) ); 252 253 /** 254 * If enabled, ssl hostname verifier does not check hostname. Disable this will use a browser compat hostname 255 * verifier <b>disabled by default</b> 256 */ 257 private static final boolean SSL_ALLOW_ALL = 258 Boolean.valueOf( System.getProperty( "maven.wagon.http.ssl.allowall", "false" ) ); 259 260 261 /** 262 * Maximum concurrent connections per distinct route. 263 * <b>20 by default</b> 264 */ 265 private static final int MAX_CONN_PER_ROUTE = 266 Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.maxPerRoute", "20" ) ); 267 268 /** 269 * Maximum concurrent connections in total. 270 * <b>40 by default</b> 271 */ 272 private static final int MAX_CONN_TOTAL = 273 Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.maxTotal", "40" ) ); 274 275 /** 276 * Time to live in seconds for an HTTP connection. After that time, the connection will be dropped. 277 * Intermediates tend to drop connections after some idle period. Set to -1 to maintain connections 278 * indefinitely. This value defaults to 300 seconds. 279 * 280 * @since 3.2 281 */ 282 private static final long CONN_TTL = 283 Long.getLong( "maven.wagon.httpconnectionManager.ttlSeconds", 300L ); 284 285 /** 286 * Internal connection manager 287 */ 288 private static HttpClientConnectionManager httpClientConnectionManager = createConnManager(); 289 290 291 /** 292 * See RFC6585 293 */ 294 protected static final int SC_TOO_MANY_REQUESTS = 429; 295 296 /** 297 * For exponential backoff. 298 */ 299 300 /** 301 * Initial seconds to back off when a HTTP 429 received. 302 * Subsequent 429 responses result in exponental backoff. 303 * <b>5 by default</b> 304 * 305 * @since 2.7 306 */ 307 private int initialBackoffSeconds = 308 Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.backoffSeconds", "5" ) ); 309 310 /** 311 * The maximum amount of time we want to back off in the case of 312 * repeated HTTP 429 response codes. 313 * 314 * @since 2.7 315 */ 316 private static final int MAX_BACKOFF_WAIT_SECONDS = 317 Integer.parseInt( System.getProperty( "maven.wagon.httpconnectionManager.maxBackoffSeconds", "180" ) ); 318 319 protected int backoff( int wait, String url ) 320 throws InterruptedException, TransferFailedException 321 { 322 TimeUnit.SECONDS.sleep( wait ); 323 int nextWait = wait * 2; 324 if ( nextWait >= getMaxBackoffWaitSeconds() ) 325 { 326 throw new TransferFailedException( 327 "Waited too long to access: " + url + ". Return code is: " + SC_TOO_MANY_REQUESTS ); 328 } 329 return nextWait; 330 } 331 332 @SuppressWarnings( "checkstyle:linelength" ) 333 private static PoolingHttpClientConnectionManager createConnManager() 334 { 335 336 String sslProtocolsStr = System.getProperty( "https.protocols" ); 337 String cipherSuitesStr = System.getProperty( "https.cipherSuites" ); 338 String[] sslProtocols = sslProtocolsStr != null ? sslProtocolsStr.split( " *, *" ) : null; 339 String[] cipherSuites = cipherSuitesStr != null ? cipherSuitesStr.split( " *, *" ) : null; 340 341 SSLConnectionSocketFactory sslConnectionSocketFactory; 342 if ( SSL_INSECURE ) 343 { 344 try 345 { 346 SSLContext sslContext = new SSLContextBuilder().useSSL().loadTrustMaterial( null, 347 new RelaxedTrustStrategy( 348 IGNORE_SSL_VALIDITY_DATES ) ).build(); 349 sslConnectionSocketFactory = new SSLConnectionSocketFactory( sslContext, sslProtocols, cipherSuites, 350 SSL_ALLOW_ALL 351 ? SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER 352 : SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER ); 353 } 354 catch ( Exception ex ) 355 { 356 throw new SSLInitializationException( ex.getMessage(), ex ); 357 } 358 } 359 else 360 { 361 sslConnectionSocketFactory = 362 new SSLConnectionSocketFactory( HttpsURLConnection.getDefaultSSLSocketFactory(), sslProtocols, 363 cipherSuites, 364 SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER ); 365 } 366 367 Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register( "http", 368 PlainConnectionSocketFactory.INSTANCE ).register( 369 "https", sslConnectionSocketFactory ).build(); 370 371 PoolingHttpClientConnectionManager connManager = 372 new PoolingHttpClientConnectionManager( registry, null, null, null, CONN_TTL, TimeUnit.SECONDS ); 373 if ( persistentPool ) 374 { 375 connManager.setDefaultMaxPerRoute( MAX_CONN_PER_ROUTE ); 376 connManager.setMaxTotal( MAX_CONN_TOTAL ); 377 } 378 else 379 { 380 connManager.setMaxTotal( 1 ); 381 } 382 return connManager; 383 } 384 385 /** 386 * The type of the retry handler, defaults to {@code standard}. 387 * Values can be {@link default DefaultHttpRequestRetryHandler}, 388 * or {@link standard StandardHttpRequestRetryHandler}, 389 * or a fully qualified name class with a no-arg. 390 * 391 * @since 3.2 392 */ 393 private static final String RETRY_HANDLER_CLASS = 394 System.getProperty( "maven.wagon.http.retryHandler.class", "standard" ); 395 396 /** 397 * Whether or not methods that have successfully sent their request will be retried, 398 * defaults to {@code false}. 399 * Note: only used for default and standard retry handlers. 400 * 401 * @since 3.2 402 */ 403 private static final boolean RETRY_HANDLER_REQUEST_SENT_ENABLED = 404 Boolean.getBoolean( "maven.wagon.http.retryHandler.requestSentEnabled" ); 405 406 /** 407 * Number of retries for the retry handler, defaults to 3. 408 * Note: only used for default and standard retry handlers. 409 * 410 * @since 3.2 411 */ 412 private static final int RETRY_HANDLER_COUNT = 413 Integer.getInteger( "maven.wagon.http.retryHandler.count", 3 ); 414 415 /** 416 * Comma-separated list of non-retryable exception classes. 417 * Note: only used for default retry handler. 418 * 419 * @since 3.2 420 */ 421 private static final String RETRY_HANDLER_EXCEPTIONS = 422 System.getProperty( "maven.wagon.http.retryHandler.nonRetryableClasses" ); 423 424 private static HttpRequestRetryHandler createRetryHandler() 425 { 426 switch ( RETRY_HANDLER_CLASS ) 427 { 428 case "default": 429 if ( StringUtils.isEmpty( RETRY_HANDLER_EXCEPTIONS ) ) 430 { 431 return new DefaultHttpRequestRetryHandler( 432 RETRY_HANDLER_COUNT, RETRY_HANDLER_REQUEST_SENT_ENABLED ); 433 } 434 return new DefaultHttpRequestRetryHandler( 435 RETRY_HANDLER_COUNT, RETRY_HANDLER_REQUEST_SENT_ENABLED, getNonRetryableExceptions() ) 436 { 437 }; 438 case "standard": 439 return new StandardHttpRequestRetryHandler( RETRY_HANDLER_COUNT, RETRY_HANDLER_REQUEST_SENT_ENABLED ); 440 default: 441 try 442 { 443 final ClassLoader classLoader = AbstractHttpClientWagon.class.getClassLoader(); 444 return HttpRequestRetryHandler.class.cast( classLoader.loadClass( RETRY_HANDLER_CLASS ) 445 .getConstructor().newInstance() ); 446 } 447 catch ( final Exception e ) 448 { 449 throw new IllegalArgumentException( e ); 450 } 451 } 452 } 453 454 private static Registry<AuthSchemeProvider> createAuthSchemeRegistry() 455 { 456 return RegistryBuilder.<AuthSchemeProvider>create() 457 .register( AuthSchemes.BASIC, new BasicSchemeFactory( StandardCharsets.UTF_8 ) ) 458 .register( AuthSchemes.DIGEST, new DigestSchemeFactory( StandardCharsets.UTF_8 ) ) 459 .register( AuthSchemes.NTLM, new NTLMSchemeFactory() ) 460 .build(); 461 } 462 463 private static Collection<Class<? extends IOException>> getNonRetryableExceptions() 464 { 465 final List<Class<? extends IOException>> exceptions = new ArrayList<>(); 466 final ClassLoader loader = AbstractHttpClientWagon.class.getClassLoader(); 467 for ( final String ex : RETRY_HANDLER_EXCEPTIONS.split( "," ) ) 468 { 469 try 470 { 471 exceptions.add( ( Class<? extends IOException> ) loader.loadClass( ex ) ); 472 } 473 catch ( final ClassNotFoundException e ) 474 { 475 throw new IllegalArgumentException( e ); 476 } 477 } 478 return exceptions; 479 } 480 481 private static CloseableHttpClient httpClient = createClient(); 482 483 private static CloseableHttpClient createClient() 484 { 485 return HttpClientBuilder.create() // 486 .useSystemProperties() // 487 .disableConnectionState() // 488 .setConnectionManager( httpClientConnectionManager ) // 489 .setRetryHandler( createRetryHandler() ) 490 .setDefaultAuthSchemeRegistry( createAuthSchemeRegistry() ) 491 .build(); 492 } 493 494 private CredentialsProvider credentialsProvider; 495 496 private AuthCache authCache; 497 498 private Closeable closeable; 499 500 /** 501 * @plexus.configuration 502 * @deprecated Use httpConfiguration instead. 503 */ 504 private Properties httpHeaders; 505 506 /** 507 * @since 1.0-beta-6 508 */ 509 private HttpConfiguration httpConfiguration; 510 511 /** 512 * Basic auth scope overrides 513 * @since 2.8 514 */ 515 private BasicAuthScope basicAuth; 516 517 /** 518 * Proxy basic auth scope overrides 519 * @since 2.8 520 */ 521 private BasicAuthScope proxyAuth; 522 523 public void openConnectionInternal() 524 { 525 repository.setUrl( getURL( repository ) ); 526 527 credentialsProvider = new BasicCredentialsProvider(); 528 authCache = new BasicAuthCache(); 529 530 if ( authenticationInfo != null ) 531 { 532 533 String username = authenticationInfo.getUserName(); 534 String password = authenticationInfo.getPassword(); 535 536 if ( StringUtils.isNotEmpty( username ) && StringUtils.isNotEmpty( password ) ) 537 { 538 Credentials creds = new UsernamePasswordCredentials( username, password ); 539 540 String host = getRepository().getHost(); 541 int port = getRepository().getPort(); 542 543 credentialsProvider.setCredentials( getBasicAuthScope().getScope( host, port ), creds ); 544 } 545 } 546 547 ProxyInfo proxyInfo = getProxyInfo( getRepository().getProtocol(), getRepository().getHost() ); 548 if ( proxyInfo != null ) 549 { 550 String proxyUsername = proxyInfo.getUserName(); 551 String proxyPassword = proxyInfo.getPassword(); 552 String proxyHost = proxyInfo.getHost(); 553 String proxyNtlmHost = proxyInfo.getNtlmHost(); 554 String proxyNtlmDomain = proxyInfo.getNtlmDomain(); 555 if ( proxyHost != null ) 556 { 557 if ( proxyUsername != null && proxyPassword != null ) 558 { 559 Credentials creds; 560 if ( proxyNtlmHost != null || proxyNtlmDomain != null ) 561 { 562 creds = new NTCredentials( proxyUsername, proxyPassword, proxyNtlmHost, proxyNtlmDomain ); 563 } 564 else 565 { 566 creds = new UsernamePasswordCredentials( proxyUsername, proxyPassword ); 567 } 568 569 int proxyPort = proxyInfo.getPort(); 570 571 AuthScope authScope = getProxyBasicAuthScope().getScope( proxyHost, proxyPort ); 572 credentialsProvider.setCredentials( authScope, creds ); 573 } 574 } 575 } 576 } 577 578 public void closeConnection() 579 { 580 if ( !persistentPool ) 581 { 582 httpClientConnectionManager.closeIdleConnections( 0, TimeUnit.MILLISECONDS ); 583 } 584 585 if ( authCache != null ) 586 { 587 authCache.clear(); 588 authCache = null; 589 } 590 591 if ( credentialsProvider != null ) 592 { 593 credentialsProvider.clear(); 594 credentialsProvider = null; 595 } 596 } 597 598 public static CloseableHttpClient getHttpClient() 599 { 600 return httpClient; 601 } 602 603 public static void setPersistentPool( boolean persistentPool ) 604 { 605 persistentPool = persistentPool; 606 } 607 608 public static void setPoolingHttpClientConnectionManager( 609 PoolingHttpClientConnectionManager poolingHttpClientConnectionManager ) 610 { 611 httpClientConnectionManager = poolingHttpClientConnectionManager; 612 httpClient = createClient(); 613 } 614 615 public void put( File source, String resourceName ) 616 throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException 617 { 618 Resource resource = new Resource( resourceName ); 619 620 firePutInitiated( resource, source ); 621 622 resource.setContentLength( source.length() ); 623 624 resource.setLastModified( source.lastModified() ); 625 626 put( null, resource, source ); 627 } 628 629 public void putFromStream( final InputStream stream, String destination, long contentLength, long lastModified ) 630 throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException 631 { 632 Resource resource = new Resource( destination ); 633 634 firePutInitiated( resource, null ); 635 636 resource.setContentLength( contentLength ); 637 638 resource.setLastModified( lastModified ); 639 640 put( stream, resource, null ); 641 } 642 643 private void put( final InputStream stream, Resource resource, File source ) 644 throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException 645 { 646 put( resource, source, new RequestEntityImplementation( stream, resource, this, source ) ); 647 } 648 649 private void put( Resource resource, File source, HttpEntity httpEntity ) 650 throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException 651 { 652 put( resource, source, httpEntity, buildUrl( resource ) ); 653 } 654 655 /** 656 * Builds a complete URL string from the repository URL and the relative path of the resource passed. 657 * 658 * @param resource the resource to extract the relative path from. 659 * @return the complete URL 660 */ 661 private String buildUrl( Resource resource ) 662 { 663 return EncodingUtil.encodeURLToString( getRepository().getUrl(), resource.getName() ); 664 } 665 666 667 private void put( Resource resource, File source, HttpEntity httpEntity, String url ) 668 throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException 669 { 670 put( getInitialBackoffSeconds(), resource, source, httpEntity, url ); 671 } 672 673 674 private void put( int wait, Resource resource, File source, HttpEntity httpEntity, String url ) 675 throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException 676 { 677 678 //Parent directories need to be created before posting 679 try 680 { 681 mkdirs( PathUtils.dirname( resource.getName() ) ); 682 } 683 catch ( HttpException he ) 684 { 685 fireTransferError( resource, he, TransferEvent.REQUEST_PUT ); 686 } 687 catch ( IOException e ) 688 { 689 fireTransferError( resource, e, TransferEvent.REQUEST_PUT ); 690 } 691 692 // preemptive for put 693 // TODO: is it a good idea, though? 'Expect-continue' handshake would serve much better 694 695 Repository repo = getRepository(); 696 HttpHost targetHost = new HttpHost( repo.getHost(), repo.getPort(), repo.getProtocol() ); 697 AuthScope targetScope = getBasicAuthScope().getScope( targetHost ); 698 699 if ( credentialsProvider.getCredentials( targetScope ) != null ) 700 { 701 BasicScheme targetAuth = new BasicScheme(); 702 authCache.put( targetHost, targetAuth ); 703 } 704 705 HttpPut putMethod = new HttpPut( url ); 706 707 firePutStarted( resource, source ); 708 709 try 710 { 711 putMethod.setEntity( httpEntity ); 712 713 CloseableHttpResponse response = execute( putMethod ); 714 try 715 { 716 int statusCode = response.getStatusLine().getStatusCode(); 717 String reasonPhrase = response.getStatusLine().getReasonPhrase(); 718 StringBuilder debugMessage = new StringBuilder(); 719 debugMessage.append( url ); 720 debugMessage.append( " -- " ); 721 debugMessage.append( "status code: " ).append( statusCode ); 722 if ( StringUtils.isNotEmpty( reasonPhrase ) ) 723 { 724 debugMessage.append( ", reason phrase: " ).append( reasonPhrase ); 725 } 726 fireTransferDebug( debugMessage.toString() ); 727 728 // Check that we didn't run out of retries. 729 switch ( statusCode ) 730 { 731 // Success Codes 732 case HttpStatus.SC_OK: // 200 733 case HttpStatus.SC_CREATED: // 201 734 case HttpStatus.SC_ACCEPTED: // 202 735 case HttpStatus.SC_NO_CONTENT: // 204 736 break; 737 // handle all redirect even if http specs says " the user agent MUST NOT automatically redirect 738 // the request unless it can be confirmed by the user" 739 case HttpStatus.SC_MOVED_PERMANENTLY: // 301 740 case HttpStatus.SC_MOVED_TEMPORARILY: // 302 741 case HttpStatus.SC_SEE_OTHER: // 303 742 put( resource, source, httpEntity, calculateRelocatedUrl( response ) ); 743 return; 744 case HttpStatus.SC_FORBIDDEN: 745 fireSessionConnectionRefused(); 746 throw new AuthorizationException( "Access denied to: " + url ); 747 748 case HttpStatus.SC_NOT_FOUND: 749 throw new ResourceDoesNotExistException( "File " + url + " does not exist" ); 750 751 case SC_TOO_MANY_REQUESTS: 752 put( backoff( wait, url ), resource, source, httpEntity, url ); 753 break; 754 //add more entries here 755 default: 756 TransferFailedException e = new TransferFailedException( 757 "Failed to transfer file " + url + " with status code " + statusCode ); 758 fireTransferError( resource, e, TransferEvent.REQUEST_PUT ); 759 throw e; 760 } 761 762 firePutCompleted( resource, source ); 763 764 EntityUtils.consume( response.getEntity() ); 765 } 766 finally 767 { 768 response.close(); 769 } 770 } 771 catch ( IOException e ) 772 { 773 fireTransferError( resource, e, TransferEvent.REQUEST_PUT ); 774 775 throw new TransferFailedException( e.getMessage(), e ); 776 } 777 catch ( HttpException e ) 778 { 779 fireTransferError( resource, e, TransferEvent.REQUEST_PUT ); 780 781 throw new TransferFailedException( e.getMessage(), e ); 782 } 783 catch ( InterruptedException e ) 784 { 785 fireTransferError( resource, e, TransferEvent.REQUEST_PUT ); 786 787 throw new TransferFailedException( e.getMessage(), e ); 788 } 789 790 } 791 792 protected String calculateRelocatedUrl( HttpResponse response ) 793 { 794 Header locationHeader = response.getFirstHeader( "Location" ); 795 String locationField = locationHeader.getValue(); 796 // is it a relative Location or a full ? 797 return locationField.startsWith( "http" ) ? locationField : getURL( getRepository() ) + '/' + locationField; 798 } 799 800 protected void mkdirs( String dirname ) 801 throws HttpException, IOException 802 { 803 // nothing to do 804 } 805 806 public boolean resourceExists( String resourceName ) 807 throws TransferFailedException, AuthorizationException 808 { 809 return resourceExists( getInitialBackoffSeconds(), resourceName ); 810 } 811 812 813 private boolean resourceExists( int wait, String resourceName ) 814 throws TransferFailedException, AuthorizationException 815 { 816 String repositoryUrl = getRepository().getUrl(); 817 String url = repositoryUrl + ( repositoryUrl.endsWith( "/" ) ? "" : "/" ) + resourceName; 818 HttpHead headMethod = new HttpHead( url ); 819 try 820 { 821 CloseableHttpResponse response = execute( headMethod ); 822 try 823 { 824 int statusCode = response.getStatusLine().getStatusCode(); 825 boolean result; 826 switch ( statusCode ) 827 { 828 case HttpStatus.SC_OK: 829 result = true; 830 break; 831 case HttpStatus.SC_NOT_MODIFIED: 832 result = true; 833 break; 834 case HttpStatus.SC_FORBIDDEN: 835 throw new AuthorizationException( "Access denied to: " + url ); 836 837 case HttpStatus.SC_UNAUTHORIZED: 838 throw new AuthorizationException( "Not authorized" ); 839 840 case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED: 841 throw new AuthorizationException( "Not authorized by proxy" ); 842 843 case HttpStatus.SC_NOT_FOUND: 844 result = false; 845 break; 846 847 case SC_TOO_MANY_REQUESTS: 848 return resourceExists( backoff( wait, resourceName ), resourceName ); 849 850 //add more entries here 851 default: 852 throw new TransferFailedException( 853 "Failed to transfer file " + url + " with status code " + statusCode ); 854 } 855 856 EntityUtils.consume( response.getEntity() ); 857 return result; 858 } 859 finally 860 { 861 response.close(); 862 } 863 } 864 catch ( IOException e ) 865 { 866 throw new TransferFailedException( e.getMessage(), e ); 867 } 868 catch ( HttpException e ) 869 { 870 throw new TransferFailedException( e.getMessage(), e ); 871 } 872 catch ( InterruptedException e ) 873 { 874 throw new TransferFailedException( e.getMessage(), e ); 875 } 876 877 } 878 879 protected CloseableHttpResponse execute( HttpUriRequest httpMethod ) 880 throws HttpException, IOException 881 { 882 setHeaders( httpMethod ); 883 String userAgent = getUserAgent( httpMethod ); 884 if ( userAgent != null ) 885 { 886 httpMethod.setHeader( HTTP.USER_AGENT, userAgent ); 887 } 888 889 RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); 890 // WAGON-273: default the cookie-policy to browser compatible 891 requestConfigBuilder.setCookieSpec( CookieSpecs.BROWSER_COMPATIBILITY ); 892 893 Repository repo = getRepository(); 894 ProxyInfo proxyInfo = getProxyInfo( repo.getProtocol(), repo.getHost() ); 895 if ( proxyInfo != null ) 896 { 897 HttpHost proxy = new HttpHost( proxyInfo.getHost(), proxyInfo.getPort() ); 898 requestConfigBuilder.setProxy( proxy ); 899 } 900 901 HttpMethodConfiguration config = 902 httpConfiguration == null ? null : httpConfiguration.getMethodConfiguration( httpMethod ); 903 904 if ( config != null ) 905 { 906 ConfigurationUtils.copyConfig( config, requestConfigBuilder ); 907 } 908 else 909 { 910 requestConfigBuilder.setSocketTimeout( getReadTimeout() ); 911 if ( httpMethod instanceof HttpPut ) 912 { 913 requestConfigBuilder.setExpectContinueEnabled( true ); 914 } 915 } 916 917 if ( httpMethod instanceof HttpPut ) 918 { 919 requestConfigBuilder.setRedirectsEnabled( false ); 920 } 921 922 HttpClientContext localContext = HttpClientContext.create(); 923 localContext.setCredentialsProvider( credentialsProvider ); 924 localContext.setAuthCache( authCache ); 925 localContext.setRequestConfig( requestConfigBuilder.build() ); 926 927 if ( config != null && config.isUsePreemptive() ) 928 { 929 HttpHost targetHost = new HttpHost( repo.getHost(), repo.getPort(), repo.getProtocol() ); 930 AuthScope targetScope = getBasicAuthScope().getScope( targetHost ); 931 932 if ( credentialsProvider.getCredentials( targetScope ) != null ) 933 { 934 BasicScheme targetAuth = new BasicScheme(); 935 authCache.put( targetHost, targetAuth ); 936 } 937 } 938 939 if ( proxyInfo != null ) 940 { 941 if ( proxyInfo.getHost() != null ) 942 { 943 HttpHost proxyHost = new HttpHost( proxyInfo.getHost(), proxyInfo.getPort() ); 944 AuthScope proxyScope = getProxyBasicAuthScope().getScope( proxyHost ); 945 946 if ( credentialsProvider.getCredentials( proxyScope ) != null ) 947 { 948 /* This is extremely ugly because we need to set challengeState to PROXY, but 949 * the constructor is deprecated. Alternatively, we could subclass BasicScheme 950 * to ProxyBasicScheme and set the state internally in the constructor. 951 */ 952 BasicScheme proxyAuth = new BasicScheme( ChallengeState.PROXY ); 953 authCache.put( proxyHost, proxyAuth ); 954 } 955 } 956 } 957 958 return httpClient.execute( httpMethod, localContext ); 959 } 960 961 public void setHeaders( HttpUriRequest method ) 962 { 963 HttpMethodConfiguration config = 964 httpConfiguration == null ? null : httpConfiguration.getMethodConfiguration( method ); 965 if ( config == null || config.isUseDefaultHeaders() ) 966 { 967 // TODO: merge with the other headers and have some better defaults, unify with lightweight headers 968 method.addHeader( "Cache-control", "no-cache" ); 969 method.addHeader( "Cache-store", "no-store" ); 970 method.addHeader( "Pragma", "no-cache" ); 971 } 972 973 if ( httpHeaders != null ) 974 { 975 for ( Map.Entry<Object, Object> entry : httpHeaders.entrySet() ) 976 { 977 method.setHeader( (String) entry.getKey(), (String) entry.getValue() ); 978 } 979 } 980 981 Header[] headers = config == null ? null : config.asRequestHeaders(); 982 if ( headers != null ) 983 { 984 for ( Header header : headers ) 985 { 986 method.setHeader( header ); 987 } 988 } 989 990 Header userAgentHeader = method.getFirstHeader( HTTP.USER_AGENT ); 991 if ( userAgentHeader == null ) 992 { 993 String userAgent = getUserAgent( method ); 994 if ( userAgent != null ) 995 { 996 method.setHeader( HTTP.USER_AGENT, userAgent ); 997 } 998 } 999 } 1000 1001 protected String getUserAgent( HttpUriRequest method ) 1002 { 1003 if ( httpHeaders != null ) 1004 { 1005 String value = (String) httpHeaders.get( HTTP.USER_AGENT ); 1006 if ( value != null ) 1007 { 1008 return value; 1009 } 1010 } 1011 HttpMethodConfiguration config = 1012 httpConfiguration == null ? null : httpConfiguration.getMethodConfiguration( method ); 1013 1014 if ( config != null ) 1015 { 1016 return (String) config.getHeaders().get( HTTP.USER_AGENT ); 1017 } 1018 return null; 1019 } 1020 1021 /** 1022 * getUrl 1023 * Implementors can override this to remove unwanted parts of the url such as role-hints 1024 * 1025 * @param repository 1026 * @return 1027 */ 1028 protected String getURL( Repository repository ) 1029 { 1030 return repository.getUrl(); 1031 } 1032 1033 public HttpConfiguration getHttpConfiguration() 1034 { 1035 return httpConfiguration; 1036 } 1037 1038 public void setHttpConfiguration( HttpConfiguration httpConfiguration ) 1039 { 1040 this.httpConfiguration = httpConfiguration; 1041 } 1042 1043 /** 1044 * Get the override values for standard HttpClient AuthScope 1045 * 1046 * @return the basicAuth 1047 */ 1048 public BasicAuthScope getBasicAuthScope() 1049 { 1050 if ( basicAuth == null ) 1051 { 1052 basicAuth = new BasicAuthScope(); 1053 } 1054 return basicAuth; 1055 } 1056 1057 /** 1058 * Set the override values for standard HttpClient AuthScope 1059 * 1060 * @param basicAuth the AuthScope to set 1061 */ 1062 public void setBasicAuthScope( BasicAuthScope basicAuth ) 1063 { 1064 this.basicAuth = basicAuth; 1065 } 1066 1067 /** 1068 * Get the override values for proxy HttpClient AuthScope 1069 * 1070 * @return the proxyAuth 1071 */ 1072 public BasicAuthScope getProxyBasicAuthScope() 1073 { 1074 if ( proxyAuth == null ) 1075 { 1076 proxyAuth = new BasicAuthScope(); 1077 } 1078 return proxyAuth; 1079 } 1080 1081 /** 1082 * Set the override values for proxy HttpClient AuthScope 1083 * 1084 * @param proxyAuth the AuthScope to set 1085 */ 1086 public void setProxyBasicAuthScope( BasicAuthScope proxyAuth ) 1087 { 1088 this.proxyAuth = proxyAuth; 1089 } 1090 1091 public void fillInputData( InputData inputData ) 1092 throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException 1093 { 1094 fillInputData( getInitialBackoffSeconds(), inputData ); 1095 } 1096 1097 private void fillInputData( int wait, InputData inputData ) 1098 throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException 1099 { 1100 Resource resource = inputData.getResource(); 1101 1102 String repositoryUrl = getRepository().getUrl(); 1103 String url = repositoryUrl + ( repositoryUrl.endsWith( "/" ) ? "" : "/" ) + resource.getName(); 1104 HttpGet getMethod = new HttpGet( url ); 1105 long timestamp = resource.getLastModified(); 1106 if ( timestamp > 0 ) 1107 { 1108 SimpleDateFormat fmt = new SimpleDateFormat( "EEE, dd-MMM-yy HH:mm:ss zzz", Locale.US ); 1109 fmt.setTimeZone( GMT_TIME_ZONE ); 1110 Header hdr = new BasicHeader( "If-Modified-Since", fmt.format( new Date( timestamp ) ) ); 1111 fireTransferDebug( "sending ==> " + hdr + "(" + timestamp + ")" ); 1112 getMethod.addHeader( hdr ); 1113 } 1114 1115 try 1116 { 1117 CloseableHttpResponse response = execute( getMethod ); 1118 closeable = response; 1119 int statusCode = response.getStatusLine().getStatusCode(); 1120 String reasonPhrase = response.getStatusLine().getReasonPhrase(); 1121 StringBuilder debugMessage = new StringBuilder(); 1122 debugMessage.append( url ); 1123 debugMessage.append( " -- " ); 1124 debugMessage.append( "status code: " ).append( statusCode ); 1125 if ( StringUtils.isNotEmpty( reasonPhrase ) ) 1126 { 1127 debugMessage.append( ", reason phrase: " ).append( reasonPhrase ); 1128 } 1129 fireTransferDebug( debugMessage.toString() ); 1130 1131 switch ( statusCode ) 1132 { 1133 case HttpStatus.SC_OK: 1134 break; 1135 1136 case HttpStatus.SC_NOT_MODIFIED: 1137 // return, leaving last modified set to original value so getIfNewer should return unmodified 1138 return; 1139 case HttpStatus.SC_FORBIDDEN: 1140 fireSessionConnectionRefused(); 1141 throw new AuthorizationException( "Access denied to: " + url ); 1142 1143 case HttpStatus.SC_UNAUTHORIZED: 1144 fireSessionConnectionRefused(); 1145 throw new AuthorizationException( "Not authorized" ); 1146 1147 case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED: 1148 fireSessionConnectionRefused(); 1149 throw new AuthorizationException( "Not authorized by proxy" ); 1150 1151 case HttpStatus.SC_NOT_FOUND: 1152 throw new ResourceDoesNotExistException( "File " + url + " does not exist" ); 1153 1154 case SC_TOO_MANY_REQUESTS: 1155 fillInputData( backoff( wait, url ), inputData ); 1156 break; 1157 1158 // add more entries here 1159 default: 1160 cleanupGetTransfer( resource ); 1161 TransferFailedException e = new TransferFailedException( 1162 "Failed to transfer file " + url + " with status code " + statusCode ); 1163 fireTransferError( resource, e, TransferEvent.REQUEST_GET ); 1164 throw e; 1165 } 1166 1167 Header contentLengthHeader = response.getFirstHeader( "Content-Length" ); 1168 1169 if ( contentLengthHeader != null ) 1170 { 1171 try 1172 { 1173 long contentLength = Long.parseLong( contentLengthHeader.getValue() ); 1174 1175 resource.setContentLength( contentLength ); 1176 } 1177 catch ( NumberFormatException e ) 1178 { 1179 fireTransferDebug( 1180 "error parsing content length header '" + contentLengthHeader.getValue() + "' " + e ); 1181 } 1182 } 1183 1184 Header lastModifiedHeader = response.getFirstHeader( "Last-Modified" ); 1185 if ( lastModifiedHeader != null ) 1186 { 1187 Date lastModified = DateUtils.parseDate( lastModifiedHeader.getValue() ); 1188 if ( lastModified != null ) 1189 { 1190 resource.setLastModified( lastModified.getTime() ); 1191 fireTransferDebug( "last-modified = " + lastModifiedHeader.getValue() + " (" 1192 + lastModified.getTime() + ")" ); 1193 } 1194 } 1195 1196 HttpEntity entity = response.getEntity(); 1197 if ( entity != null ) 1198 { 1199 inputData.setInputStream( entity.getContent() ); 1200 } 1201 } 1202 catch ( IOException e ) 1203 { 1204 fireTransferError( resource, e, TransferEvent.REQUEST_GET ); 1205 1206 throw new TransferFailedException( e.getMessage(), e ); 1207 } 1208 catch ( HttpException e ) 1209 { 1210 fireTransferError( resource, e, TransferEvent.REQUEST_GET ); 1211 1212 throw new TransferFailedException( e.getMessage(), e ); 1213 } 1214 catch ( InterruptedException e ) 1215 { 1216 fireTransferError( resource, e, TransferEvent.REQUEST_GET ); 1217 1218 throw new TransferFailedException( e.getMessage(), e ); 1219 } 1220 1221 } 1222 1223 protected void cleanupGetTransfer( Resource resource ) 1224 { 1225 if ( closeable != null ) 1226 { 1227 try 1228 { 1229 closeable.close(); 1230 } 1231 catch ( IOException ignore ) 1232 { 1233 // ignore 1234 } 1235 1236 } 1237 } 1238 1239 1240 @Override 1241 public void putFromStream( InputStream stream, String destination ) 1242 throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException 1243 { 1244 putFromStream( stream, destination, -1, -1 ); 1245 } 1246 1247 @Override 1248 protected void putFromStream( InputStream stream, Resource resource ) 1249 throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException 1250 { 1251 putFromStream( stream, resource.getName(), -1, -1 ); 1252 } 1253 1254 public Properties getHttpHeaders() 1255 { 1256 return httpHeaders; 1257 } 1258 1259 public void setHttpHeaders( Properties httpHeaders ) 1260 { 1261 this.httpHeaders = httpHeaders; 1262 } 1263 1264 @Override 1265 public void fillOutputData( OutputData outputData ) 1266 throws TransferFailedException 1267 { 1268 // no needed in this implementation but throw an Exception if used 1269 throw new IllegalStateException( "this wagon http client must not use fillOutputData" ); 1270 } 1271 1272 protected CredentialsProvider getCredentialsProvider() 1273 { 1274 return credentialsProvider; 1275 } 1276 1277 protected AuthCache getAuthCache() 1278 { 1279 return authCache; 1280 } 1281 1282 public int getInitialBackoffSeconds() 1283 { 1284 return initialBackoffSeconds; 1285 } 1286 1287 public void setInitialBackoffSeconds( int initialBackoffSeconds ) 1288 { 1289 this.initialBackoffSeconds = initialBackoffSeconds; 1290 } 1291 1292 public static int getMaxBackoffWaitSeconds() 1293 { 1294 return MAX_BACKOFF_WAIT_SECONDS; 1295 } 1296}