View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.eclipse.aether.transport.url;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.net.HttpURLConnection;
24  import java.net.InetSocketAddress;
25  import java.net.Proxy;
26  import java.net.URI;
27  import java.net.URISyntaxException;
28  import java.nio.charset.Charset;
29  import java.nio.file.Path;
30  import java.util.ArrayList;
31  import java.util.Base64;
32  import java.util.Collections;
33  import java.util.Map;
34  import java.util.function.Consumer;
35  import java.util.function.Function;
36  import java.util.zip.GZIPInputStream;
37  import java.util.zip.InflaterInputStream;
38  
39  import org.eclipse.aether.Keys;
40  import org.eclipse.aether.RepositorySystemSession;
41  import org.eclipse.aether.repository.AuthenticationContext;
42  import org.eclipse.aether.repository.RemoteRepository;
43  import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
44  import org.eclipse.aether.spi.connector.transport.GetTask;
45  import org.eclipse.aether.spi.connector.transport.PeekTask;
46  import org.eclipse.aether.spi.connector.transport.PutTask;
47  import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
48  import org.eclipse.aether.spi.connector.transport.http.HttpConstants;
49  import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
50  import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
51  import org.eclipse.aether.spi.io.PathProcessor;
52  import org.eclipse.aether.transfer.NoTransporterException;
53  import org.eclipse.aether.util.ConfigUtils;
54  import org.eclipse.aether.util.connector.transport.http.HttpTransporterUtils;
55  
56  /**
57   * A special, "read only" and limited capability transport usable for bootstrapping. It provides HTTP with minimal
58   * support (only basic auth, only GET/HEAD). It is implemented using {@link java.net.HttpURLConnection} class.
59   *
60   * @since 2.0.21
61   */
62  public class UrlTransporter extends AbstractTransporter implements HttpTransporter {
63  
64      private static final String METHOD_GET = "GET";
65      private static final String METHOD_HEAD = "HEAD";
66      private static final String HEADER_LOCATION = "Location";
67      private static final String HEADER_AUTHORIZATION = "Authorization";
68      private static final String HEADER_PROXY_AUTHORIZATION = "Proxy-Authorization";
69      private static final String AUTH_SCHEME_BASIC = "Basic";
70      private static final int HTTP_STATUS_TEMPORARY_REDIRECT = 307;
71      private static final int HTTP_STATUS_PERMANENT_REDIRECT = 308;
72  
73      private enum RedirectMode {
74          /**
75           * No redirects allowed.
76           */
77          NONE,
78          /**
79           * Redirects only within same authority.
80           */
81          SAME_AUTHORITY,
82          /**
83           * Any redirect is followed.
84           */
85          ANY
86      }
87  
88      private final ChecksumExtractor checksumExtractor;
89      private final PathProcessor pathProcessor;
90      private final URI baseUri;
91      private final Map<String, String> headers;
92      private final String userAgent;
93      private final int connectTimeout;
94      private final int requestTimeout;
95      private final boolean preemptiveAuth;
96      private final Charset authEncoding;
97      private final String auth;
98      private final Proxy proxy;
99      private final String proxyAuth;
100     private final RedirectMode redirectMode;
101     private final boolean redirectAllowDowngrade;
102     private final int maxRedirects;
103     private final boolean closeConnection;
104 
105     private final Object authKey;
106     private final Object proxyAuthKey;
107     private final Function<Object, Boolean> cacheGetter;
108     private final Consumer<Object> cacheSetter;
109 
110     @FunctionalInterface
111     private interface IOSupplier<T> {
112         T get() throws IOException;
113     }
114 
115     public UrlTransporter(
116             RemoteRepository repository,
117             RepositorySystemSession session,
118             ChecksumExtractor checksumExtractor,
119             PathProcessor pathProcessor)
120             throws NoTransporterException {
121         this.checksumExtractor = checksumExtractor;
122         this.pathProcessor = pathProcessor;
123         try {
124             this.baseUri = HttpTransporterUtils.getBaseUri(repository);
125         } catch (URISyntaxException e) {
126             throw new NoTransporterException(repository, e.getMessage(), e);
127         }
128 
129         this.headers = HttpTransporterUtils.getHttpHeaders(session, repository);
130         this.userAgent = HttpTransporterUtils.getUserAgent(session, repository);
131         this.connectTimeout = HttpTransporterUtils.getHttpConnectTimeout(session, repository);
132         this.requestTimeout = HttpTransporterUtils.getHttpRequestTimeout(session, repository);
133         String authString = null;
134         try (AuthenticationContext repoAuthContext = AuthenticationContext.forRepository(session, repository)) {
135             if (repoAuthContext != null) {
136                 String username = repoAuthContext.get(AuthenticationContext.USERNAME);
137                 String password = repoAuthContext.get(AuthenticationContext.PASSWORD);
138                 if (username != null && password != null) {
139                     authString = username + ":" + password;
140                 }
141             }
142         }
143         this.authEncoding = HttpTransporterUtils.getHttpCredentialsEncoding(session, repository);
144         this.auth = authString;
145         this.preemptiveAuth = this.auth != null && HttpTransporterUtils.isHttpPreemptiveAuth(session, repository);
146 
147         org.eclipse.aether.repository.Proxy repoProxy = repository.getProxy();
148         this.proxy = repoProxy != null
149                 ? new Proxy(Proxy.Type.HTTP, new InetSocketAddress(repoProxy.getHost(), repoProxy.getPort()))
150                 : Proxy.NO_PROXY;
151         String proxyAuthString = null;
152         try (AuthenticationContext proxyAuthContext = AuthenticationContext.forProxy(session, repository)) {
153             if (proxyAuthContext != null) {
154                 String username = proxyAuthContext.get(AuthenticationContext.USERNAME);
155                 String password = proxyAuthContext.get(AuthenticationContext.PASSWORD);
156                 if (username != null && password != null) {
157                     proxyAuthString = username + ":" + password;
158                 }
159             }
160         }
161         this.proxyAuth = proxyAuthString;
162 
163         this.redirectMode = RedirectMode.valueOf(ConfigUtils.getString(
164                 session,
165                 UrlTransporterConfigurationKeys.DEFAULT_REDIRECT_MODE,
166                 UrlTransporterConfigurationKeys.CONFIG_PROP_REDIRECT_MODE + "." + repository.getId(),
167                 UrlTransporterConfigurationKeys.CONFIG_PROP_REDIRECT_MODE));
168         this.redirectAllowDowngrade = ConfigUtils.getBoolean(
169                 session,
170                 UrlTransporterConfigurationKeys.DEFAULT_REDIRECT_ALLOW_DOWNGRADE,
171                 UrlTransporterConfigurationKeys.CONFIG_PROP_REDIRECT_ALLOW_DOWNGRADE + "." + repository.getId(),
172                 UrlTransporterConfigurationKeys.CONFIG_PROP_REDIRECT_ALLOW_DOWNGRADE);
173         this.maxRedirects = ConfigUtils.getInteger(
174                 session,
175                 UrlTransporterConfigurationKeys.DEFAULT_MAX_REDIRECT_COUNT,
176                 UrlTransporterConfigurationKeys.CONFIG_PROP_MAX_REDIRECT_COUNT + "." + repository.getId(),
177                 UrlTransporterConfigurationKeys.CONFIG_PROP_MAX_REDIRECT_COUNT);
178         this.closeConnection = ConfigUtils.getBoolean(
179                 session,
180                 UrlTransporterConfigurationKeys.DEFAULT_CLOSE_CONNECTION,
181                 UrlTransporterConfigurationKeys.CONFIG_PROP_CLOSE_CONNECTION + "." + repository.getId(),
182                 UrlTransporterConfigurationKeys.CONFIG_PROP_CLOSE_CONNECTION);
183 
184         this.authKey = Keys.of(UrlTransporter.class, repository, "auth");
185         this.proxyAuthKey = Keys.of(UrlTransporter.class, repository, "proxyAuth");
186         if (session.getCache() != null) {
187             this.cacheGetter = k -> {
188                 Boolean ret = (Boolean) session.getCache().get(session, k);
189                 if (ret == null) {
190                     return false;
191                 } else {
192                     return ret;
193                 }
194             };
195             this.cacheSetter = k -> session.getCache().put(session, k, Boolean.TRUE);
196         } else {
197             this.cacheGetter = k -> false;
198             this.cacheSetter = k -> {};
199         }
200     }
201 
202     @Override
203     protected void implPeek(PeekTask task) throws Exception {
204         HttpURLConnection con = perform(METHOD_HEAD, baseUri.resolve(task.getLocation()), null);
205         try {
206             int responseCode = con.getResponseCode();
207             if (HttpURLConnection.HTTP_OK != responseCode) {
208                 throw new HttpTransporterException(responseCode);
209             }
210         } finally {
211             con.disconnect();
212         }
213     }
214 
215     @Override
216     protected void implGet(GetTask task) throws Exception {
217         HttpURLConnection con = perform(METHOD_GET, baseUri.resolve(task.getLocation()), task);
218         try {
219             int responseCode = con.getResponseCode();
220             if (HttpURLConnection.HTTP_OK != responseCode) {
221                 throw new HttpTransporterException(responseCode);
222             }
223             IOSupplier<InputStream> inputStreamSupplier = () -> {
224                 String contentEncoding = con.getHeaderField("Content-Encoding");
225                 if (contentEncoding != null) {
226                     if ("gzip".equalsIgnoreCase(contentEncoding)) {
227                         return new GZIPInputStream(con.getInputStream());
228                     } else if ("deflate".equalsIgnoreCase(contentEncoding)) {
229                         return new InflaterInputStream(con.getInputStream());
230                     }
231                 }
232                 return con.getInputStream();
233             };
234             final Path dataFile = task.getDataPath();
235             if (dataFile == null) {
236                 try (InputStream is = inputStreamSupplier.get()) {
237                     utilGet(task, is, true, con.getContentLengthLong(), false);
238                 }
239             } else {
240                 try (PathProcessor.CollocatedTempFile tempFile = pathProcessor.newTempFile(dataFile)) {
241                     task.setDataPath(tempFile.getPath(), false);
242                     try (InputStream is = inputStreamSupplier.get()) {
243                         utilGet(task, is, true, con.getContentLengthLong(), false);
244                     }
245                     tempFile.move();
246                 } finally {
247                     task.setDataPath(dataFile);
248                 }
249             }
250             if (task.getDataPath() != null) {
251                 long lastModified = con.getLastModified();
252                 if (lastModified != 0) {
253                     pathProcessor.setLastModified(task.getDataPath(), lastModified);
254                 }
255             }
256         } finally {
257             con.disconnect();
258         }
259     }
260 
261     @Override
262     protected void implPut(PutTask task) {
263         throw new UnsupportedOperationException("PUT method unsupported");
264     }
265 
266     @Override
267     protected void implClose() {
268         // nothing
269     }
270 
271     private HttpURLConnection perform(String method, URI target, GetTask task) throws IOException {
272         String currAuth = preemptiveAuth ? auth : null;
273         String currProxyAuth = null;
274         if (cacheGetter.apply(authKey)) {
275             currAuth = this.auth;
276         }
277         if (cacheGetter.apply(proxyAuthKey)) {
278             currProxyAuth = this.proxyAuth;
279         }
280         return perform(method, new ArrayList<>(Collections.singletonList(target)), currAuth, currProxyAuth, task);
281     }
282 
283     private HttpURLConnection perform(
284             String method, ArrayList<URI> target, String currAuth, String currProxyAuth, GetTask task)
285             throws IOException {
286         if (target.size() - 1 > maxRedirects) {
287             throw new IOException("Too many redirects");
288         }
289         HttpURLConnection con = (HttpURLConnection) target.get(0).toURL().openConnection(proxy);
290         con.setConnectTimeout(connectTimeout);
291         con.setReadTimeout(requestTimeout);
292         con.setRequestMethod(method);
293         con.setUseCaches(false);
294         con.setInstanceFollowRedirects(false);
295         con.setRequestProperty(HttpConstants.ACCEPT_ENCODING, "gzip,deflate");
296         con.setRequestProperty(HttpConstants.CACHE_CONTROL, "no-cache, no-store");
297         con.setRequestProperty("Pragma", "no-cache");
298         con.setRequestProperty(HttpConstants.USER_AGENT, userAgent);
299         headers.forEach(con::setRequestProperty);
300         if (closeConnection) {
301             con.setRequestProperty("Connection", "close");
302         }
303         if (currAuth != null) {
304             con.setRequestProperty(HEADER_AUTHORIZATION, basicAuthorization(currAuth));
305         }
306         if (currProxyAuth != null) {
307             con.setRequestProperty(HEADER_PROXY_AUTHORIZATION, basicAuthorization(currProxyAuth));
308         }
309         int responseCode = con.getResponseCode();
310         if (responseCode == HttpURLConnection.HTTP_OK) {
311             if (task != null) {
312                 Map<String, String> checksums = checksumExtractor.extractChecksums(con::getHeaderField);
313                 if (checksums != null && !checksums.isEmpty()) {
314                     checksums.forEach(task::setChecksum);
315                 }
316             }
317         } else if (responseCode == HttpURLConnection.HTTP_MOVED_PERM
318                 || responseCode == HttpURLConnection.HTTP_MOVED_TEMP
319                 || responseCode == HttpURLConnection.HTTP_SEE_OTHER
320                 || responseCode == HTTP_STATUS_TEMPORARY_REDIRECT
321                 || responseCode == HTTP_STATUS_PERMANENT_REDIRECT) {
322             if (redirectMode == RedirectMode.NONE) {
323                 con.disconnect();
324                 throw new IOException("Refusing to follow redirects");
325             }
326             final String location = con.getHeaderField(HEADER_LOCATION);
327             if (location == null) {
328                 con.disconnect();
329                 throw new IOException("Redirect response missing Location header");
330             }
331             final URI currentUri;
332             final URI redirectUri;
333             try {
334                 currentUri = URI.create(con.getURL().toString());
335                 redirectUri = currentUri.resolve(location);
336             } catch (IllegalArgumentException e) {
337                 con.disconnect();
338                 throw new IOException("Redirect response has invalid Location header: " + location, e);
339             }
340             // ensure we are HTTP or HTTPS after redirect
341             if (!"http".equalsIgnoreCase(redirectUri.getScheme())
342                     && !"https".equalsIgnoreCase(redirectUri.getScheme())) {
343                 con.disconnect();
344                 throw new IOException("Unsupported redirect protocol: " + redirectUri.getScheme());
345             }
346             String currentAuthority = currentUri.getAuthority();
347             String redirectAuthority = redirectUri.getAuthority();
348             if (currentAuthority == null || !currentAuthority.equalsIgnoreCase(redirectAuthority)) {
349                 if (redirectMode == RedirectMode.SAME_AUTHORITY) {
350                     con.disconnect();
351                     throw new IOException("Refusing to follow redirect to different authority: " + redirectAuthority);
352                 } else {
353                     // reset auth if authority differs after redirect
354                     currAuth = null;
355                 }
356             }
357             if ("https".equalsIgnoreCase(currentUri.getScheme())
358                     && "http".equalsIgnoreCase(redirectUri.getScheme())
359                     && !redirectAllowDowngrade) {
360                 // forbid HTTPS -> HTTP downgrade during redirect
361                 con.disconnect();
362                 throw new IOException("Refusing to downgrade from HTTPS to HTTP protocol");
363             }
364             target.add(0, redirectUri);
365             con.disconnect();
366             return perform(method, target, currAuth, currProxyAuth, task);
367         } else if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED && currAuth == null && this.auth != null) {
368             con.disconnect();
369             return perform(method, target, this.auth, currProxyAuth, task);
370         } else if (responseCode == HttpURLConnection.HTTP_PROXY_AUTH
371                 && currProxyAuth == null
372                 && this.proxyAuth != null) {
373             con.disconnect();
374             return perform(method, target, currAuth, this.proxyAuth, task);
375         }
376         if (currAuth != null) {
377             cacheSetter.accept(authKey);
378         }
379         if (currProxyAuth != null) {
380             cacheSetter.accept(proxyAuthKey);
381         }
382         return con;
383     }
384 
385     private String basicAuthorization(String credentials) {
386         return AUTH_SCHEME_BASIC + " " + Base64.getEncoder().encodeToString(credentials.getBytes(authEncoding));
387     }
388 }