View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.eclipse.aether.transport.url;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.net.HttpURLConnection;
24  import java.net.InetSocketAddress;
25  import java.net.Proxy;
26  import java.net.URI;
27  import java.net.URISyntaxException;
28  import java.nio.charset.Charset;
29  import java.nio.file.Path;
30  import java.util.ArrayList;
31  import java.util.Base64;
32  import java.util.Collections;
33  import java.util.Map;
34  import java.util.function.Consumer;
35  import java.util.function.Function;
36  import java.util.zip.GZIPInputStream;
37  import java.util.zip.InflaterInputStream;
38  
39  import org.eclipse.aether.Keys;
40  import org.eclipse.aether.RepositorySystemSession;
41  import org.eclipse.aether.repository.AuthenticationContext;
42  import org.eclipse.aether.repository.RemoteRepository;
43  import org.eclipse.aether.spi.connector.transport.AbstractTransporter;
44  import org.eclipse.aether.spi.connector.transport.GetTask;
45  import org.eclipse.aether.spi.connector.transport.PeekTask;
46  import org.eclipse.aether.spi.connector.transport.PutTask;
47  import org.eclipse.aether.spi.connector.transport.http.ChecksumExtractor;
48  import org.eclipse.aether.spi.connector.transport.http.HttpConstants;
49  import org.eclipse.aether.spi.connector.transport.http.HttpTransporter;
50  import org.eclipse.aether.spi.connector.transport.http.HttpTransporterException;
51  import org.eclipse.aether.spi.io.PathProcessor;
52  import org.eclipse.aether.transfer.NoTransporterException;
53  import org.eclipse.aether.util.ConfigUtils;
54  import org.eclipse.aether.util.connector.transport.http.HttpTransporterUtils;
55  
56  /**
57   * A special, "read only" and limited capability transport usable for bootstrapping. It provides HTTP with minimal
58   * support (only basic auth, only GET/HEAD). It is implemented using {@link java.net.HttpURLConnection} class.
59   *
60   * @since 2.0.21
61   */
62  public class UrlTransporter extends AbstractTransporter implements HttpTransporter {
63  
64      private static final String METHOD_GET = "GET";
65      private static final String METHOD_HEAD = "HEAD";
66      private static final String HEADER_LOCATION = "Location";
67      private static final String HEADER_AUTHORIZATION = "Authorization";
68      private static final String HEADER_PROXY_AUTHORIZATION = "Proxy-Authorization";
69      private static final String AUTH_SCHEME_BASIC = "Basic";
70      private static final int HTTP_STATUS_TEMPORARY_REDIRECT = 307;
71      private static final int HTTP_STATUS_PERMANENT_REDIRECT = 308;
72  
73      private enum RedirectMode {
74          /**
75           * No redirects allowed.
76           */
77          NONE,
78          /**
79           * Redirects only within same authority.
80           */
81          SAME_AUTHORITY,
82          /**
83           * Any redirect is followed.
84           */
85          ANY
86      }
87  
88      private final ChecksumExtractor checksumExtractor;
89      private final PathProcessor pathProcessor;
90      private final URI baseUri;
91      private final Map<String, String> headers;
92      private final String userAgent;
93      private final int connectTimeout;
94      private final int requestTimeout;
95      private final boolean preemptiveAuth;
96      private final Charset authEncoding;
97      private final String auth;
98      private final Proxy proxy;
99      private final String proxyAuth;
100     private final RedirectMode redirectMode;
101     private final boolean redirectAllowDowngrade;
102     private final int maxRedirects;
103     private final boolean closeConnection;
104 
105     private final Object authKey;
106     private final Object proxyAuthKey;
107     private final Function<Object, Boolean> cacheGetter;
108     private final Consumer<Object> cacheSetter;
109 
110     @FunctionalInterface
111     private interface IOSupplier<T> {
112         T get() throws IOException;
113     }
114 
115     public UrlTransporter(
116             RemoteRepository repository,
117             RepositorySystemSession session,
118             ChecksumExtractor checksumExtractor,
119             PathProcessor pathProcessor)
120             throws NoTransporterException {
121         this.checksumExtractor = checksumExtractor;
122         this.pathProcessor = pathProcessor;
123         try {
124             this.baseUri = HttpTransporterUtils.getBaseUri(repository);
125         } catch (URISyntaxException e) {
126             throw new NoTransporterException(repository, e.getMessage(), e);
127         }
128 
129         this.headers = HttpTransporterUtils.getHttpHeaders(session, repository);
130         this.userAgent = HttpTransporterUtils.getUserAgent(session, repository);
131         this.connectTimeout = HttpTransporterUtils.getHttpConnectTimeout(session, repository);
132         this.requestTimeout = HttpTransporterUtils.getHttpRequestTimeout(session, repository);
133         String authString = null;
134         try (AuthenticationContext repoAuthContext = AuthenticationContext.forRepository(session, repository)) {
135             if (repoAuthContext != null) {
136                 String username = repoAuthContext.get(AuthenticationContext.USERNAME);
137                 String password = repoAuthContext.get(AuthenticationContext.PASSWORD);
138                 if (username != null && password != null) {
139                     authString = username + ":" + password;
140                 }
141             }
142         }
143         this.authEncoding = HttpTransporterUtils.getHttpCredentialsEncoding(session, repository);
144         this.auth = authString;
145         this.preemptiveAuth = this.auth != null && HttpTransporterUtils.isHttpPreemptiveAuth(session, repository);
146 
147         org.eclipse.aether.repository.Proxy repoProxy = repository.getProxy();
148         this.proxy = repoProxy != null
149                 ? new Proxy(Proxy.Type.HTTP, new InetSocketAddress(repoProxy.getHost(), repoProxy.getPort()))
150                 : Proxy.NO_PROXY;
151         String proxyAuthString = null;
152         try (AuthenticationContext proxyAuthContext = AuthenticationContext.forProxy(session, repository)) {
153             if (proxyAuthContext != null) {
154                 String username = proxyAuthContext.get(AuthenticationContext.USERNAME);
155                 String password = proxyAuthContext.get(AuthenticationContext.PASSWORD);
156                 if (username != null && password != null) {
157                     proxyAuthString = username + ":" + password;
158                 }
159             }
160         }
161         this.proxyAuth = proxyAuthString;
162 
163         this.redirectMode = RedirectMode.valueOf(ConfigUtils.getString(
164                 session,
165                 UrlTransporterConfigurationKeys.DEFAULT_REDIRECT_MODE,
166                 UrlTransporterConfigurationKeys.CONFIG_PROP_REDIRECT_MODE + "." + repository.getId(),
167                 UrlTransporterConfigurationKeys.CONFIG_PROP_REDIRECT_MODE));
168         this.redirectAllowDowngrade = ConfigUtils.getBoolean(
169                 session,
170                 UrlTransporterConfigurationKeys.DEFAULT_REDIRECT_ALLOW_DOWNGRADE,
171                 UrlTransporterConfigurationKeys.CONFIG_PROP_REDIRECT_ALLOW_DOWNGRADE + "." + repository.getId(),
172                 UrlTransporterConfigurationKeys.CONFIG_PROP_REDIRECT_ALLOW_DOWNGRADE);
173         this.maxRedirects = ConfigUtils.getInteger(
174                 session,
175                 UrlTransporterConfigurationKeys.DEFAULT_MAX_REDIRECT_COUNT,
176                 UrlTransporterConfigurationKeys.CONFIG_PROP_MAX_REDIRECT_COUNT + "." + repository.getId(),
177                 UrlTransporterConfigurationKeys.CONFIG_PROP_MAX_REDIRECT_COUNT);
178         this.closeConnection = ConfigUtils.getBoolean(
179                 session,
180                 UrlTransporterConfigurationKeys.DEFAULT_CLOSE_CONNECTION,
181                 UrlTransporterConfigurationKeys.CONFIG_PROP_CLOSE_CONNECTION + "." + repository.getId(),
182                 UrlTransporterConfigurationKeys.CONFIG_PROP_CLOSE_CONNECTION);
183 
184         this.authKey = Keys.of(UrlTransporter.class, repository, "auth");
185         this.proxyAuthKey = Keys.of(UrlTransporter.class, repository, "proxyAuth");
186         if (session.getCache() != null) {
187             this.cacheGetter = k -> {
188                 Boolean ret = (Boolean) session.getCache().get(session, k);
189                 if (ret == null) {
190                     return false;
191                 } else {
192                     return ret;
193                 }
194             };
195             this.cacheSetter = k -> session.getCache().put(session, k, Boolean.TRUE);
196         } else {
197             this.cacheGetter = k -> false;
198             this.cacheSetter = k -> {};
199         }
200     }
201 
202     @Override
203     protected void implPeek(PeekTask task) throws Exception {
204         HttpURLConnection con = perform(METHOD_HEAD, baseUri.resolve(task.getLocation()), null);
205         try {
206             int responseCode = con.getResponseCode();
207             if (HttpURLConnection.HTTP_OK != responseCode) {
208                 throw new HttpTransporterException(responseCode);
209             }
210         } finally {
211             con.disconnect();
212         }
213     }
214 
215     @Override
216     protected void implGet(GetTask task) throws Exception {
217         HttpURLConnection con = perform(METHOD_GET, baseUri.resolve(task.getLocation()), task);
218         try {
219             int responseCode = con.getResponseCode();
220             if (HttpURLConnection.HTTP_OK != responseCode) {
221                 throw new HttpTransporterException(responseCode);
222             }
223             IOSupplier<InputStream> inputStreamSupplier = () -> {
224                 String contentEncoding = con.getHeaderField("Content-Encoding");
225                 if (contentEncoding != null) {
226                     if ("gzip".equalsIgnoreCase(contentEncoding)) {
227                         return new GZIPInputStream(con.getInputStream());
228                     } else if ("deflate".equalsIgnoreCase(contentEncoding)) {
229                         return new InflaterInputStream(con.getInputStream());
230                     }
231                 }
232                 return con.getInputStream();
233             };
234             final Path dataFile = task.getDataPath();
235             if (dataFile == null) {
236                 try (InputStream is = inputStreamSupplier.get()) {
237                     utilGet(task, is, true, con.getContentLengthLong(), false);
238                 }
239             } else {
240                 try (PathProcessor.CollocatedTempFile tempFile = pathProcessor.newTempFile(dataFile)) {
241                     task.setDataPath(tempFile.getPath(), false);
242                     try (InputStream is = inputStreamSupplier.get()) {
243                         utilGet(task, is, true, con.getContentLengthLong(), false);
244                     }
245                     tempFile.move();
246                 } finally {
247                     task.setDataPath(dataFile);
248                 }
249             }
250             if (task.getDataPath() != null) {
251                 long lastModified = con.getLastModified();
252                 if (lastModified != 0) {
253                     pathProcessor.setLastModified(
254                             task.getDataPath(), HttpTransporterUtils.clampRemoteLastModified(lastModified));
255                 }
256             }
257         } finally {
258             con.disconnect();
259         }
260     }
261 
262     @Override
263     protected void implPut(PutTask task) {
264         throw new UnsupportedOperationException("PUT method unsupported");
265     }
266 
267     @Override
268     protected void implClose() {
269         // nothing
270     }
271 
272     private HttpURLConnection perform(String method, URI target, GetTask task) throws IOException {
273         String currAuth = preemptiveAuth ? auth : null;
274         String currProxyAuth = null;
275         if (cacheGetter.apply(authKey)) {
276             currAuth = this.auth;
277         }
278         if (cacheGetter.apply(proxyAuthKey)) {
279             currProxyAuth = this.proxyAuth;
280         }
281         return perform(method, new ArrayList<>(Collections.singletonList(target)), currAuth, currProxyAuth, task);
282     }
283 
284     private HttpURLConnection perform(
285             String method, ArrayList<URI> target, String currAuth, String currProxyAuth, GetTask task)
286             throws IOException {
287         if (target.size() - 1 > maxRedirects) {
288             throw new IOException("Too many redirects");
289         }
290         HttpURLConnection con = (HttpURLConnection) target.get(0).toURL().openConnection(proxy);
291         con.setConnectTimeout(connectTimeout);
292         con.setReadTimeout(requestTimeout);
293         con.setRequestMethod(method);
294         con.setUseCaches(false);
295         con.setInstanceFollowRedirects(false);
296         con.setRequestProperty(HttpConstants.ACCEPT_ENCODING, "gzip,deflate");
297         con.setRequestProperty(HttpConstants.CACHE_CONTROL, "no-cache, no-store");
298         con.setRequestProperty("Pragma", "no-cache");
299         con.setRequestProperty(HttpConstants.USER_AGENT, userAgent);
300         headers.forEach(con::setRequestProperty);
301         if (closeConnection) {
302             con.setRequestProperty("Connection", "close");
303         }
304         if (currAuth != null) {
305             con.setRequestProperty(HEADER_AUTHORIZATION, basicAuthorization(currAuth));
306         }
307         if (currProxyAuth != null) {
308             con.setRequestProperty(HEADER_PROXY_AUTHORIZATION, basicAuthorization(currProxyAuth));
309         }
310         int responseCode = con.getResponseCode();
311         if (responseCode == HttpURLConnection.HTTP_OK) {
312             if (task != null) {
313                 Map<String, String> checksums = checksumExtractor.extractChecksums(con::getHeaderField);
314                 if (checksums != null && !checksums.isEmpty()) {
315                     checksums.forEach(task::setChecksum);
316                 }
317             }
318         } else if (responseCode == HttpURLConnection.HTTP_MOVED_PERM
319                 || responseCode == HttpURLConnection.HTTP_MOVED_TEMP
320                 || responseCode == HttpURLConnection.HTTP_SEE_OTHER
321                 || responseCode == HTTP_STATUS_TEMPORARY_REDIRECT
322                 || responseCode == HTTP_STATUS_PERMANENT_REDIRECT) {
323             if (redirectMode == RedirectMode.NONE) {
324                 con.disconnect();
325                 throw new IOException("Refusing to follow redirects");
326             }
327             final String location = con.getHeaderField(HEADER_LOCATION);
328             if (location == null) {
329                 con.disconnect();
330                 throw new IOException("Redirect response missing Location header");
331             }
332             final URI currentUri;
333             final URI redirectUri;
334             try {
335                 currentUri = URI.create(con.getURL().toString());
336                 redirectUri = currentUri.resolve(location);
337             } catch (IllegalArgumentException e) {
338                 con.disconnect();
339                 throw new IOException("Redirect response has invalid Location header: " + location, e);
340             }
341             // ensure we are HTTP or HTTPS after redirect
342             if (!"http".equalsIgnoreCase(redirectUri.getScheme())
343                     && !"https".equalsIgnoreCase(redirectUri.getScheme())) {
344                 con.disconnect();
345                 throw new IOException("Unsupported redirect protocol: " + redirectUri.getScheme());
346             }
347             String currentAuthority = currentUri.getAuthority();
348             String redirectAuthority = redirectUri.getAuthority();
349             if (currentAuthority == null || !currentAuthority.equalsIgnoreCase(redirectAuthority)) {
350                 if (redirectMode == RedirectMode.SAME_AUTHORITY) {
351                     con.disconnect();
352                     throw new IOException("Refusing to follow redirect to different authority: " + redirectAuthority);
353                 } else {
354                     // reset auth if authority differs after redirect
355                     currAuth = null;
356                 }
357             }
358             if ("https".equalsIgnoreCase(currentUri.getScheme())
359                     && "http".equalsIgnoreCase(redirectUri.getScheme())
360                     && !redirectAllowDowngrade) {
361                 // forbid HTTPS -> HTTP downgrade during redirect
362                 con.disconnect();
363                 throw new IOException("Refusing to downgrade from HTTPS to HTTP protocol");
364             }
365             target.add(0, redirectUri);
366             con.disconnect();
367             return perform(method, target, currAuth, currProxyAuth, task);
368         } else if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED && currAuth == null && this.auth != null) {
369             con.disconnect();
370             return perform(method, target, this.auth, currProxyAuth, task);
371         } else if (responseCode == HttpURLConnection.HTTP_PROXY_AUTH
372                 && currProxyAuth == null
373                 && this.proxyAuth != null) {
374             con.disconnect();
375             return perform(method, target, currAuth, this.proxyAuth, task);
376         }
377         if (currAuth != null) {
378             cacheSetter.accept(authKey);
379         }
380         if (currProxyAuth != null) {
381             cacheSetter.accept(proxyAuthKey);
382         }
383         return con;
384     }
385 
386     private String basicAuthorization(String credentials) {
387         return AUTH_SCHEME_BASIC + " " + Base64.getEncoder().encodeToString(credentials.getBytes(authEncoding));
388     }
389 }