001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.IOException;
026import java.io.UncheckedIOException;
027import java.nio.file.Files;
028import java.nio.file.Path;
029import java.util.ArrayList;
030import java.util.Collection;
031import java.util.Collections;
032import java.util.HashMap;
033import java.util.List;
034import java.util.Map;
035import java.util.concurrent.Executor;
036
037import org.eclipse.aether.ConfigurationProperties;
038import org.eclipse.aether.RepositoryEvent;
039import org.eclipse.aether.RepositoryEvent.EventType;
040import org.eclipse.aether.RepositorySystemSession;
041import org.eclipse.aether.RequestTrace;
042import org.eclipse.aether.SyncContext;
043import org.eclipse.aether.impl.MetadataResolver;
044import org.eclipse.aether.impl.OfflineController;
045import org.eclipse.aether.impl.RemoteRepositoryFilterManager;
046import org.eclipse.aether.impl.RemoteRepositoryManager;
047import org.eclipse.aether.impl.RepositoryConnectorProvider;
048import org.eclipse.aether.impl.RepositoryEventDispatcher;
049import org.eclipse.aether.impl.UpdateCheck;
050import org.eclipse.aether.impl.UpdateCheckManager;
051import org.eclipse.aether.metadata.Metadata;
052import org.eclipse.aether.repository.ArtifactRepository;
053import org.eclipse.aether.repository.LocalMetadataRegistration;
054import org.eclipse.aether.repository.LocalMetadataRequest;
055import org.eclipse.aether.repository.LocalMetadataResult;
056import org.eclipse.aether.repository.LocalRepository;
057import org.eclipse.aether.repository.LocalRepositoryManager;
058import org.eclipse.aether.repository.RemoteRepository;
059import org.eclipse.aether.repository.RepositoryPolicy;
060import org.eclipse.aether.resolution.MetadataRequest;
061import org.eclipse.aether.resolution.MetadataResult;
062import org.eclipse.aether.spi.connector.MetadataDownload;
063import org.eclipse.aether.spi.connector.RepositoryConnector;
064import org.eclipse.aether.spi.connector.filter.RemoteRepositoryFilter;
065import org.eclipse.aether.spi.io.PathProcessor;
066import org.eclipse.aether.spi.synccontext.SyncContextFactory;
067import org.eclipse.aether.transfer.MetadataNotFoundException;
068import org.eclipse.aether.transfer.MetadataTransferException;
069import org.eclipse.aether.transfer.NoRepositoryConnectorException;
070import org.eclipse.aether.transfer.RepositoryOfflineException;
071import org.eclipse.aether.util.concurrency.ExecutorUtils;
072import org.eclipse.aether.util.concurrency.RunnableErrorForwarder;
073
074import static java.util.Objects.requireNonNull;
075
076/**
077 */
078@Singleton
079@Named
080public class DefaultMetadataResolver implements MetadataResolver {
081    private static final String CONFIG_PROPS_PREFIX = ConfigurationProperties.PREFIX_AETHER + "metadataResolver.";
082
083    /**
084     * Number of threads to use in parallel for resolving metadata.
085     *
086     * @since 0.9.0.M4
087     * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
088     * @configurationType {@link java.lang.Integer}
089     * @configurationDefaultValue {@link #DEFAULT_THREADS}
090     */
091    public static final String CONFIG_PROP_THREADS = CONFIG_PROPS_PREFIX + "threads";
092
093    public static final int DEFAULT_THREADS = 4;
094
095    private final RepositoryEventDispatcher repositoryEventDispatcher;
096
097    private final UpdateCheckManager updateCheckManager;
098
099    private final RepositoryConnectorProvider repositoryConnectorProvider;
100
101    private final RemoteRepositoryManager remoteRepositoryManager;
102
103    private final SyncContextFactory syncContextFactory;
104
105    private final OfflineController offlineController;
106
107    private final RemoteRepositoryFilterManager remoteRepositoryFilterManager;
108
109    private final PathProcessor pathProcessor;
110
111    @SuppressWarnings("checkstyle:parameternumber")
112    @Inject
113    public DefaultMetadataResolver(
114            RepositoryEventDispatcher repositoryEventDispatcher,
115            UpdateCheckManager updateCheckManager,
116            RepositoryConnectorProvider repositoryConnectorProvider,
117            RemoteRepositoryManager remoteRepositoryManager,
118            SyncContextFactory syncContextFactory,
119            OfflineController offlineController,
120            RemoteRepositoryFilterManager remoteRepositoryFilterManager,
121            PathProcessor pathProcessor) {
122        this.repositoryEventDispatcher =
123                requireNonNull(repositoryEventDispatcher, "repository event dispatcher cannot be null");
124        this.updateCheckManager = requireNonNull(updateCheckManager, "update check manager cannot be null");
125        this.repositoryConnectorProvider =
126                requireNonNull(repositoryConnectorProvider, "repository connector provider cannot be null");
127        this.remoteRepositoryManager =
128                requireNonNull(remoteRepositoryManager, "remote repository provider cannot be null");
129        this.syncContextFactory = requireNonNull(syncContextFactory, "sync context factory cannot be null");
130        this.offlineController = requireNonNull(offlineController, "offline controller cannot be null");
131        this.remoteRepositoryFilterManager =
132                requireNonNull(remoteRepositoryFilterManager, "remote repository filter manager cannot be null");
133        this.pathProcessor = requireNonNull(pathProcessor, "path processor cannot be null");
134    }
135
136    @Override
137    public List<MetadataResult> resolveMetadata(
138            RepositorySystemSession session, Collection<? extends MetadataRequest> requests) {
139        requireNonNull(session, "session cannot be null");
140        requireNonNull(requests, "requests cannot be null");
141        try (SyncContext shared = syncContextFactory.newInstance(session, true);
142                SyncContext exclusive = syncContextFactory.newInstance(session, false)) {
143            Collection<Metadata> metadata = new ArrayList<>(requests.size());
144            for (MetadataRequest request : requests) {
145                metadata.add(request.getMetadata());
146            }
147
148            return resolve(shared, exclusive, metadata, session, requests);
149        }
150    }
151
152    @SuppressWarnings("checkstyle:methodlength")
153    private List<MetadataResult> resolve(
154            SyncContext shared,
155            SyncContext exclusive,
156            Collection<Metadata> subjects,
157            RepositorySystemSession session,
158            Collection<? extends MetadataRequest> requests) {
159        SyncContext current = shared;
160        try {
161            while (true) {
162                current.acquire(null, subjects);
163
164                final List<MetadataResult> results = new ArrayList<>(requests.size());
165                final List<ResolveTask> tasks = new ArrayList<>(requests.size());
166                final Map<Path, Long> localLastUpdates = new HashMap<>();
167                final RemoteRepositoryFilter remoteRepositoryFilter =
168                        remoteRepositoryFilterManager.getRemoteRepositoryFilter(session);
169
170                for (MetadataRequest request : requests) {
171                    RequestTrace trace = RequestTrace.newChild(request.getTrace(), request);
172
173                    MetadataResult result = new MetadataResult(request);
174                    results.add(result);
175
176                    Metadata metadata = request.getMetadata();
177                    RemoteRepository repository = request.getRepository();
178
179                    if (repository == null) {
180                        LocalRepository localRepo =
181                                session.getLocalRepositoryManager().getRepository();
182
183                        metadataResolving(session, trace, metadata, localRepo);
184
185                        Path localFile = getLocalFile(session, metadata);
186
187                        if (localFile != null) {
188                            metadata = metadata.setPath(localFile);
189                            result.setMetadata(metadata);
190                        } else {
191                            result.setException(new MetadataNotFoundException(metadata, localRepo));
192                        }
193
194                        metadataResolved(session, trace, metadata, localRepo, result.getException());
195                        continue;
196                    }
197
198                    if (remoteRepositoryFilter != null) {
199                        RemoteRepositoryFilter.Result filterResult =
200                                remoteRepositoryFilter.acceptMetadata(repository, metadata);
201                        if (!filterResult.isAccepted()) {
202                            result.setException(
203                                    new MetadataNotFoundException(metadata, repository, filterResult.reasoning()));
204                            continue;
205                        }
206                    }
207
208                    List<RemoteRepository> repositories =
209                            getEnabledSourceRepositories(repository, metadata.getNature());
210
211                    if (repositories.isEmpty()) {
212                        continue;
213                    }
214
215                    metadataResolving(session, trace, metadata, repository);
216                    LocalRepositoryManager lrm = session.getLocalRepositoryManager();
217                    LocalMetadataRequest localRequest =
218                            new LocalMetadataRequest(metadata, repository, request.getRequestContext());
219                    LocalMetadataResult lrmResult = lrm.find(session, localRequest);
220
221                    Path metadataPath = lrmResult.getPath();
222
223                    try {
224                        Utils.checkOffline(session, offlineController, repository);
225                    } catch (RepositoryOfflineException e) {
226                        if (metadataPath != null) {
227                            metadata = metadata.setPath(metadataPath);
228                            result.setMetadata(metadata);
229                        } else {
230                            String msg = "Cannot access " + repository.getId() + " (" + repository.getUrl()
231                                    + ") in offline mode and the metadata " + metadata
232                                    + " has not been downloaded from it before";
233                            result.setException(new MetadataNotFoundException(metadata, repository, msg, e));
234                        }
235
236                        metadataResolved(session, trace, metadata, repository, result.getException());
237                        continue;
238                    }
239
240                    Long localLastUpdate = null;
241                    if (request.isFavorLocalRepository()) {
242                        Path localPath = getLocalFile(session, metadata);
243                        localLastUpdate = localLastUpdates.get(localPath);
244                        if (localLastUpdate == null) {
245                            localLastUpdate = localPath != null ? pathProcessor.lastModified(localPath, 0L) : 0L;
246                            localLastUpdates.put(localPath, localLastUpdate);
247                        }
248                    }
249
250                    List<UpdateCheck<Metadata, MetadataTransferException>> checks = new ArrayList<>();
251                    Exception exception = null;
252                    for (RemoteRepository repo : repositories) {
253                        RepositoryPolicy policy = getPolicy(session, repo, metadata.getNature());
254
255                        UpdateCheck<Metadata, MetadataTransferException> check = new UpdateCheck<>();
256                        check.setLocalLastUpdated((localLastUpdate != null) ? localLastUpdate : 0);
257                        check.setItem(metadata);
258
259                        // use 'main' installation file for the check (-> use requested repository)
260                        Path checkPath = session.getLocalRepository()
261                                .getBasePath()
262                                .resolve(session.getLocalRepositoryManager()
263                                        .getPathForRemoteMetadata(metadata, repository, request.getRequestContext()));
264                        check.setPath(checkPath);
265                        check.setRepository(repository);
266                        check.setAuthoritativeRepository(repo);
267                        check.setArtifactPolicy(policy.getArtifactUpdatePolicy());
268                        check.setMetadataPolicy(policy.getMetadataUpdatePolicy());
269
270                        if (lrmResult.isStale()) {
271                            checks.add(check);
272                        } else {
273                            updateCheckManager.checkMetadata(session, check);
274                            if (check.isRequired()) {
275                                checks.add(check);
276                            } else if (exception == null) {
277                                exception = check.getException();
278                            }
279                        }
280                    }
281
282                    if (!checks.isEmpty()) {
283                        RepositoryPolicy policy = getPolicy(session, repository, metadata.getNature());
284
285                        // install path may be different from lookup path
286                        Path installPath = session.getLocalRepository()
287                                .getBasePath()
288                                .resolve(session.getLocalRepositoryManager()
289                                        .getPathForRemoteMetadata(
290                                                metadata, request.getRepository(), request.getRequestContext()));
291
292                        ResolveTask task = new ResolveTask(
293                                session, trace, result, installPath, checks, policy.getChecksumPolicy());
294                        tasks.add(task);
295                    } else {
296                        result.setException(exception);
297                        if (metadataPath != null) {
298                            metadata = metadata.setPath(metadataPath);
299                            result.setMetadata(metadata);
300                        }
301                        metadataResolved(session, trace, metadata, repository, result.getException());
302                    }
303                }
304
305                if (!tasks.isEmpty() && current == shared) {
306                    current.close();
307                    current = exclusive;
308                    continue;
309                }
310
311                if (!tasks.isEmpty()) {
312                    int threads = ExecutorUtils.threadCount(session, DEFAULT_THREADS, CONFIG_PROP_THREADS);
313                    Executor executor = ExecutorUtils.executor(
314                            Math.min(tasks.size(), threads), getClass().getSimpleName() + '-');
315                    try {
316                        RunnableErrorForwarder errorForwarder = new RunnableErrorForwarder();
317
318                        for (ResolveTask task : tasks) {
319                            metadataDownloading(
320                                    task.session, task.trace, task.request.getMetadata(), task.request.getRepository());
321
322                            executor.execute(errorForwarder.wrap(task));
323                        }
324
325                        errorForwarder.await();
326
327                        for (ResolveTask task : tasks) {
328                            /*
329                             * NOTE: Touch after registration with local repo to ensure concurrent resolution is not
330                             * rejected with "already updated" via session data when actual update to local repo is
331                             * still pending.
332                             */
333                            for (UpdateCheck<Metadata, MetadataTransferException> check : task.checks) {
334                                updateCheckManager.touchMetadata(task.session, check.setException(task.exception));
335                            }
336
337                            metadataDownloaded(
338                                    session,
339                                    task.trace,
340                                    task.request.getMetadata(),
341                                    task.request.getRepository(),
342                                    task.metadataPath,
343                                    task.exception);
344
345                            task.result.setException(task.exception);
346                        }
347                    } finally {
348                        ExecutorUtils.shutdown(executor);
349                    }
350                    for (ResolveTask task : tasks) {
351                        Metadata metadata = task.request.getMetadata();
352                        // re-lookup metadata for resolve
353                        LocalMetadataRequest localRequest = new LocalMetadataRequest(
354                                metadata, task.request.getRepository(), task.request.getRequestContext());
355                        Path metadataPath = session.getLocalRepositoryManager()
356                                .find(session, localRequest)
357                                .getPath();
358                        if (metadataPath != null) {
359                            metadata = metadata.setPath(metadataPath);
360                            task.result.setMetadata(metadata);
361                        }
362                        if (task.result.getException() == null) {
363                            task.result.setUpdated(true);
364                        }
365                        metadataResolved(
366                                session,
367                                task.trace,
368                                metadata,
369                                task.request.getRepository(),
370                                task.result.getException());
371                    }
372                }
373
374                return results;
375            }
376        } finally {
377            current.close();
378        }
379    }
380
381    private Path getLocalFile(RepositorySystemSession session, Metadata metadata) {
382        LocalRepositoryManager lrm = session.getLocalRepositoryManager();
383        LocalMetadataResult localResult = lrm.find(session, new LocalMetadataRequest(metadata, null, null));
384        return localResult.getPath();
385    }
386
387    private List<RemoteRepository> getEnabledSourceRepositories(RemoteRepository repository, Metadata.Nature nature) {
388        List<RemoteRepository> repositories = new ArrayList<>();
389
390        if (repository.isRepositoryManager()) {
391            for (RemoteRepository repo : repository.getMirroredRepositories()) {
392                if (isEnabled(repo, nature)) {
393                    repositories.add(repo);
394                }
395            }
396        } else if (isEnabled(repository, nature)) {
397            repositories.add(repository);
398        }
399
400        return repositories;
401    }
402
403    private boolean isEnabled(RemoteRepository repository, Metadata.Nature nature) {
404        if (!Metadata.Nature.SNAPSHOT.equals(nature)
405                && repository.getPolicy(false).isEnabled()) {
406            return true;
407        }
408        return !Metadata.Nature.RELEASE.equals(nature)
409                && repository.getPolicy(true).isEnabled();
410    }
411
412    private RepositoryPolicy getPolicy(
413            RepositorySystemSession session, RemoteRepository repository, Metadata.Nature nature) {
414        boolean releases = !Metadata.Nature.SNAPSHOT.equals(nature);
415        boolean snapshots = !Metadata.Nature.RELEASE.equals(nature);
416        return remoteRepositoryManager.getPolicy(session, repository, releases, snapshots);
417    }
418
419    private void metadataResolving(
420            RepositorySystemSession session, RequestTrace trace, Metadata metadata, ArtifactRepository repository) {
421        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_RESOLVING);
422        event.setTrace(trace);
423        event.setMetadata(metadata);
424        event.setRepository(repository);
425
426        repositoryEventDispatcher.dispatch(event.build());
427    }
428
429    private void metadataResolved(
430            RepositorySystemSession session,
431            RequestTrace trace,
432            Metadata metadata,
433            ArtifactRepository repository,
434            Exception exception) {
435        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_RESOLVED);
436        event.setTrace(trace);
437        event.setMetadata(metadata);
438        event.setRepository(repository);
439        event.setException(exception);
440        event.setPath(metadata.getPath());
441
442        repositoryEventDispatcher.dispatch(event.build());
443    }
444
445    private void metadataDownloading(
446            RepositorySystemSession session, RequestTrace trace, Metadata metadata, ArtifactRepository repository) {
447        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_DOWNLOADING);
448        event.setTrace(trace);
449        event.setMetadata(metadata);
450        event.setRepository(repository);
451
452        repositoryEventDispatcher.dispatch(event.build());
453    }
454
455    private void metadataDownloaded(
456            RepositorySystemSession session,
457            RequestTrace trace,
458            Metadata metadata,
459            ArtifactRepository repository,
460            Path path,
461            Exception exception) {
462        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_DOWNLOADED);
463        event.setTrace(trace);
464        event.setMetadata(metadata);
465        event.setRepository(repository);
466        event.setException(exception);
467        event.setPath(path);
468
469        repositoryEventDispatcher.dispatch(event.build());
470    }
471
472    class ResolveTask implements Runnable {
473        final RepositorySystemSession session;
474
475        final RequestTrace trace;
476
477        final MetadataResult result;
478
479        final MetadataRequest request;
480
481        final Path metadataPath;
482
483        final String policy;
484
485        final List<UpdateCheck<Metadata, MetadataTransferException>> checks;
486
487        volatile MetadataTransferException exception;
488
489        ResolveTask(
490                RepositorySystemSession session,
491                RequestTrace trace,
492                MetadataResult result,
493                Path metadataPath,
494                List<UpdateCheck<Metadata, MetadataTransferException>> checks,
495                String policy) {
496            this.session = session;
497            this.trace = trace;
498            this.result = result;
499            this.request = result.getRequest();
500            this.metadataPath = metadataPath;
501            this.policy = policy;
502            this.checks = checks;
503        }
504
505        public void run() {
506            Metadata metadata = request.getMetadata();
507            RemoteRepository requestRepository = request.getRepository();
508
509            try {
510                List<RemoteRepository> repositories = new ArrayList<>();
511                for (UpdateCheck<Metadata, MetadataTransferException> check : checks) {
512                    repositories.add(check.getAuthoritativeRepository());
513                }
514
515                MetadataDownload download = new MetadataDownload();
516                download.setMetadata(metadata);
517                download.setRequestContext(request.getRequestContext());
518                download.setPath(metadataPath);
519                download.setChecksumPolicy(policy);
520                download.setRepositories(repositories);
521                download.setListener(SafeTransferListener.wrap(session));
522                download.setTrace(trace);
523
524                try (RepositoryConnector connector =
525                        repositoryConnectorProvider.newRepositoryConnector(session, requestRepository)) {
526                    connector.get(null, Collections.singletonList(download));
527                }
528
529                exception = download.getException();
530
531                if (exception == null) {
532
533                    List<String> contexts = Collections.singletonList(request.getRequestContext());
534                    LocalMetadataRegistration registration =
535                            new LocalMetadataRegistration(metadata, requestRepository, contexts);
536
537                    session.getLocalRepositoryManager().add(session, registration);
538                } else if (request.isDeleteLocalCopyIfMissing() && exception instanceof MetadataNotFoundException) {
539                    try {
540                        Files.deleteIfExists(download.getPath());
541                    } catch (IOException e) {
542                        throw new UncheckedIOException(e);
543                    }
544                }
545            } catch (NoRepositoryConnectorException e) {
546                exception = new MetadataTransferException(metadata, requestRepository, e);
547            }
548        }
549    }
550}