001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.File;
026import java.util.ArrayList;
027import java.util.Collection;
028import java.util.Collections;
029import java.util.HashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.concurrent.Executor;
033
034import org.eclipse.aether.ConfigurationProperties;
035import org.eclipse.aether.RepositoryEvent;
036import org.eclipse.aether.RepositoryEvent.EventType;
037import org.eclipse.aether.RepositorySystemSession;
038import org.eclipse.aether.RequestTrace;
039import org.eclipse.aether.SyncContext;
040import org.eclipse.aether.impl.MetadataResolver;
041import org.eclipse.aether.impl.OfflineController;
042import org.eclipse.aether.impl.RemoteRepositoryFilterManager;
043import org.eclipse.aether.impl.RemoteRepositoryManager;
044import org.eclipse.aether.impl.RepositoryConnectorProvider;
045import org.eclipse.aether.impl.RepositoryEventDispatcher;
046import org.eclipse.aether.impl.UpdateCheck;
047import org.eclipse.aether.impl.UpdateCheckManager;
048import org.eclipse.aether.metadata.Metadata;
049import org.eclipse.aether.repository.ArtifactRepository;
050import org.eclipse.aether.repository.LocalMetadataRegistration;
051import org.eclipse.aether.repository.LocalMetadataRequest;
052import org.eclipse.aether.repository.LocalMetadataResult;
053import org.eclipse.aether.repository.LocalRepository;
054import org.eclipse.aether.repository.LocalRepositoryManager;
055import org.eclipse.aether.repository.RemoteRepository;
056import org.eclipse.aether.repository.RepositoryPolicy;
057import org.eclipse.aether.resolution.MetadataRequest;
058import org.eclipse.aether.resolution.MetadataResult;
059import org.eclipse.aether.spi.connector.MetadataDownload;
060import org.eclipse.aether.spi.connector.RepositoryConnector;
061import org.eclipse.aether.spi.connector.filter.RemoteRepositoryFilter;
062import org.eclipse.aether.spi.synccontext.SyncContextFactory;
063import org.eclipse.aether.transfer.MetadataNotFoundException;
064import org.eclipse.aether.transfer.MetadataTransferException;
065import org.eclipse.aether.transfer.NoRepositoryConnectorException;
066import org.eclipse.aether.transfer.RepositoryOfflineException;
067import org.eclipse.aether.util.concurrency.ExecutorUtils;
068import org.eclipse.aether.util.concurrency.RunnableErrorForwarder;
069
070import static java.util.Objects.requireNonNull;
071
072/**
073 */
074@Singleton
075@Named
076public class DefaultMetadataResolver implements MetadataResolver {
077    private static final String CONFIG_PROPS_PREFIX = ConfigurationProperties.PREFIX_AETHER + "metadataResolver.";
078
079    /**
080     * Number of threads to use in parallel for resolving metadata.
081     *
082     * @since 0.9.0.M4
083     * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
084     * @configurationType {@link java.lang.Integer}
085     * @configurationDefaultValue {@link #DEFAULT_THREADS}
086     */
087    public static final String CONFIG_PROP_THREADS = CONFIG_PROPS_PREFIX + "threads";
088
089    public static final int DEFAULT_THREADS = 4;
090
091    private final RepositoryEventDispatcher repositoryEventDispatcher;
092
093    private final UpdateCheckManager updateCheckManager;
094
095    private final RepositoryConnectorProvider repositoryConnectorProvider;
096
097    private final RemoteRepositoryManager remoteRepositoryManager;
098
099    private final SyncContextFactory syncContextFactory;
100
101    private final OfflineController offlineController;
102
103    private final RemoteRepositoryFilterManager remoteRepositoryFilterManager;
104
105    @Inject
106    public DefaultMetadataResolver(
107            RepositoryEventDispatcher repositoryEventDispatcher,
108            UpdateCheckManager updateCheckManager,
109            RepositoryConnectorProvider repositoryConnectorProvider,
110            RemoteRepositoryManager remoteRepositoryManager,
111            SyncContextFactory syncContextFactory,
112            OfflineController offlineController,
113            RemoteRepositoryFilterManager remoteRepositoryFilterManager) {
114        this.repositoryEventDispatcher =
115                requireNonNull(repositoryEventDispatcher, "repository event dispatcher cannot be null");
116        this.updateCheckManager = requireNonNull(updateCheckManager, "update check manager cannot be null");
117        this.repositoryConnectorProvider =
118                requireNonNull(repositoryConnectorProvider, "repository connector provider cannot be null");
119        this.remoteRepositoryManager =
120                requireNonNull(remoteRepositoryManager, "remote repository provider cannot be null");
121        this.syncContextFactory = requireNonNull(syncContextFactory, "sync context factory cannot be null");
122        this.offlineController = requireNonNull(offlineController, "offline controller cannot be null");
123        this.remoteRepositoryFilterManager =
124                requireNonNull(remoteRepositoryFilterManager, "remote repository filter manager cannot be null");
125    }
126
127    @Override
128    public List<MetadataResult> resolveMetadata(
129            RepositorySystemSession session, Collection<? extends MetadataRequest> requests) {
130        requireNonNull(session, "session cannot be null");
131        requireNonNull(requests, "requests cannot be null");
132        try (SyncContext shared = syncContextFactory.newInstance(session, true);
133                SyncContext exclusive = syncContextFactory.newInstance(session, false)) {
134            Collection<Metadata> metadata = new ArrayList<>(requests.size());
135            for (MetadataRequest request : requests) {
136                metadata.add(request.getMetadata());
137            }
138
139            return resolve(shared, exclusive, metadata, session, requests);
140        }
141    }
142
143    @SuppressWarnings("checkstyle:methodlength")
144    private List<MetadataResult> resolve(
145            SyncContext shared,
146            SyncContext exclusive,
147            Collection<Metadata> subjects,
148            RepositorySystemSession session,
149            Collection<? extends MetadataRequest> requests) {
150        SyncContext current = shared;
151        try {
152            while (true) {
153                current.acquire(null, subjects);
154
155                final List<MetadataResult> results = new ArrayList<>(requests.size());
156                final List<ResolveTask> tasks = new ArrayList<>(requests.size());
157                final Map<File, Long> localLastUpdates = new HashMap<>();
158                final RemoteRepositoryFilter remoteRepositoryFilter =
159                        remoteRepositoryFilterManager.getRemoteRepositoryFilter(session);
160
161                for (MetadataRequest request : requests) {
162                    RequestTrace trace = RequestTrace.newChild(request.getTrace(), request);
163
164                    MetadataResult result = new MetadataResult(request);
165                    results.add(result);
166
167                    Metadata metadata = request.getMetadata();
168                    RemoteRepository repository = request.getRepository();
169
170                    if (repository == null) {
171                        LocalRepository localRepo =
172                                session.getLocalRepositoryManager().getRepository();
173
174                        metadataResolving(session, trace, metadata, localRepo);
175
176                        File localFile = getLocalFile(session, metadata);
177
178                        if (localFile != null) {
179                            metadata = metadata.setFile(localFile);
180                            result.setMetadata(metadata);
181                        } else {
182                            result.setException(new MetadataNotFoundException(metadata, localRepo));
183                        }
184
185                        metadataResolved(session, trace, metadata, localRepo, result.getException());
186                        continue;
187                    }
188
189                    if (remoteRepositoryFilter != null) {
190                        RemoteRepositoryFilter.Result filterResult =
191                                remoteRepositoryFilter.acceptMetadata(repository, metadata);
192                        if (!filterResult.isAccepted()) {
193                            result.setException(
194                                    new MetadataNotFoundException(metadata, repository, filterResult.reasoning()));
195                            continue;
196                        }
197                    }
198
199                    List<RemoteRepository> repositories =
200                            getEnabledSourceRepositories(repository, metadata.getNature());
201
202                    if (repositories.isEmpty()) {
203                        continue;
204                    }
205
206                    metadataResolving(session, trace, metadata, repository);
207                    LocalRepositoryManager lrm = session.getLocalRepositoryManager();
208                    LocalMetadataRequest localRequest =
209                            new LocalMetadataRequest(metadata, repository, request.getRequestContext());
210                    LocalMetadataResult lrmResult = lrm.find(session, localRequest);
211
212                    File metadataFile = lrmResult.getFile();
213
214                    try {
215                        Utils.checkOffline(session, offlineController, repository);
216                    } catch (RepositoryOfflineException e) {
217                        if (metadataFile != null) {
218                            metadata = metadata.setFile(metadataFile);
219                            result.setMetadata(metadata);
220                        } else {
221                            String msg = "Cannot access " + repository.getId() + " (" + repository.getUrl()
222                                    + ") in offline mode and the metadata " + metadata
223                                    + " has not been downloaded from it before";
224                            result.setException(new MetadataNotFoundException(metadata, repository, msg, e));
225                        }
226
227                        metadataResolved(session, trace, metadata, repository, result.getException());
228                        continue;
229                    }
230
231                    Long localLastUpdate = null;
232                    if (request.isFavorLocalRepository()) {
233                        File localFile = getLocalFile(session, metadata);
234                        localLastUpdate = localLastUpdates.get(localFile);
235                        if (localLastUpdate == null) {
236                            localLastUpdate = localFile != null ? localFile.lastModified() : 0;
237                            localLastUpdates.put(localFile, localLastUpdate);
238                        }
239                    }
240
241                    List<UpdateCheck<Metadata, MetadataTransferException>> checks = new ArrayList<>();
242                    Exception exception = null;
243                    for (RemoteRepository repo : repositories) {
244                        RepositoryPolicy policy = getPolicy(session, repo, metadata.getNature());
245
246                        UpdateCheck<Metadata, MetadataTransferException> check = new UpdateCheck<>();
247                        check.setLocalLastUpdated((localLastUpdate != null) ? localLastUpdate : 0);
248                        check.setItem(metadata);
249
250                        // use 'main' installation file for the check (-> use requested repository)
251                        File checkFile = new File(
252                                session.getLocalRepository().getBasedir(),
253                                session.getLocalRepositoryManager()
254                                        .getPathForRemoteMetadata(metadata, repository, request.getRequestContext()));
255                        check.setFile(checkFile);
256                        check.setRepository(repository);
257                        check.setAuthoritativeRepository(repo);
258                        check.setArtifactPolicy(policy.getArtifactUpdatePolicy());
259                        check.setMetadataPolicy(policy.getMetadataUpdatePolicy());
260
261                        if (lrmResult.isStale()) {
262                            checks.add(check);
263                        } else {
264                            updateCheckManager.checkMetadata(session, check);
265                            if (check.isRequired()) {
266                                checks.add(check);
267                            } else if (exception == null) {
268                                exception = check.getException();
269                            }
270                        }
271                    }
272
273                    if (!checks.isEmpty()) {
274                        RepositoryPolicy policy = getPolicy(session, repository, metadata.getNature());
275
276                        // install path may be different from lookup path
277                        File installFile = new File(
278                                session.getLocalRepository().getBasedir(),
279                                session.getLocalRepositoryManager()
280                                        .getPathForRemoteMetadata(
281                                                metadata, request.getRepository(), request.getRequestContext()));
282
283                        ResolveTask task = new ResolveTask(
284                                session, trace, result, installFile, checks, policy.getChecksumPolicy());
285                        tasks.add(task);
286                    } else {
287                        result.setException(exception);
288                        if (metadataFile != null) {
289                            metadata = metadata.setFile(metadataFile);
290                            result.setMetadata(metadata);
291                        }
292                        metadataResolved(session, trace, metadata, repository, result.getException());
293                    }
294                }
295
296                if (!tasks.isEmpty() && current == shared) {
297                    current.close();
298                    current = exclusive;
299                    continue;
300                }
301
302                if (!tasks.isEmpty()) {
303                    int threads = ExecutorUtils.threadCount(session, DEFAULT_THREADS, CONFIG_PROP_THREADS);
304                    Executor executor = ExecutorUtils.executor(
305                            Math.min(tasks.size(), threads), getClass().getSimpleName() + '-');
306                    try {
307                        RunnableErrorForwarder errorForwarder = new RunnableErrorForwarder();
308
309                        for (ResolveTask task : tasks) {
310                            metadataDownloading(
311                                    task.session, task.trace, task.request.getMetadata(), task.request.getRepository());
312
313                            executor.execute(errorForwarder.wrap(task));
314                        }
315
316                        errorForwarder.await();
317
318                        for (ResolveTask task : tasks) {
319                            /*
320                             * NOTE: Touch after registration with local repo to ensure concurrent resolution is not
321                             * rejected with "already updated" via session data when actual update to local repo is
322                             * still pending.
323                             */
324                            for (UpdateCheck<Metadata, MetadataTransferException> check : task.checks) {
325                                updateCheckManager.touchMetadata(task.session, check.setException(task.exception));
326                            }
327
328                            metadataDownloaded(
329                                    session,
330                                    task.trace,
331                                    task.request.getMetadata(),
332                                    task.request.getRepository(),
333                                    task.metadataFile,
334                                    task.exception);
335
336                            task.result.setException(task.exception);
337                        }
338                    } finally {
339                        ExecutorUtils.shutdown(executor);
340                    }
341                    for (ResolveTask task : tasks) {
342                        Metadata metadata = task.request.getMetadata();
343                        // re-lookup metadata for resolve
344                        LocalMetadataRequest localRequest = new LocalMetadataRequest(
345                                metadata, task.request.getRepository(), task.request.getRequestContext());
346                        File metadataFile = session.getLocalRepositoryManager()
347                                .find(session, localRequest)
348                                .getFile();
349                        if (metadataFile != null) {
350                            metadata = metadata.setFile(metadataFile);
351                            task.result.setMetadata(metadata);
352                        }
353                        if (task.result.getException() == null) {
354                            task.result.setUpdated(true);
355                        }
356                        metadataResolved(
357                                session,
358                                task.trace,
359                                metadata,
360                                task.request.getRepository(),
361                                task.result.getException());
362                    }
363                }
364
365                return results;
366            }
367        } finally {
368            current.close();
369        }
370    }
371
372    private File getLocalFile(RepositorySystemSession session, Metadata metadata) {
373        LocalRepositoryManager lrm = session.getLocalRepositoryManager();
374        LocalMetadataResult localResult = lrm.find(session, new LocalMetadataRequest(metadata, null, null));
375        return localResult.getFile();
376    }
377
378    private List<RemoteRepository> getEnabledSourceRepositories(RemoteRepository repository, Metadata.Nature nature) {
379        List<RemoteRepository> repositories = new ArrayList<>();
380
381        if (repository.isRepositoryManager()) {
382            for (RemoteRepository repo : repository.getMirroredRepositories()) {
383                if (isEnabled(repo, nature)) {
384                    repositories.add(repo);
385                }
386            }
387        } else if (isEnabled(repository, nature)) {
388            repositories.add(repository);
389        }
390
391        return repositories;
392    }
393
394    private boolean isEnabled(RemoteRepository repository, Metadata.Nature nature) {
395        if (!Metadata.Nature.SNAPSHOT.equals(nature)
396                && repository.getPolicy(false).isEnabled()) {
397            return true;
398        }
399        return !Metadata.Nature.RELEASE.equals(nature)
400                && repository.getPolicy(true).isEnabled();
401    }
402
403    private RepositoryPolicy getPolicy(
404            RepositorySystemSession session, RemoteRepository repository, Metadata.Nature nature) {
405        boolean releases = !Metadata.Nature.SNAPSHOT.equals(nature);
406        boolean snapshots = !Metadata.Nature.RELEASE.equals(nature);
407        return remoteRepositoryManager.getPolicy(session, repository, releases, snapshots);
408    }
409
410    private void metadataResolving(
411            RepositorySystemSession session, RequestTrace trace, Metadata metadata, ArtifactRepository repository) {
412        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_RESOLVING);
413        event.setTrace(trace);
414        event.setMetadata(metadata);
415        event.setRepository(repository);
416
417        repositoryEventDispatcher.dispatch(event.build());
418    }
419
420    private void metadataResolved(
421            RepositorySystemSession session,
422            RequestTrace trace,
423            Metadata metadata,
424            ArtifactRepository repository,
425            Exception exception) {
426        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_RESOLVED);
427        event.setTrace(trace);
428        event.setMetadata(metadata);
429        event.setRepository(repository);
430        event.setException(exception);
431        event.setFile(metadata.getFile());
432
433        repositoryEventDispatcher.dispatch(event.build());
434    }
435
436    private void metadataDownloading(
437            RepositorySystemSession session, RequestTrace trace, Metadata metadata, ArtifactRepository repository) {
438        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_DOWNLOADING);
439        event.setTrace(trace);
440        event.setMetadata(metadata);
441        event.setRepository(repository);
442
443        repositoryEventDispatcher.dispatch(event.build());
444    }
445
446    private void metadataDownloaded(
447            RepositorySystemSession session,
448            RequestTrace trace,
449            Metadata metadata,
450            ArtifactRepository repository,
451            File file,
452            Exception exception) {
453        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_DOWNLOADED);
454        event.setTrace(trace);
455        event.setMetadata(metadata);
456        event.setRepository(repository);
457        event.setException(exception);
458        event.setFile(file);
459
460        repositoryEventDispatcher.dispatch(event.build());
461    }
462
463    class ResolveTask implements Runnable {
464        final RepositorySystemSession session;
465
466        final RequestTrace trace;
467
468        final MetadataResult result;
469
470        final MetadataRequest request;
471
472        final File metadataFile;
473
474        final String policy;
475
476        final List<UpdateCheck<Metadata, MetadataTransferException>> checks;
477
478        volatile MetadataTransferException exception;
479
480        ResolveTask(
481                RepositorySystemSession session,
482                RequestTrace trace,
483                MetadataResult result,
484                File metadataFile,
485                List<UpdateCheck<Metadata, MetadataTransferException>> checks,
486                String policy) {
487            this.session = session;
488            this.trace = trace;
489            this.result = result;
490            this.request = result.getRequest();
491            this.metadataFile = metadataFile;
492            this.policy = policy;
493            this.checks = checks;
494        }
495
496        public void run() {
497            Metadata metadata = request.getMetadata();
498            RemoteRepository requestRepository = request.getRepository();
499
500            try {
501                List<RemoteRepository> repositories = new ArrayList<>();
502                for (UpdateCheck<Metadata, MetadataTransferException> check : checks) {
503                    repositories.add(check.getAuthoritativeRepository());
504                }
505
506                MetadataDownload download = new MetadataDownload();
507                download.setMetadata(metadata);
508                download.setRequestContext(request.getRequestContext());
509                download.setFile(metadataFile);
510                download.setChecksumPolicy(policy);
511                download.setRepositories(repositories);
512                download.setListener(SafeTransferListener.wrap(session));
513                download.setTrace(trace);
514
515                try (RepositoryConnector connector =
516                        repositoryConnectorProvider.newRepositoryConnector(session, requestRepository)) {
517                    connector.get(null, Collections.singletonList(download));
518                }
519
520                exception = download.getException();
521
522                if (exception == null) {
523
524                    List<String> contexts = Collections.singletonList(request.getRequestContext());
525                    LocalMetadataRegistration registration =
526                            new LocalMetadataRegistration(metadata, requestRepository, contexts);
527
528                    session.getLocalRepositoryManager().add(session, registration);
529                } else if (request.isDeleteLocalCopyIfMissing() && exception instanceof MetadataNotFoundException) {
530                    download.getFile().delete();
531                }
532            } catch (NoRepositoryConnectorException e) {
533                exception = new MetadataTransferException(metadata, requestRepository, e);
534            }
535        }
536    }
537}