001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.IOException;
026import java.io.UncheckedIOException;
027import java.nio.file.Files;
028import java.nio.file.Path;
029import java.util.ArrayList;
030import java.util.Collection;
031import java.util.Collections;
032import java.util.HashMap;
033import java.util.List;
034import java.util.Map;
035
036import org.eclipse.aether.ConfigurationProperties;
037import org.eclipse.aether.RepositoryEvent;
038import org.eclipse.aether.RepositoryEvent.EventType;
039import org.eclipse.aether.RepositorySystemSession;
040import org.eclipse.aether.RequestTrace;
041import org.eclipse.aether.SyncContext;
042import org.eclipse.aether.impl.MetadataResolver;
043import org.eclipse.aether.impl.OfflineController;
044import org.eclipse.aether.impl.RemoteRepositoryFilterManager;
045import org.eclipse.aether.impl.RemoteRepositoryManager;
046import org.eclipse.aether.impl.RepositoryConnectorProvider;
047import org.eclipse.aether.impl.RepositoryEventDispatcher;
048import org.eclipse.aether.impl.UpdateCheck;
049import org.eclipse.aether.impl.UpdateCheckManager;
050import org.eclipse.aether.metadata.Metadata;
051import org.eclipse.aether.repository.ArtifactRepository;
052import org.eclipse.aether.repository.LocalMetadataRegistration;
053import org.eclipse.aether.repository.LocalMetadataRequest;
054import org.eclipse.aether.repository.LocalMetadataResult;
055import org.eclipse.aether.repository.LocalRepository;
056import org.eclipse.aether.repository.LocalRepositoryManager;
057import org.eclipse.aether.repository.RemoteRepository;
058import org.eclipse.aether.repository.RepositoryPolicy;
059import org.eclipse.aether.resolution.MetadataRequest;
060import org.eclipse.aether.resolution.MetadataResult;
061import org.eclipse.aether.spi.connector.MetadataDownload;
062import org.eclipse.aether.spi.connector.RepositoryConnector;
063import org.eclipse.aether.spi.connector.filter.RemoteRepositoryFilter;
064import org.eclipse.aether.spi.io.PathProcessor;
065import org.eclipse.aether.spi.synccontext.SyncContextFactory;
066import org.eclipse.aether.transfer.MetadataNotFoundException;
067import org.eclipse.aether.transfer.MetadataTransferException;
068import org.eclipse.aether.transfer.NoRepositoryConnectorException;
069import org.eclipse.aether.transfer.RepositoryOfflineException;
070import org.eclipse.aether.util.ConfigUtils;
071import org.eclipse.aether.util.concurrency.RunnableErrorForwarder;
072import org.eclipse.aether.util.concurrency.SmartExecutor;
073import org.eclipse.aether.util.concurrency.SmartExecutorUtils;
074
075import static java.util.Objects.requireNonNull;
076
077/**
078 */
079@Singleton
080@Named
081public class DefaultMetadataResolver implements MetadataResolver {
082    private static final String CONFIG_PROPS_PREFIX = ConfigurationProperties.PREFIX_AETHER + "metadataResolver.";
083
084    /**
085     * Number of threads to use in parallel for resolving metadata.
086     *
087     * @since 0.9.0.M4
088     * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
089     * @configurationType {@link java.lang.Integer}
090     * @configurationDefaultValue {@link #DEFAULT_THREADS}
091     */
092    public static final String CONFIG_PROP_THREADS = CONFIG_PROPS_PREFIX + "threads";
093
094    public static final int DEFAULT_THREADS = 4;
095
096    private final RepositoryEventDispatcher repositoryEventDispatcher;
097
098    private final UpdateCheckManager updateCheckManager;
099
100    private final RepositoryConnectorProvider repositoryConnectorProvider;
101
102    private final RemoteRepositoryManager remoteRepositoryManager;
103
104    private final SyncContextFactory syncContextFactory;
105
106    private final OfflineController offlineController;
107
108    private final RemoteRepositoryFilterManager remoteRepositoryFilterManager;
109
110    private final PathProcessor pathProcessor;
111
112    @SuppressWarnings("checkstyle:parameternumber")
113    @Inject
114    public DefaultMetadataResolver(
115            RepositoryEventDispatcher repositoryEventDispatcher,
116            UpdateCheckManager updateCheckManager,
117            RepositoryConnectorProvider repositoryConnectorProvider,
118            RemoteRepositoryManager remoteRepositoryManager,
119            SyncContextFactory syncContextFactory,
120            OfflineController offlineController,
121            RemoteRepositoryFilterManager remoteRepositoryFilterManager,
122            PathProcessor pathProcessor) {
123        this.repositoryEventDispatcher =
124                requireNonNull(repositoryEventDispatcher, "repository event dispatcher cannot be null");
125        this.updateCheckManager = requireNonNull(updateCheckManager, "update check manager cannot be null");
126        this.repositoryConnectorProvider =
127                requireNonNull(repositoryConnectorProvider, "repository connector provider cannot be null");
128        this.remoteRepositoryManager =
129                requireNonNull(remoteRepositoryManager, "remote repository provider cannot be null");
130        this.syncContextFactory = requireNonNull(syncContextFactory, "sync context factory cannot be null");
131        this.offlineController = requireNonNull(offlineController, "offline controller cannot be null");
132        this.remoteRepositoryFilterManager =
133                requireNonNull(remoteRepositoryFilterManager, "remote repository filter manager cannot be null");
134        this.pathProcessor = requireNonNull(pathProcessor, "path processor cannot be null");
135    }
136
137    @Override
138    public List<MetadataResult> resolveMetadata(
139            RepositorySystemSession session, Collection<? extends MetadataRequest> requests) {
140        requireNonNull(session, "session cannot be null");
141        requireNonNull(requests, "requests cannot be null");
142        Collection<Metadata> metadata = new ArrayList<>(requests.size());
143        for (MetadataRequest request : requests) {
144            metadata.add(request.getMetadata());
145        }
146
147        try (SyncContext shared = new CloseOnceSyncContext(syncContextFactory.newInstance(session, true));
148                SyncContext exclusive = new CloseOnceSyncContext(syncContextFactory.newInstance(session, false))) {
149            return resolve(shared, exclusive, metadata, session, requests);
150        }
151    }
152
153    @SuppressWarnings("checkstyle:methodlength")
154    private List<MetadataResult> resolve(
155            SyncContext shared,
156            SyncContext exclusive,
157            Collection<Metadata> subjects,
158            RepositorySystemSession session,
159            Collection<? extends MetadataRequest> requests) {
160        SyncContext current = shared;
161        try {
162            while (true) {
163                current.acquire(null, subjects);
164
165                final List<MetadataResult> results = new ArrayList<>(requests.size());
166                final List<ResolveTask> tasks = new ArrayList<>(requests.size());
167                final Map<Path, Long> localLastUpdates = new HashMap<>();
168                final RemoteRepositoryFilter remoteRepositoryFilter =
169                        remoteRepositoryFilterManager.getRemoteRepositoryFilter(session);
170
171                for (MetadataRequest request : requests) {
172                    RequestTrace trace = RequestTrace.newChild(request.getTrace(), request);
173
174                    MetadataResult result = new MetadataResult(request);
175                    results.add(result);
176
177                    Metadata metadata = request.getMetadata();
178                    RemoteRepository repository = request.getRepository();
179
180                    if (repository == null) {
181                        LocalRepository localRepo =
182                                session.getLocalRepositoryManager().getRepository();
183
184                        metadataResolving(session, trace, metadata, localRepo);
185
186                        Path localFile = getLocalFile(session, metadata);
187
188                        if (localFile != null) {
189                            metadata = metadata.setPath(localFile);
190                            result.setMetadata(metadata);
191                        } else {
192                            result.setException(new MetadataNotFoundException(metadata, localRepo));
193                        }
194
195                        metadataResolved(session, trace, metadata, localRepo, result.getException());
196                        continue;
197                    }
198
199                    if (remoteRepositoryFilter != null) {
200                        RemoteRepositoryFilter.Result filterResult =
201                                remoteRepositoryFilter.acceptMetadata(repository, metadata);
202                        if (!filterResult.isAccepted()) {
203                            result.setException(
204                                    new MetadataNotFoundException(metadata, repository, filterResult.reasoning()));
205                            continue;
206                        }
207                    }
208
209                    List<RemoteRepository> repositories =
210                            getEnabledSourceRepositories(repository, metadata.getNature());
211
212                    if (repositories.isEmpty()) {
213                        continue;
214                    }
215
216                    metadataResolving(session, trace, metadata, repository);
217                    LocalRepositoryManager lrm = session.getLocalRepositoryManager();
218                    LocalMetadataRequest localRequest =
219                            new LocalMetadataRequest(metadata, repository, request.getRequestContext());
220                    LocalMetadataResult lrmResult = lrm.find(session, localRequest);
221
222                    Path metadataPath = lrmResult.getPath();
223
224                    try {
225                        Utils.checkOffline(session, offlineController, repository);
226                    } catch (RepositoryOfflineException e) {
227                        if (metadataPath != null) {
228                            metadata = metadata.setPath(metadataPath);
229                            result.setMetadata(metadata);
230                        } else {
231                            String msg = "Cannot access " + repository.getId() + " (" + repository.getUrl()
232                                    + ") in offline mode and the metadata " + metadata
233                                    + " has not been downloaded from it before";
234                            result.setException(new MetadataNotFoundException(metadata, repository, msg, e));
235                        }
236
237                        metadataResolved(session, trace, metadata, repository, result.getException());
238                        continue;
239                    }
240
241                    Long localLastUpdate = null;
242                    if (request.isFavorLocalRepository()) {
243                        Path localPath = getLocalFile(session, metadata);
244                        localLastUpdate = localLastUpdates.get(localPath);
245                        if (localLastUpdate == null) {
246                            localLastUpdate = localPath != null ? pathProcessor.lastModified(localPath, 0L) : 0L;
247                            localLastUpdates.put(localPath, localLastUpdate);
248                        }
249                    }
250
251                    List<UpdateCheck<Metadata, MetadataTransferException>> checks = new ArrayList<>();
252                    Exception exception = null;
253                    for (RemoteRepository repo : repositories) {
254                        RepositoryPolicy policy = getPolicy(session, repo, metadata.getNature());
255
256                        UpdateCheck<Metadata, MetadataTransferException> check = new UpdateCheck<>();
257                        check.setLocalLastUpdated((localLastUpdate != null) ? localLastUpdate : 0);
258                        check.setItem(metadata);
259
260                        // use 'main' installation file for the check (-> use requested repository)
261                        Path checkPath = session.getLocalRepositoryManager()
262                                .getAbsolutePathForRemoteMetadata(metadata, repository, request.getRequestContext());
263                        check.setPath(checkPath);
264                        check.setRepository(repository);
265                        check.setAuthoritativeRepository(repo);
266                        check.setArtifactPolicy(policy.getArtifactUpdatePolicy());
267                        check.setMetadataPolicy(policy.getMetadataUpdatePolicy());
268
269                        if (lrmResult.isStale()) {
270                            checks.add(check);
271                        } else {
272                            updateCheckManager.checkMetadata(session, check);
273                            if (check.isRequired()) {
274                                checks.add(check);
275                            } else if (exception == null) {
276                                exception = check.getException();
277                            }
278                        }
279                    }
280
281                    if (!checks.isEmpty()) {
282                        RepositoryPolicy policy = getPolicy(session, repository, metadata.getNature());
283
284                        // install path may be different from lookup path
285                        Path installPath = session.getLocalRepositoryManager()
286                                .getAbsolutePathForRemoteMetadata(
287                                        metadata, request.getRepository(), request.getRequestContext());
288
289                        ResolveTask task = new ResolveTask(
290                                session, trace, result, installPath, checks, policy.getChecksumPolicy());
291                        tasks.add(task);
292                    } else {
293                        result.setException(exception);
294                        if (metadataPath != null) {
295                            metadata = metadata.setPath(metadataPath);
296                            result.setMetadata(metadata);
297                        }
298                        metadataResolved(session, trace, metadata, repository, result.getException());
299                    }
300                }
301
302                if (!tasks.isEmpty() && current == shared) {
303                    SyncContext sharedContext = current;
304                    current = exclusive;
305                    sharedContext.close();
306                    continue;
307                }
308
309                if (!tasks.isEmpty()) {
310                    try (SmartExecutor executor = SmartExecutorUtils.smartExecutor(
311                            session,
312                            tasks.size(), // we DON'T want global executor; call can be recursive (pool depletion)
313                            ConfigUtils.getInteger(session, DEFAULT_THREADS, CONFIG_PROP_THREADS),
314                            getClass().getSimpleName() + "-")) {
315                        RunnableErrorForwarder errorForwarder = new RunnableErrorForwarder();
316
317                        for (ResolveTask task : tasks) {
318                            metadataDownloading(
319                                    task.session, task.trace, task.request.getMetadata(), task.request.getRepository());
320
321                            executor.submit(errorForwarder.wrap(task));
322                        }
323
324                        errorForwarder.await();
325
326                        for (ResolveTask task : tasks) {
327                            /*
328                             * NOTE: Touch after registration with local repo to ensure concurrent resolution is not
329                             * rejected with "already updated" via session data when actual update to local repo is
330                             * still pending.
331                             */
332                            for (UpdateCheck<Metadata, MetadataTransferException> check : task.checks) {
333                                updateCheckManager.touchMetadata(task.session, check.setException(task.exception));
334                            }
335
336                            metadataDownloaded(
337                                    session,
338                                    task.trace,
339                                    task.request.getMetadata(),
340                                    task.request.getRepository(),
341                                    task.metadataPath,
342                                    task.exception);
343
344                            task.result.setException(task.exception);
345                        }
346                    }
347                    for (ResolveTask task : tasks) {
348                        Metadata metadata = task.request.getMetadata();
349                        // re-lookup metadata for resolve
350                        LocalMetadataRequest localRequest = new LocalMetadataRequest(
351                                metadata, task.request.getRepository(), task.request.getRequestContext());
352                        Path metadataPath = session.getLocalRepositoryManager()
353                                .find(session, localRequest)
354                                .getPath();
355                        if (metadataPath != null) {
356                            metadata = metadata.setPath(metadataPath);
357                            task.result.setMetadata(metadata);
358                        }
359                        if (task.result.getException() == null) {
360                            task.result.setUpdated(true);
361                        }
362                        metadataResolved(
363                                session,
364                                task.trace,
365                                metadata,
366                                task.request.getRepository(),
367                                task.result.getException());
368                    }
369                }
370
371                return results;
372            }
373        } finally {
374            try {
375                current.close();
376            } finally {
377                if (current == shared) {
378                    exclusive.close();
379                }
380            }
381        }
382    }
383
384    private Path getLocalFile(RepositorySystemSession session, Metadata metadata) {
385        LocalRepositoryManager lrm = session.getLocalRepositoryManager();
386        LocalMetadataResult localResult = lrm.find(session, new LocalMetadataRequest(metadata, null, null));
387        return localResult.getPath();
388    }
389
390    private List<RemoteRepository> getEnabledSourceRepositories(RemoteRepository repository, Metadata.Nature nature) {
391        List<RemoteRepository> repositories = new ArrayList<>();
392
393        if (repository.isRepositoryManager()) {
394            for (RemoteRepository repo : repository.getMirroredRepositories()) {
395                if (isEnabled(repo, nature)) {
396                    repositories.add(repo);
397                }
398            }
399        } else if (isEnabled(repository, nature)) {
400            repositories.add(repository);
401        }
402
403        return repositories;
404    }
405
406    private boolean isEnabled(RemoteRepository repository, Metadata.Nature nature) {
407        if (!Metadata.Nature.SNAPSHOT.equals(nature)
408                && repository.getPolicy(false).isEnabled()) {
409            return true;
410        }
411        return !Metadata.Nature.RELEASE.equals(nature)
412                && repository.getPolicy(true).isEnabled();
413    }
414
415    private RepositoryPolicy getPolicy(
416            RepositorySystemSession session, RemoteRepository repository, Metadata.Nature nature) {
417        boolean releases = !Metadata.Nature.SNAPSHOT.equals(nature);
418        boolean snapshots = !Metadata.Nature.RELEASE.equals(nature);
419        return remoteRepositoryManager.getPolicy(session, repository, releases, snapshots);
420    }
421
422    private void metadataResolving(
423            RepositorySystemSession session, RequestTrace trace, Metadata metadata, ArtifactRepository repository) {
424        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_RESOLVING);
425        event.setTrace(trace);
426        event.setMetadata(metadata);
427        event.setRepository(repository);
428
429        repositoryEventDispatcher.dispatch(event.build());
430    }
431
432    private void metadataResolved(
433            RepositorySystemSession session,
434            RequestTrace trace,
435            Metadata metadata,
436            ArtifactRepository repository,
437            Exception exception) {
438        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_RESOLVED);
439        event.setTrace(trace);
440        event.setMetadata(metadata);
441        event.setRepository(repository);
442        event.setException(exception);
443        event.setPath(metadata.getPath());
444
445        repositoryEventDispatcher.dispatch(event.build());
446    }
447
448    private void metadataDownloading(
449            RepositorySystemSession session, RequestTrace trace, Metadata metadata, ArtifactRepository repository) {
450        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_DOWNLOADING);
451        event.setTrace(trace);
452        event.setMetadata(metadata);
453        event.setRepository(repository);
454
455        repositoryEventDispatcher.dispatch(event.build());
456    }
457
458    private void metadataDownloaded(
459            RepositorySystemSession session,
460            RequestTrace trace,
461            Metadata metadata,
462            ArtifactRepository repository,
463            Path path,
464            Exception exception) {
465        RepositoryEvent.Builder event = new RepositoryEvent.Builder(session, EventType.METADATA_DOWNLOADED);
466        event.setTrace(trace);
467        event.setMetadata(metadata);
468        event.setRepository(repository);
469        event.setException(exception);
470        event.setPath(path);
471
472        repositoryEventDispatcher.dispatch(event.build());
473    }
474
475    class ResolveTask implements Runnable {
476        final RepositorySystemSession session;
477
478        final RequestTrace trace;
479
480        final MetadataResult result;
481
482        final MetadataRequest request;
483
484        final Path metadataPath;
485
486        final String policy;
487
488        final List<UpdateCheck<Metadata, MetadataTransferException>> checks;
489
490        volatile MetadataTransferException exception;
491
492        ResolveTask(
493                RepositorySystemSession session,
494                RequestTrace trace,
495                MetadataResult result,
496                Path metadataPath,
497                List<UpdateCheck<Metadata, MetadataTransferException>> checks,
498                String policy) {
499            this.session = session;
500            this.trace = trace;
501            this.result = result;
502            this.request = result.getRequest();
503            this.metadataPath = metadataPath;
504            this.policy = policy;
505            this.checks = checks;
506        }
507
508        public void run() {
509            Metadata metadata = request.getMetadata();
510            RemoteRepository requestRepository = request.getRepository();
511
512            try {
513                List<RemoteRepository> repositories = new ArrayList<>();
514                for (UpdateCheck<Metadata, MetadataTransferException> check : checks) {
515                    repositories.add(check.getAuthoritativeRepository());
516                }
517
518                MetadataDownload download = new MetadataDownload();
519                download.setMetadata(metadata);
520                download.setRequestContext(request.getRequestContext());
521                download.setPath(metadataPath);
522                download.setChecksumPolicy(policy);
523                download.setRepositories(repositories);
524                download.setListener(SafeTransferListener.wrap(session));
525                download.setTrace(trace);
526
527                try (RepositoryConnector connector =
528                        repositoryConnectorProvider.newRepositoryConnector(session, requestRepository)) {
529                    connector.get(null, Collections.singletonList(download));
530                }
531
532                exception = download.getException();
533
534                if (exception == null) {
535
536                    List<String> contexts = Collections.singletonList(request.getRequestContext());
537                    LocalMetadataRegistration registration =
538                            new LocalMetadataRegistration(metadata, requestRepository, contexts);
539
540                    session.getLocalRepositoryManager().add(session, registration);
541                } else if (request.isDeleteLocalCopyIfMissing() && exception instanceof MetadataNotFoundException) {
542                    try {
543                        Files.deleteIfExists(download.getPath());
544                    } catch (IOException e) {
545                        throw new UncheckedIOException(e);
546                    }
547                }
548            } catch (NoRepositoryConnectorException e) {
549                exception = new MetadataTransferException(metadata, requestRepository, e);
550            }
551        }
552    }
553}