001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl.collect.bf;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.Closeable;
026import java.util.ArrayDeque;
027import java.util.ArrayList;
028import java.util.Collections;
029import java.util.LinkedHashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.Queue;
034import java.util.Set;
035import java.util.concurrent.Callable;
036import java.util.concurrent.ConcurrentHashMap;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.ExecutorService;
039import java.util.concurrent.Future;
040import java.util.concurrent.TimeUnit;
041import java.util.concurrent.TimeoutException;
042import java.util.stream.Collectors;
043import java.util.stream.Stream;
044
045import org.eclipse.aether.RepositorySystemSession;
046import org.eclipse.aether.RequestTrace;
047import org.eclipse.aether.artifact.Artifact;
048import org.eclipse.aether.artifact.ArtifactType;
049import org.eclipse.aether.artifact.DefaultArtifact;
050import org.eclipse.aether.collection.CollectRequest;
051import org.eclipse.aether.collection.DependencyManager;
052import org.eclipse.aether.collection.DependencySelector;
053import org.eclipse.aether.collection.DependencyTraverser;
054import org.eclipse.aether.collection.VersionFilter;
055import org.eclipse.aether.graph.DefaultDependencyNode;
056import org.eclipse.aether.graph.Dependency;
057import org.eclipse.aether.graph.DependencyNode;
058import org.eclipse.aether.impl.ArtifactDescriptorReader;
059import org.eclipse.aether.impl.RemoteRepositoryManager;
060import org.eclipse.aether.impl.VersionRangeResolver;
061import org.eclipse.aether.internal.impl.collect.DataPool;
062import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext;
063import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext;
064import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate;
065import org.eclipse.aether.internal.impl.collect.PremanagedDependency;
066import org.eclipse.aether.repository.RemoteRepository;
067import org.eclipse.aether.resolution.ArtifactDescriptorException;
068import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
069import org.eclipse.aether.resolution.ArtifactDescriptorResult;
070import org.eclipse.aether.resolution.VersionRangeRequest;
071import org.eclipse.aether.resolution.VersionRangeResult;
072import org.eclipse.aether.spi.locator.Service;
073import org.eclipse.aether.util.ConfigUtils;
074import org.eclipse.aether.util.artifact.ArtifactIdUtils;
075import org.eclipse.aether.util.concurrency.ExecutorUtils;
076import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
077import org.eclipse.aether.version.Version;
078
079import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find;
080
081/**
082 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector}
083 *
084 * @since 1.8.0
085 */
086@Singleton
087@Named(BfDependencyCollector.NAME)
088public class BfDependencyCollector extends DependencyCollectorDelegate implements Service {
089    public static final String NAME = "bf";
090
091    /**
092     * The key in the repository session's {@link RepositorySystemSession#getConfigProperties()
093     * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode.
094     *
095     * @since 1.8.0
096     */
097    static final String CONFIG_PROP_SKIPPER = "aether.dependencyCollector.bf.skipper";
098
099    /**
100     * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}.
101     *
102     * @since 1.8.0
103     */
104    static final boolean CONFIG_PROP_SKIPPER_DEFAULT = true;
105
106    /**
107     * The count of threads to be used when collecting POMs in parallel, default value 5.
108     *
109     * @since 1.9.0
110     */
111    static final String CONFIG_PROP_THREADS = "aether.dependencyCollector.bf.threads";
112
113    /**
114     * Default ctor for SL.
115     *
116     * @deprecated Will be dropped once SL gone.
117     */
118    @Deprecated
119    public BfDependencyCollector() {
120        // enables default constructor
121    }
122
123    @Inject
124    public BfDependencyCollector(
125            RemoteRepositoryManager remoteRepositoryManager,
126            ArtifactDescriptorReader artifactDescriptorReader,
127            VersionRangeResolver versionRangeResolver) {
128        super(remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver);
129    }
130
131    @SuppressWarnings("checkstyle:parameternumber")
132    @Override
133    protected void doCollectDependencies(
134            RepositorySystemSession session,
135            RequestTrace trace,
136            DataPool pool,
137            DefaultDependencyCollectionContext context,
138            DefaultVersionFilterContext versionContext,
139            CollectRequest request,
140            DependencyNode node,
141            List<RemoteRepository> repositories,
142            List<Dependency> dependencies,
143            List<Dependency> managedDependencies,
144            Results results) {
145        boolean useSkip = ConfigUtils.getBoolean(session, CONFIG_PROP_SKIPPER_DEFAULT, CONFIG_PROP_SKIPPER);
146        int nThreads = ExecutorUtils.threadCount(session, 5, CONFIG_PROP_THREADS, "maven.artifact.threads");
147        logger.debug("Using thread pool with {} threads to resolve descriptors.", nThreads);
148
149        if (useSkip) {
150            logger.debug("Collector skip mode enabled");
151        }
152
153        try (DependencyResolutionSkipper skipper = useSkip
154                        ? DependencyResolutionSkipper.defaultSkipper()
155                        : DependencyResolutionSkipper.neverSkipper();
156                ParallelDescriptorResolver parallelDescriptorResolver = new ParallelDescriptorResolver(nThreads)) {
157            Args args = new Args(session, pool, context, versionContext, request, skipper, parallelDescriptorResolver);
158
159            DependencySelector rootDepSelector = session.getDependencySelector() != null
160                    ? session.getDependencySelector().deriveChildSelector(context)
161                    : null;
162            DependencyManager rootDepManager = session.getDependencyManager() != null
163                    ? session.getDependencyManager().deriveChildManager(context)
164                    : null;
165            DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null
166                    ? session.getDependencyTraverser().deriveChildTraverser(context)
167                    : null;
168            VersionFilter rootVerFilter = session.getVersionFilter() != null
169                    ? session.getVersionFilter().deriveChildFilter(context)
170                    : null;
171
172            List<DependencyNode> parents = Collections.singletonList(node);
173            for (Dependency dependency : dependencies) {
174                RequestTrace childTrace =
175                        collectStepTrace(trace, args.request.getRequestContext(), parents, dependency);
176                DependencyProcessingContext processingContext = new DependencyProcessingContext(
177                        rootDepSelector,
178                        rootDepManager,
179                        rootDepTraverser,
180                        rootVerFilter,
181                        childTrace,
182                        repositories,
183                        managedDependencies,
184                        parents,
185                        dependency,
186                        PremanagedDependency.create(rootDepManager, dependency, false, args.premanagedState));
187                if (!filter(processingContext)) {
188                    processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
189                    resolveArtifactDescriptorAsync(args, processingContext, results);
190                    args.dependencyProcessingQueue.add(processingContext);
191                }
192            }
193
194            while (!args.dependencyProcessingQueue.isEmpty()) {
195                processDependency(
196                        args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(), false);
197            }
198        }
199    }
200
201    @SuppressWarnings("checkstyle:parameternumber")
202    private void processDependency(
203            Args args,
204            Results results,
205            DependencyProcessingContext context,
206            List<Artifact> relocations,
207            boolean disableVersionManagement) {
208        Dependency dependency = context.dependency;
209        PremanagedDependency preManaged = context.premanagedDependency;
210
211        boolean noDescriptor = isLackingDescriptor(dependency.getArtifact());
212        boolean traverse =
213                !noDescriptor && (context.depTraverser == null || context.depTraverser.traverseDependency(dependency));
214
215        Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find(dependency.getArtifact());
216        DescriptorResolutionResult resolutionResult;
217        VersionRangeResult rangeResult;
218        try {
219            resolutionResult = resolutionResultFuture.get();
220            rangeResult = resolutionResult.rangeResult;
221        } catch (Exception e) {
222            results.addException(dependency, e, context.parents);
223            return;
224        }
225
226        Set<Version> versions = resolutionResult.descriptors.keySet();
227        for (Version version : versions) {
228            Artifact originalArtifact = dependency.getArtifact().setVersion(version.toString());
229            Dependency d = dependency.setArtifact(originalArtifact);
230
231            final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get(version);
232            if (descriptorResult != null) {
233                d = d.setArtifact(descriptorResult.getArtifact());
234
235                int cycleEntry = find(context.parents, d.getArtifact());
236                if (cycleEntry >= 0) {
237                    results.addCycle(context.parents, cycleEntry, d);
238                    DependencyNode cycleNode = context.parents.get(cycleEntry);
239                    if (cycleNode.getDependency() != null) {
240                        DefaultDependencyNode child = createDependencyNode(
241                                relocations, preManaged, rangeResult, version, d, descriptorResult, cycleNode);
242                        context.getParent().getChildren().add(child);
243                        continue;
244                    }
245                }
246
247                if (!descriptorResult.getRelocations().isEmpty()) {
248                    boolean disableVersionManagementSubsequently =
249                            originalArtifact.getGroupId().equals(d.getArtifact().getGroupId())
250                                    && originalArtifact
251                                            .getArtifactId()
252                                            .equals(d.getArtifact().getArtifactId());
253
254                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
255                            context.depManager, d, disableVersionManagementSubsequently, args.premanagedState);
256                    DependencyProcessingContext relocatedContext = new DependencyProcessingContext(
257                            context.depSelector,
258                            context.depManager,
259                            context.depTraverser,
260                            context.verFilter,
261                            context.trace,
262                            context.repositories,
263                            descriptorResult.getManagedDependencies(),
264                            context.parents,
265                            d,
266                            premanagedDependency);
267
268                    if (!filter(relocatedContext)) {
269                        relocatedContext.withDependency(premanagedDependency.getManagedDependency());
270                        resolveArtifactDescriptorAsync(args, relocatedContext, results);
271                        processDependency(
272                                args,
273                                results,
274                                relocatedContext,
275                                descriptorResult.getRelocations(),
276                                disableVersionManagementSubsequently);
277                    }
278
279                    return;
280                } else {
281                    d = args.pool.intern(d.setArtifact(args.pool.intern(d.getArtifact())));
282
283                    List<RemoteRepository> repos =
284                            getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
285
286                    DefaultDependencyNode child = createDependencyNode(
287                            relocations,
288                            preManaged,
289                            rangeResult,
290                            version,
291                            d,
292                            descriptorResult.getAliases(),
293                            repos,
294                            args.request.getRequestContext());
295
296                    context.getParent().getChildren().add(child);
297
298                    boolean recurse =
299                            traverse && !descriptorResult.getDependencies().isEmpty();
300                    DependencyProcessingContext parentContext = context.withDependency(d);
301                    if (recurse) {
302                        doRecurse(args, parentContext, descriptorResult, child, results, disableVersionManagement);
303                    } else if (!args.skipper.skipResolution(child, parentContext.parents)) {
304                        List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
305                        parents.addAll(parentContext.parents);
306                        parents.add(child);
307                        args.skipper.cache(child, parents);
308                    }
309                }
310            } else {
311                List<RemoteRepository> repos =
312                        getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
313                DefaultDependencyNode child = createDependencyNode(
314                        relocations,
315                        preManaged,
316                        rangeResult,
317                        version,
318                        d,
319                        null,
320                        repos,
321                        args.request.getRequestContext());
322                context.getParent().getChildren().add(child);
323            }
324        }
325    }
326
327    @SuppressWarnings("checkstyle:parameternumber")
328    private void doRecurse(
329            Args args,
330            DependencyProcessingContext parentContext,
331            ArtifactDescriptorResult descriptorResult,
332            DefaultDependencyNode child,
333            Results results,
334            boolean disableVersionManagement) {
335        DefaultDependencyCollectionContext context = args.collectionContext;
336        context.set(parentContext.dependency, descriptorResult.getManagedDependencies());
337
338        DependencySelector childSelector =
339                parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector(context) : null;
340        DependencyManager childManager =
341                parentContext.depManager != null ? parentContext.depManager.deriveChildManager(context) : null;
342        DependencyTraverser childTraverser =
343                parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser(context) : null;
344        VersionFilter childFilter =
345                parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter(context) : null;
346
347        final List<RemoteRepository> childRepos = args.ignoreRepos
348                ? parentContext.repositories
349                : remoteRepositoryManager.aggregateRepositories(
350                        args.session, parentContext.repositories, descriptorResult.getRepositories(), true);
351
352        Object key = args.pool.toKey(
353                parentContext.dependency.getArtifact(),
354                childRepos,
355                childSelector,
356                childManager,
357                childTraverser,
358                childFilter);
359
360        List<DependencyNode> children = args.pool.getChildren(key);
361        if (children == null) {
362            boolean skipResolution = args.skipper.skipResolution(child, parentContext.parents);
363            if (!skipResolution) {
364                List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
365                parents.addAll(parentContext.parents);
366                parents.add(child);
367                for (Dependency dependency : descriptorResult.getDependencies()) {
368                    RequestTrace childTrace = collectStepTrace(
369                            parentContext.trace, args.request.getRequestContext(), parents, dependency);
370                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
371                            childManager, dependency, disableVersionManagement, args.premanagedState);
372                    DependencyProcessingContext processingContext = new DependencyProcessingContext(
373                            childSelector,
374                            childManager,
375                            childTraverser,
376                            childFilter,
377                            childTrace,
378                            childRepos,
379                            descriptorResult.getManagedDependencies(),
380                            parents,
381                            dependency,
382                            premanagedDependency);
383                    if (!filter(processingContext)) {
384                        // resolve descriptors ahead for managed dependency
385                        processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
386                        resolveArtifactDescriptorAsync(args, processingContext, results);
387                        args.dependencyProcessingQueue.add(processingContext);
388                    }
389                }
390                args.pool.putChildren(key, child.getChildren());
391                args.skipper.cache(child, parents);
392            }
393        } else {
394            child.setChildren(children);
395        }
396    }
397
398    private boolean filter(DependencyProcessingContext context) {
399        return context.depSelector != null && !context.depSelector.selectDependency(context.dependency);
400    }
401
402    private void resolveArtifactDescriptorAsync(Args args, DependencyProcessingContext context, Results results) {
403        Dependency dependency = context.dependency;
404        args.resolver.resolveDescriptors(dependency.getArtifact(), () -> {
405            VersionRangeRequest rangeRequest = createVersionRangeRequest(
406                    args.request.getRequestContext(), context.trace, context.repositories, dependency);
407            VersionRangeResult rangeResult = cachedResolveRangeResult(rangeRequest, args.pool, args.session);
408            List<? extends Version> versions =
409                    filterVersions(dependency, rangeResult, context.verFilter, args.versionContext);
410
411            // resolve newer version first to maximize benefits of skipper
412            Collections.reverse(versions);
413
414            Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>(versions.size());
415            Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream();
416            stream.forEach(version -> Optional.ofNullable(
417                            resolveDescriptorForVersion(args, context, results, dependency, version))
418                    .ifPresent(r -> descriptors.put(version, r)));
419
420            DescriptorResolutionResult resolutionResult =
421                    new DescriptorResolutionResult(dependency.getArtifact(), rangeResult);
422            // keep original sequence
423            versions.forEach(version -> resolutionResult.descriptors.put(version, descriptors.get(version)));
424            // populate for versions in version range
425            resolutionResult.flatten().forEach(dr -> args.resolver.cacheVersionRangeDescriptor(dr.artifact, dr));
426
427            return resolutionResult;
428        });
429    }
430
431    private ArtifactDescriptorResult resolveDescriptorForVersion(
432            Args args, DependencyProcessingContext context, Results results, Dependency dependency, Version version) {
433        Artifact original = dependency.getArtifact();
434        Artifact newArtifact = new DefaultArtifact(
435                original.getGroupId(),
436                original.getArtifactId(),
437                original.getClassifier(),
438                original.getExtension(),
439                version.toString(),
440                original.getProperties(),
441                (ArtifactType) null);
442        Dependency newDependency =
443                new Dependency(newArtifact, dependency.getScope(), dependency.isOptional(), dependency.getExclusions());
444        DependencyProcessingContext newContext = context.copy();
445
446        ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest(
447                args.request.getRequestContext(), context.trace, newContext.repositories, newDependency);
448        return isLackingDescriptor(newArtifact)
449                ? new ArtifactDescriptorResult(descriptorRequest)
450                : resolveCachedArtifactDescriptor(
451                        args.pool, descriptorRequest, args.session, newContext.withDependency(newDependency), results);
452    }
453
454    private ArtifactDescriptorResult resolveCachedArtifactDescriptor(
455            DataPool pool,
456            ArtifactDescriptorRequest descriptorRequest,
457            RepositorySystemSession session,
458            DependencyProcessingContext context,
459            Results results) {
460        Object key = pool.toKey(descriptorRequest);
461        ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest);
462        if (descriptorResult == null) {
463            try {
464                descriptorResult = descriptorReader.readArtifactDescriptor(session, descriptorRequest);
465                pool.putDescriptor(key, descriptorResult);
466            } catch (ArtifactDescriptorException e) {
467                results.addException(context.dependency, e, context.parents);
468                pool.putDescriptor(key, e);
469                return null;
470            }
471
472        } else if (descriptorResult == DataPool.NO_DESCRIPTOR) {
473            return null;
474        }
475
476        return descriptorResult;
477    }
478
479    static class ParallelDescriptorResolver implements Closeable {
480        private final ExecutorService executorService;
481
482        /**
483         * Artifact ID -> Future of DescriptorResolutionResult
484         */
485        private final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>(256);
486
487        ParallelDescriptorResolver(int threads) {
488            this.executorService = ExecutorUtils.threadPool(threads, getClass().getSimpleName() + "-");
489        }
490
491        void resolveDescriptors(Artifact artifact, Callable<DescriptorResolutionResult> callable) {
492            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> this.executorService.submit(callable));
493        }
494
495        void cacheVersionRangeDescriptor(Artifact artifact, DescriptorResolutionResult resolutionResult) {
496            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> new DoneFuture<>(resolutionResult));
497        }
498
499        Future<DescriptorResolutionResult> find(Artifact artifact) {
500            return results.get(ArtifactIdUtils.toId(artifact));
501        }
502
503        @Override
504        public void close() {
505            executorService.shutdown();
506        }
507    }
508
509    static class DoneFuture<V> implements Future<V> {
510        private final V v;
511
512        DoneFuture(V v) {
513            this.v = v;
514        }
515
516        @Override
517        public boolean cancel(boolean mayInterruptIfRunning) {
518            return false;
519        }
520
521        @Override
522        public boolean isCancelled() {
523            return false;
524        }
525
526        @Override
527        public boolean isDone() {
528            return true;
529        }
530
531        @Override
532        public V get() throws InterruptedException, ExecutionException {
533            return v;
534        }
535
536        @Override
537        public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
538            return v;
539        }
540    }
541
542    static class DescriptorResolutionResult {
543        Artifact artifact;
544
545        VersionRangeResult rangeResult;
546
547        Map<Version, ArtifactDescriptorResult> descriptors;
548
549        DescriptorResolutionResult(Artifact artifact, VersionRangeResult rangeResult) {
550            this.artifact = artifact;
551            this.rangeResult = rangeResult;
552            this.descriptors = new LinkedHashMap<>(rangeResult.getVersions().size());
553        }
554
555        DescriptorResolutionResult(
556                VersionRangeResult rangeResult, Version version, ArtifactDescriptorResult descriptor) {
557            // NOTE: In case of A1 -> A2 relocation this happens:
558            // ArtifactDescriptorResult read by ArtifactDescriptorResultReader for A1
559            // will return instance that will have artifact = A2 (as RelocatedArtifact).
560            // So to properly "key" this instance, we need to use "originally requested" A1 instead!
561            // In short:
562            // ArtifactDescriptorRequest.artifact != ArtifactDescriptorResult.artifact WHEN relocation in play
563            // otherwise (no relocation), they are EQUAL.
564            this(descriptor.getRequest().getArtifact(), rangeResult);
565            this.descriptors.put(version, descriptor);
566        }
567
568        List<DescriptorResolutionResult> flatten() {
569            if (descriptors.size() > 1) {
570                return descriptors.entrySet().stream()
571                        .map(e -> new DescriptorResolutionResult(rangeResult, e.getKey(), e.getValue()))
572                        .collect(Collectors.toList());
573            } else {
574                return Collections.emptyList();
575            }
576        }
577    }
578
579    static class Args {
580
581        final RepositorySystemSession session;
582
583        final boolean ignoreRepos;
584
585        final boolean premanagedState;
586
587        final DataPool pool;
588
589        final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>(128);
590
591        final DefaultDependencyCollectionContext collectionContext;
592
593        final DefaultVersionFilterContext versionContext;
594
595        final CollectRequest request;
596
597        final DependencyResolutionSkipper skipper;
598
599        final ParallelDescriptorResolver resolver;
600
601        Args(
602                RepositorySystemSession session,
603                DataPool pool,
604                DefaultDependencyCollectionContext collectionContext,
605                DefaultVersionFilterContext versionContext,
606                CollectRequest request,
607                DependencyResolutionSkipper skipper,
608                ParallelDescriptorResolver resolver) {
609            this.session = session;
610            this.request = request;
611            this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories();
612            this.premanagedState = ConfigUtils.getBoolean(session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE);
613            this.pool = pool;
614            this.collectionContext = collectionContext;
615            this.versionContext = versionContext;
616            this.skipper = skipper;
617            this.resolver = resolver;
618        }
619    }
620}