Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl.collect.bf;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.Closeable;
026import java.util.ArrayDeque;
027import java.util.ArrayList;
028import java.util.Collections;
029import java.util.LinkedHashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.Queue;
034import java.util.Set;
035import java.util.concurrent.Callable;
036import java.util.concurrent.ConcurrentHashMap;
037import java.util.concurrent.ExecutorService;
038import java.util.concurrent.Future;
039import java.util.stream.Collectors;
040import java.util.stream.Stream;
041
042import org.apache.commons.lang3.concurrent.ConcurrentUtils;
043import org.eclipse.aether.RepositorySystemSession;
044import org.eclipse.aether.RequestTrace;
045import org.eclipse.aether.artifact.Artifact;
046import org.eclipse.aether.artifact.ArtifactType;
047import org.eclipse.aether.artifact.DefaultArtifact;
048import org.eclipse.aether.collection.CollectRequest;
049import org.eclipse.aether.collection.DependencyManager;
050import org.eclipse.aether.collection.DependencySelector;
051import org.eclipse.aether.collection.DependencyTraverser;
052import org.eclipse.aether.collection.VersionFilter;
053import org.eclipse.aether.graph.DefaultDependencyNode;
054import org.eclipse.aether.graph.Dependency;
055import org.eclipse.aether.graph.DependencyNode;
056import org.eclipse.aether.impl.ArtifactDescriptorReader;
057import org.eclipse.aether.impl.RemoteRepositoryManager;
058import org.eclipse.aether.impl.VersionRangeResolver;
059import org.eclipse.aether.internal.impl.collect.DataPool;
060import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext;
061import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext;
062import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate;
063import org.eclipse.aether.internal.impl.collect.PremanagedDependency;
064import org.eclipse.aether.repository.RemoteRepository;
065import org.eclipse.aether.resolution.ArtifactDescriptorException;
066import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
067import org.eclipse.aether.resolution.ArtifactDescriptorResult;
068import org.eclipse.aether.resolution.VersionRangeRequest;
069import org.eclipse.aether.resolution.VersionRangeResult;
070import org.eclipse.aether.spi.locator.Service;
071import org.eclipse.aether.util.ConfigUtils;
072import org.eclipse.aether.util.artifact.ArtifactIdUtils;
073import org.eclipse.aether.util.concurrency.ExecutorUtils;
074import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
075import org.eclipse.aether.version.Version;
076
077import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find;
078
079/**
080 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector}
081 *
082 * @since 1.8.0
083 */
084@Singleton
085@Named(BfDependencyCollector.NAME)
086public class BfDependencyCollector extends DependencyCollectorDelegate implements Service {
087    public static final String NAME = "bf";
088
089    /**
090     * The key in the repository session's {@link RepositorySystemSession#getConfigProperties()
091     * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode.
092     *
093     * @since 1.8.0
094     */
095    static final String CONFIG_PROP_SKIPPER = "aether.dependencyCollector.bf.skipper";
096
097    /**
098     * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}.
099     *
100     * @since 1.8.0
101     */
102    static final boolean CONFIG_PROP_SKIPPER_DEFAULT = true;
103
104    /**
105     * The count of threads to be used when collecting POMs in parallel, default value 5.
106     *
107     * @since 1.9.0
108     */
109    static final String CONFIG_PROP_THREADS = "aether.dependencyCollector.bf.threads";
110
111    /**
112     * Default ctor for SL.
113     *
114     * @deprecated Will be dropped once SL gone.
115     */
116    @Deprecated
117    public BfDependencyCollector() {
118        // enables default constructor
119    }
120
121    @Inject
122    BfDependencyCollector(
123            RemoteRepositoryManager remoteRepositoryManager,
124            ArtifactDescriptorReader artifactDescriptorReader,
125            VersionRangeResolver versionRangeResolver) {
126        super(remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver);
127    }
128
129    @SuppressWarnings("checkstyle:parameternumber")
130    @Override
131    protected void doCollectDependencies(
132            RepositorySystemSession session,
133            RequestTrace trace,
134            DataPool pool,
135            DefaultDependencyCollectionContext context,
136            DefaultVersionFilterContext versionContext,
137            CollectRequest request,
138            DependencyNode node,
139            List<RemoteRepository> repositories,
140            List<Dependency> dependencies,
141            List<Dependency> managedDependencies,
142            Results results) {
143        boolean useSkip = ConfigUtils.getBoolean(session, CONFIG_PROP_SKIPPER_DEFAULT, CONFIG_PROP_SKIPPER);
144        int nThreads = ExecutorUtils.threadCount(session, 5, CONFIG_PROP_THREADS, "maven.artifact.threads");
145        logger.debug("Using thread pool with {} threads to resolve descriptors.", nThreads);
146
147        if (useSkip) {
148            logger.debug("Collector skip mode enabled");
149        }
150
151        try (DependencyResolutionSkipper skipper = useSkip
152                        ? DependencyResolutionSkipper.defaultSkipper()
153                        : DependencyResolutionSkipper.neverSkipper();
154                ParallelDescriptorResolver parallelDescriptorResolver = new ParallelDescriptorResolver(nThreads)) {
155            Args args = new Args(session, pool, context, versionContext, request, skipper, parallelDescriptorResolver);
156
157            DependencySelector rootDepSelector = session.getDependencySelector() != null
158                    ? session.getDependencySelector().deriveChildSelector(context)
159                    : null;
160            DependencyManager rootDepManager = session.getDependencyManager() != null
161                    ? session.getDependencyManager().deriveChildManager(context)
162                    : null;
163            DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null
164                    ? session.getDependencyTraverser().deriveChildTraverser(context)
165                    : null;
166            VersionFilter rootVerFilter = session.getVersionFilter() != null
167                    ? session.getVersionFilter().deriveChildFilter(context)
168                    : null;
169
170            List<DependencyNode> parents = Collections.singletonList(node);
171            for (Dependency dependency : dependencies) {
172                RequestTrace childTrace =
173                        collectStepTrace(trace, args.request.getRequestContext(), parents, dependency);
174                DependencyProcessingContext processingContext = new DependencyProcessingContext(
175                        rootDepSelector,
176                        rootDepManager,
177                        rootDepTraverser,
178                        rootVerFilter,
179                        childTrace,
180                        repositories,
181                        managedDependencies,
182                        parents,
183                        dependency,
184                        PremanagedDependency.create(rootDepManager, dependency, false, args.premanagedState));
185                if (!filter(processingContext)) {
186                    processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
187                    resolveArtifactDescriptorAsync(args, processingContext, results);
188                    args.dependencyProcessingQueue.add(processingContext);
189                }
190            }
191
192            while (!args.dependencyProcessingQueue.isEmpty()) {
193                processDependency(
194                        args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(), false);
195            }
196        }
197    }
198
199    @SuppressWarnings("checkstyle:parameternumber")
200    private void processDependency(
201            Args args,
202            Results results,
203            DependencyProcessingContext context,
204            List<Artifact> relocations,
205            boolean disableVersionManagement) {
206        Dependency dependency = context.dependency;
207        PremanagedDependency preManaged = context.premanagedDependency;
208
209        boolean noDescriptor = isLackingDescriptor(dependency.getArtifact());
210        boolean traverse =
211                !noDescriptor && (context.depTraverser == null || context.depTraverser.traverseDependency(dependency));
212
213        Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find(dependency.getArtifact());
214        DescriptorResolutionResult resolutionResult;
215        VersionRangeResult rangeResult;
216        try {
217            resolutionResult = resolutionResultFuture.get();
218            rangeResult = resolutionResult.rangeResult;
219        } catch (Exception e) {
220            results.addException(dependency, e, context.parents);
221            return;
222        }
223
224        Set<Version> versions = resolutionResult.descriptors.keySet();
225        for (Version version : versions) {
226            Artifact originalArtifact = dependency.getArtifact().setVersion(version.toString());
227            Dependency d = dependency.setArtifact(originalArtifact);
228
229            final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get(version);
230            if (descriptorResult != null) {
231                d = d.setArtifact(descriptorResult.getArtifact());
232
233                int cycleEntry = find(context.parents, d.getArtifact());
234                if (cycleEntry >= 0) {
235                    results.addCycle(context.parents, cycleEntry, d);
236                    DependencyNode cycleNode = context.parents.get(cycleEntry);
237                    if (cycleNode.getDependency() != null) {
238                        DefaultDependencyNode child = createDependencyNode(
239                                relocations, preManaged, rangeResult, version, d, descriptorResult, cycleNode);
240                        context.getParent().getChildren().add(child);
241                        continue;
242                    }
243                }
244
245                if (!descriptorResult.getRelocations().isEmpty()) {
246                    boolean disableVersionManagementSubsequently =
247                            originalArtifact.getGroupId().equals(d.getArtifact().getGroupId())
248                                    && originalArtifact
249                                            .getArtifactId()
250                                            .equals(d.getArtifact().getArtifactId());
251
252                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
253                            context.depManager, d, disableVersionManagementSubsequently, args.premanagedState);
254                    DependencyProcessingContext relocatedContext = new DependencyProcessingContext(
255                            context.depSelector,
256                            context.depManager,
257                            context.depTraverser,
258                            context.verFilter,
259                            context.trace,
260                            context.repositories,
261                            descriptorResult.getManagedDependencies(),
262                            context.parents,
263                            d,
264                            premanagedDependency);
265
266                    if (!filter(relocatedContext)) {
267                        relocatedContext.withDependency(premanagedDependency.getManagedDependency());
268                        resolveArtifactDescriptorAsync(args, relocatedContext, results);
269                        processDependency(
270                                args,
271                                results,
272                                relocatedContext,
273                                descriptorResult.getRelocations(),
274                                disableVersionManagementSubsequently);
275                    }
276
277                    return;
278                } else {
279                    d = args.pool.intern(d.setArtifact(args.pool.intern(d.getArtifact())));
280
281                    List<RemoteRepository> repos =
282                            getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
283
284                    DefaultDependencyNode child = createDependencyNode(
285                            relocations,
286                            preManaged,
287                            rangeResult,
288                            version,
289                            d,
290                            descriptorResult.getAliases(),
291                            repos,
292                            args.request.getRequestContext());
293
294                    context.getParent().getChildren().add(child);
295
296                    boolean recurse =
297                            traverse && !descriptorResult.getDependencies().isEmpty();
298                    DependencyProcessingContext parentContext = context.withDependency(d);
299                    if (recurse) {
300                        doRecurse(args, parentContext, descriptorResult, child, results, disableVersionManagement);
301                    } else if (!args.skipper.skipResolution(child, parentContext.parents)) {
302                        List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
303                        parents.addAll(parentContext.parents);
304                        parents.add(child);
305                        args.skipper.cache(child, parents);
306                    }
307                }
308            } else {
309                List<RemoteRepository> repos =
310                        getRemoteRepositories(rangeResult.getRepository(version), context.repositories);
311                DefaultDependencyNode child = createDependencyNode(
312                        relocations,
313                        preManaged,
314                        rangeResult,
315                        version,
316                        d,
317                        null,
318                        repos,
319                        args.request.getRequestContext());
320                context.getParent().getChildren().add(child);
321            }
322        }
323    }
324
325    @SuppressWarnings("checkstyle:parameternumber")
326    private void doRecurse(
327            Args args,
328            DependencyProcessingContext parentContext,
329            ArtifactDescriptorResult descriptorResult,
330            DefaultDependencyNode child,
331            Results results,
332            boolean disableVersionManagement) {
333        DefaultDependencyCollectionContext context = args.collectionContext;
334        context.set(parentContext.dependency, descriptorResult.getManagedDependencies());
335
336        DependencySelector childSelector =
337                parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector(context) : null;
338        DependencyManager childManager =
339                parentContext.depManager != null ? parentContext.depManager.deriveChildManager(context) : null;
340        DependencyTraverser childTraverser =
341                parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser(context) : null;
342        VersionFilter childFilter =
343                parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter(context) : null;
344
345        final List<RemoteRepository> childRepos = args.ignoreRepos
346                ? parentContext.repositories
347                : remoteRepositoryManager.aggregateRepositories(
348                        args.session, parentContext.repositories, descriptorResult.getRepositories(), true);
349
350        Object key = args.pool.toKey(
351                parentContext.dependency.getArtifact(),
352                childRepos,
353                childSelector,
354                childManager,
355                childTraverser,
356                childFilter);
357
358        List<DependencyNode> children = args.pool.getChildren(key);
359        if (children == null) {
360            boolean skipResolution = args.skipper.skipResolution(child, parentContext.parents);
361            if (!skipResolution) {
362                List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1);
363                parents.addAll(parentContext.parents);
364                parents.add(child);
365                for (Dependency dependency : descriptorResult.getDependencies()) {
366                    RequestTrace childTrace = collectStepTrace(
367                            parentContext.trace, args.request.getRequestContext(), parents, dependency);
368                    PremanagedDependency premanagedDependency = PremanagedDependency.create(
369                            childManager, dependency, disableVersionManagement, args.premanagedState);
370                    DependencyProcessingContext processingContext = new DependencyProcessingContext(
371                            childSelector,
372                            childManager,
373                            childTraverser,
374                            childFilter,
375                            childTrace,
376                            childRepos,
377                            descriptorResult.getManagedDependencies(),
378                            parents,
379                            dependency,
380                            premanagedDependency);
381                    if (!filter(processingContext)) {
382                        // resolve descriptors ahead for managed dependency
383                        processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency());
384                        resolveArtifactDescriptorAsync(args, processingContext, results);
385                        args.dependencyProcessingQueue.add(processingContext);
386                    }
387                }
388                args.pool.putChildren(key, child.getChildren());
389                args.skipper.cache(child, parents);
390            }
391        } else {
392            child.setChildren(children);
393        }
394    }
395
396    private boolean filter(DependencyProcessingContext context) {
397        return context.depSelector != null && !context.depSelector.selectDependency(context.dependency);
398    }
399
400    private void resolveArtifactDescriptorAsync(Args args, DependencyProcessingContext context, Results results) {
401        Dependency dependency = context.dependency;
402        args.resolver.resolveDescriptors(dependency.getArtifact(), () -> {
403            VersionRangeRequest rangeRequest = createVersionRangeRequest(
404                    args.request.getRequestContext(), context.trace, context.repositories, dependency);
405            VersionRangeResult rangeResult = cachedResolveRangeResult(rangeRequest, args.pool, args.session);
406            List<? extends Version> versions =
407                    filterVersions(dependency, rangeResult, context.verFilter, args.versionContext);
408
409            // resolve newer version first to maximize benefits of skipper
410            Collections.reverse(versions);
411
412            Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>(versions.size());
413            Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream();
414            stream.forEach(version -> Optional.ofNullable(
415                            resolveDescriptorForVersion(args, context, results, dependency, version))
416                    .ifPresent(r -> descriptors.put(version, r)));
417
418            DescriptorResolutionResult resolutionResult =
419                    new DescriptorResolutionResult(dependency.getArtifact(), rangeResult);
420            // keep original sequence
421            versions.forEach(version -> resolutionResult.descriptors.put(version, descriptors.get(version)));
422            // populate for versions in version range
423            resolutionResult.flatten().forEach(dr -> args.resolver.cacheVersionRangeDescriptor(dr.artifact, dr));
424
425            return resolutionResult;
426        });
427    }
428
429    private ArtifactDescriptorResult resolveDescriptorForVersion(
430            Args args, DependencyProcessingContext context, Results results, Dependency dependency, Version version) {
431        Artifact original = dependency.getArtifact();
432        Artifact newArtifact = new DefaultArtifact(
433                original.getGroupId(),
434                original.getArtifactId(),
435                original.getClassifier(),
436                original.getExtension(),
437                version.toString(),
438                original.getProperties(),
439                (ArtifactType) null);
440        Dependency newDependency =
441                new Dependency(newArtifact, dependency.getScope(), dependency.isOptional(), dependency.getExclusions());
442        DependencyProcessingContext newContext = context.copy();
443
444        ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest(
445                args.request.getRequestContext(), context.trace, newContext.repositories, newDependency);
446        return isLackingDescriptor(newArtifact)
447                ? new ArtifactDescriptorResult(descriptorRequest)
448                : resolveCachedArtifactDescriptor(
449                        args.pool, descriptorRequest, args.session, newContext.withDependency(newDependency), results);
450    }
451
452    private ArtifactDescriptorResult resolveCachedArtifactDescriptor(
453            DataPool pool,
454            ArtifactDescriptorRequest descriptorRequest,
455            RepositorySystemSession session,
456            DependencyProcessingContext context,
457            Results results) {
458        Object key = pool.toKey(descriptorRequest);
459        ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest);
460        if (descriptorResult == null) {
461            try {
462                descriptorResult = descriptorReader.readArtifactDescriptor(session, descriptorRequest);
463                pool.putDescriptor(key, descriptorResult);
464            } catch (ArtifactDescriptorException e) {
465                results.addException(context.dependency, e, context.parents);
466                pool.putDescriptor(key, e);
467                return null;
468            }
469
470        } else if (descriptorResult == DataPool.NO_DESCRIPTOR) {
471            return null;
472        }
473
474        return descriptorResult;
475    }
476
477    static class ParallelDescriptorResolver implements Closeable {
478        private final ExecutorService executorService;
479
480        /**
481         * Artifact ID -> Future of DescriptorResolutionResult
482         */
483        private final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>(256);
484
485        ParallelDescriptorResolver(int threads) {
486            this.executorService = ExecutorUtils.threadPool(threads, getClass().getSimpleName() + "-");
487        }
488
489        void resolveDescriptors(Artifact artifact, Callable<DescriptorResolutionResult> callable) {
490            results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> this.executorService.submit(callable));
491        }
492
493        void cacheVersionRangeDescriptor(Artifact artifact, DescriptorResolutionResult resolutionResult) {
494            results.computeIfAbsent(
495                    ArtifactIdUtils.toId(artifact), key -> ConcurrentUtils.constantFuture(resolutionResult));
496        }
497
498        Future<DescriptorResolutionResult> find(Artifact artifact) {
499            return results.get(ArtifactIdUtils.toId(artifact));
500        }
501
502        @Override
503        public void close() {
504            executorService.shutdown();
505        }
506    }
507
508    static class DescriptorResolutionResult {
509        Artifact artifact;
510
511        VersionRangeResult rangeResult;
512
513        Map<Version, ArtifactDescriptorResult> descriptors;
514
515        DescriptorResolutionResult(Artifact artifact, VersionRangeResult rangeResult) {
516            this.artifact = artifact;
517            this.rangeResult = rangeResult;
518            this.descriptors = new LinkedHashMap<>(rangeResult.getVersions().size());
519        }
520
521        DescriptorResolutionResult(
522                VersionRangeResult rangeResult, Version version, ArtifactDescriptorResult descriptor) {
523            this(descriptor.getArtifact(), rangeResult);
524            this.descriptors.put(version, descriptor);
525        }
526
527        List<DescriptorResolutionResult> flatten() {
528            if (descriptors.size() > 1) {
529                return descriptors.entrySet().stream()
530                        .map(e -> new DescriptorResolutionResult(rangeResult, e.getKey(), e.getValue()))
531                        .collect(Collectors.toList());
532            } else {
533                return Collections.emptyList();
534            }
535        }
536    }
537
538    static class Args {
539
540        final RepositorySystemSession session;
541
542        final boolean ignoreRepos;
543
544        final boolean premanagedState;
545
546        final DataPool pool;
547
548        final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>(128);
549
550        final DefaultDependencyCollectionContext collectionContext;
551
552        final DefaultVersionFilterContext versionContext;
553
554        final CollectRequest request;
555
556        final DependencyResolutionSkipper skipper;
557
558        final ParallelDescriptorResolver resolver;
559
560        Args(
561                RepositorySystemSession session,
562                DataPool pool,
563                DefaultDependencyCollectionContext collectionContext,
564                DefaultVersionFilterContext versionContext,
565                CollectRequest request,
566                DependencyResolutionSkipper skipper,
567                ParallelDescriptorResolver resolver) {
568            this.session = session;
569            this.request = request;
570            this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories();
571            this.premanagedState = ConfigUtils.getBoolean(session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE);
572            this.pool = pool;
573            this.collectionContext = collectionContext;
574            this.versionContext = versionContext;
575            this.skipper = skipper;
576            this.resolver = resolver;
577        }
578    }
579}