Source code

001package org.eclipse.aether.internal.impl.collect.bf;
002
003/*
004 * Licensed to the Apache Software Foundation (ASF) under one
005 * or more contributor license agreements.  See the NOTICE file
006 * distributed with this work for additional information
007 * regarding copyright ownership.  The ASF licenses this file
008 * to you under the Apache License, Version 2.0 (the
009 * "License"); you may not use this file except in compliance
010 * with the License.  You may obtain a copy of the License at
011 *
012 *  http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing,
015 * software distributed under the License is distributed on an
016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
017 * KIND, either express or implied.  See the License for the
018 * specific language governing permissions and limitations
019 * under the License.
020 */
021
022import javax.inject.Inject;
023import javax.inject.Named;
024import javax.inject.Singleton;
025
026import java.util.ArrayDeque;
027import java.util.ArrayList;
028import java.util.Collections;
029import java.util.LinkedHashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Optional;
033import java.util.Queue;
034import java.util.Set;
035import java.util.concurrent.Callable;
036import java.util.concurrent.ConcurrentHashMap;
037import java.util.concurrent.ExecutorService;
038import java.util.concurrent.Future;
039import java.util.concurrent.LinkedBlockingQueue;
040import java.util.concurrent.ThreadPoolExecutor;
041import java.util.concurrent.TimeUnit;
042import java.util.stream.Collectors;
043import java.util.stream.Stream;
044
045import org.apache.commons.lang3.concurrent.ConcurrentUtils;
046import org.eclipse.aether.RepositorySystemSession;
047import org.eclipse.aether.RequestTrace;
048import org.eclipse.aether.artifact.Artifact;
049import org.eclipse.aether.artifact.ArtifactType;
050import org.eclipse.aether.artifact.DefaultArtifact;
051import org.eclipse.aether.collection.CollectRequest;
052import org.eclipse.aether.collection.DependencyManager;
053import org.eclipse.aether.collection.DependencySelector;
054import org.eclipse.aether.collection.DependencyTraverser;
055import org.eclipse.aether.collection.VersionFilter;
056import org.eclipse.aether.graph.DefaultDependencyNode;
057import org.eclipse.aether.graph.Dependency;
058import org.eclipse.aether.graph.DependencyNode;
059import org.eclipse.aether.impl.ArtifactDescriptorReader;
060import org.eclipse.aether.impl.RemoteRepositoryManager;
061import org.eclipse.aether.impl.VersionRangeResolver;
062import org.eclipse.aether.internal.impl.collect.DataPool;
063import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext;
064import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext;
065import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate;
066import org.eclipse.aether.internal.impl.collect.PremanagedDependency;
067import org.eclipse.aether.repository.RemoteRepository;
068import org.eclipse.aether.resolution.ArtifactDescriptorException;
069import org.eclipse.aether.resolution.ArtifactDescriptorRequest;
070import org.eclipse.aether.resolution.ArtifactDescriptorResult;
071import org.eclipse.aether.resolution.VersionRangeRequest;
072import org.eclipse.aether.resolution.VersionRangeResult;
073import org.eclipse.aether.spi.locator.Service;
074import org.eclipse.aether.util.ConfigUtils;
075import org.eclipse.aether.util.artifact.ArtifactIdUtils;
076import org.eclipse.aether.util.concurrency.WorkerThreadFactory;
077import org.eclipse.aether.util.graph.manager.DependencyManagerUtils;
078import org.eclipse.aether.version.Version;
079import org.slf4j.Logger;
080import org.slf4j.LoggerFactory;
081
082import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find;
083
084/**
085 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector}
086 *
087 * @since 1.8.0
088 */
089@Singleton
090@Named( BfDependencyCollector.NAME )
091public class BfDependencyCollector
092    extends DependencyCollectorDelegate implements Service
093{
094    public static final String NAME = "bf";
095
096    /**
097     * The key in the repository session's {@link RepositorySystemSession#getConfigProperties()
098     * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode.
099     *
100     * @since 1.8.0
101     */
102    static final String CONFIG_PROP_SKIPPER = "aether.dependencyCollector.bf.skipper";
103
104    /**
105     * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}.
106     *
107     * @since 1.8.0
108     */
109    static final boolean CONFIG_PROP_SKIPPER_DEFAULT = true;
110
111    /**
112     * The count of threads to be used when collecting POMs in parallel, default value 5.
113     *
114     * @since 1.9.0
115     */
116    static final String CONFIG_PROP_THREADS = "aether.dependencyCollector.bf.threads";
117
118    /**
119     * Default ctor for SL.
120     *
121     * @deprecated Will be dropped once SL gone.
122     */
123    @Deprecated
124    public BfDependencyCollector()
125    {
126        // enables default constructor
127    }
128
129    @Inject
130    BfDependencyCollector( RemoteRepositoryManager remoteRepositoryManager,
131                           ArtifactDescriptorReader artifactDescriptorReader,
132                           VersionRangeResolver versionRangeResolver )
133    {
134        super( remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver );
135    }
136
137    @SuppressWarnings( "checkstyle:parameternumber" )
138    @Override
139    protected void doCollectDependencies( RepositorySystemSession session, RequestTrace trace, DataPool pool,
140                                          DefaultDependencyCollectionContext context,
141                                          DefaultVersionFilterContext versionContext,
142                                          CollectRequest request, DependencyNode node,
143                                          List<RemoteRepository> repositories, List<Dependency> dependencies,
144                                          List<Dependency> managedDependencies, Results results )
145    {
146        boolean useSkip = ConfigUtils.getBoolean(
147                session, CONFIG_PROP_SKIPPER_DEFAULT, CONFIG_PROP_SKIPPER
148        );
149        if ( useSkip )
150        {
151            logger.debug( "Collector skip mode enabled" );
152        }
153
154        Args args =
155                new Args( session, pool, context, versionContext, request,
156                        useSkip ? DependencyResolutionSkipper.defaultSkipper()
157                                : DependencyResolutionSkipper.neverSkipper(),
158                        new ParallelDescriptorResolver( session ) );
159
160        DependencySelector rootDepSelector = session.getDependencySelector() != null
161                ? session.getDependencySelector().deriveChildSelector( context ) : null;
162        DependencyManager rootDepManager = session.getDependencyManager() != null
163                ? session.getDependencyManager().deriveChildManager( context ) : null;
164        DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null
165                ? session.getDependencyTraverser().deriveChildTraverser( context ) : null;
166        VersionFilter rootVerFilter = session.getVersionFilter() != null
167                ? session.getVersionFilter().deriveChildFilter( context ) : null;
168
169        List<DependencyNode> parents = Collections.singletonList( node );
170        for ( Dependency dependency : dependencies )
171        {
172            RequestTrace childTrace = collectStepTrace( trace, args.request.getRequestContext(), parents,
173                    dependency );
174            DependencyProcessingContext processingContext =
175                    new DependencyProcessingContext( rootDepSelector, rootDepManager, rootDepTraverser,
176                            rootVerFilter, childTrace, repositories, managedDependencies, parents, dependency,
177                            PremanagedDependency.create( rootDepManager, dependency,
178                                    false, args.premanagedState ) );
179            if ( !filter( processingContext ) )
180            {
181                processingContext.withDependency( processingContext.premanagedDependency.getManagedDependency() );
182                resolveArtifactDescriptorAsync( args, processingContext, results );
183                args.dependencyProcessingQueue.add( processingContext );
184            }
185        }
186
187        while ( !args.dependencyProcessingQueue.isEmpty() )
188        {
189            processDependency( args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(),
190                    false );
191        }
192
193        args.resolver.shutdown();
194        args.skipper.report();
195    }
196
197    @SuppressWarnings( "checkstyle:parameternumber" )
198    private void processDependency( Args args, Results results,
199                                    DependencyProcessingContext context, List<Artifact> relocations,
200                                    boolean disableVersionManagement )
201    {
202        Dependency dependency = context.dependency;
203        PremanagedDependency preManaged = context.premanagedDependency;
204
205        boolean noDescriptor = isLackingDescriptor( dependency.getArtifact() );
206        boolean traverse =
207                !noDescriptor && ( context.depTraverser == null || context.depTraverser.traverseDependency(
208                        dependency ) );
209
210        Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find( dependency.getArtifact() );
211        DescriptorResolutionResult resolutionResult;
212        VersionRangeResult rangeResult;
213        try
214        {
215            resolutionResult = resolutionResultFuture.get();
216            rangeResult = resolutionResult.rangeResult;
217        }
218        catch ( Exception e )
219        {
220            results.addException( dependency, e, context.parents );
221            return;
222        }
223
224        Set<Version> versions = resolutionResult.descriptors.keySet();
225        for ( Version version : versions )
226        {
227            Artifact originalArtifact = dependency.getArtifact().setVersion( version.toString() );
228            Dependency d = dependency.setArtifact( originalArtifact );
229
230            final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get( version );
231            if ( descriptorResult != null )
232            {
233                d = d.setArtifact( descriptorResult.getArtifact() );
234
235                int cycleEntry = find( context.parents, d.getArtifact() );
236                if ( cycleEntry >= 0 )
237                {
238                    results.addCycle( context.parents, cycleEntry, d );
239                    DependencyNode cycleNode = context.parents.get( cycleEntry );
240                    if ( cycleNode.getDependency() != null )
241                    {
242                        DefaultDependencyNode child =
243                                createDependencyNode( relocations, preManaged, rangeResult, version, d,
244                                        descriptorResult, cycleNode );
245                        context.getParent().getChildren().add( child );
246                        continue;
247                    }
248                }
249
250                if ( !descriptorResult.getRelocations().isEmpty() )
251                {
252                    boolean disableVersionManagementSubsequently =
253                        originalArtifact.getGroupId().equals( d.getArtifact().getGroupId() )
254                            && originalArtifact.getArtifactId().equals( d.getArtifact().getArtifactId() );
255
256                    PremanagedDependency premanagedDependency =
257                            PremanagedDependency.create( context.depManager, d, disableVersionManagementSubsequently,
258                                    args.premanagedState );
259                    DependencyProcessingContext relocatedContext =
260                            new DependencyProcessingContext( context.depSelector, context.depManager,
261                                    context.depTraverser, context.verFilter,
262                                    context.trace, context.repositories, descriptorResult.getManagedDependencies(),
263                                    context.parents,
264                                    d, premanagedDependency );
265
266                    if ( !filter( relocatedContext ) )
267                    {
268                        relocatedContext.withDependency( premanagedDependency.getManagedDependency() );
269                        resolveArtifactDescriptorAsync( args, relocatedContext, results );
270                        processDependency( args, results, relocatedContext, descriptorResult.getRelocations(),
271                                disableVersionManagementSubsequently );
272                    }
273
274                    return;
275                }
276                else
277                {
278                    d = args.pool.intern( d.setArtifact( args.pool.intern( d.getArtifact() ) ) );
279
280                    List<RemoteRepository> repos =
281                        getRemoteRepositories( rangeResult.getRepository( version ), context.repositories );
282
283                    DefaultDependencyNode child =
284                        createDependencyNode( relocations, preManaged, rangeResult, version, d,
285                                              descriptorResult.getAliases(), repos, args.request.getRequestContext() );
286
287                    context.getParent().getChildren().add( child );
288
289                    boolean recurse = traverse && !descriptorResult.getDependencies().isEmpty();
290                    DependencyProcessingContext parentContext = context.withDependency( d );
291                    if ( recurse )
292                    {
293                        doRecurse( args, parentContext, descriptorResult, child, results,
294                                disableVersionManagement );
295                    }
296                    else if ( !args.skipper.skipResolution( child, parentContext.parents ) )
297                    {
298                        List<DependencyNode> parents = new ArrayList<>( parentContext.parents.size() + 1 );
299                        parents.addAll( parentContext.parents );
300                        parents.add( child );
301                        args.skipper.cache( child, parents );
302                    }
303                }
304            }
305            else
306            {
307                List<RemoteRepository> repos =
308                    getRemoteRepositories( rangeResult.getRepository( version ), context.repositories );
309                DefaultDependencyNode child =
310                    createDependencyNode( relocations, preManaged, rangeResult, version, d, null, repos,
311                                          args.request.getRequestContext() );
312                context.getParent().getChildren().add( child );
313            }
314        }
315    }
316
317    @SuppressWarnings( "checkstyle:parameternumber" )
318    private void doRecurse( Args args, DependencyProcessingContext parentContext,
319                            ArtifactDescriptorResult descriptorResult, DefaultDependencyNode child, Results results,
320                            boolean disableVersionManagement )
321    {
322        DefaultDependencyCollectionContext context = args.collectionContext;
323        context.set( parentContext.dependency, descriptorResult.getManagedDependencies() );
324
325        DependencySelector childSelector =
326                parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector( context ) : null;
327        DependencyManager childManager =
328                parentContext.depManager != null ? parentContext.depManager.deriveChildManager( context ) : null;
329        DependencyTraverser childTraverser =
330                parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser( context ) : null;
331        VersionFilter childFilter =
332                parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter( context ) : null;
333
334        final List<RemoteRepository> childRepos =
335                args.ignoreRepos
336                        ? parentContext.repositories
337                        : remoteRepositoryManager.aggregateRepositories( args.session, parentContext.repositories,
338                        descriptorResult.getRepositories(), true );
339
340        Object key =
341                args.pool.toKey( parentContext.dependency.getArtifact(), childRepos, childSelector, childManager,
342                        childTraverser, childFilter );
343
344        List<DependencyNode> children = args.pool.getChildren( key );
345        if ( children == null )
346        {
347            boolean skipResolution = args.skipper.skipResolution( child, parentContext.parents );
348            if ( !skipResolution )
349            {
350                List<DependencyNode> parents = new ArrayList<>( parentContext.parents.size() + 1 );
351                parents.addAll( parentContext.parents );
352                parents.add( child );
353                for ( Dependency dependency : descriptorResult.getDependencies() )
354                {
355                    RequestTrace childTrace =
356                            collectStepTrace( parentContext.trace, args.request.getRequestContext(), parents,
357                                    dependency );
358                    PremanagedDependency premanagedDependency =
359                            PremanagedDependency.create( childManager, dependency, disableVersionManagement,
360                                    args.premanagedState );
361                    DependencyProcessingContext processingContext =
362                            new DependencyProcessingContext( childSelector, childManager, childTraverser, childFilter,
363                                    childTrace, childRepos, descriptorResult.getManagedDependencies(), parents,
364                                    dependency, premanagedDependency );
365                    if ( !filter( processingContext ) )
366                    {
367                        //resolve descriptors ahead for managed dependency
368                        processingContext.withDependency(
369                                processingContext.premanagedDependency.getManagedDependency() );
370                        resolveArtifactDescriptorAsync( args, processingContext, results );
371                        args.dependencyProcessingQueue.add( processingContext );
372                    }
373                }
374                args.pool.putChildren( key, child.getChildren() );
375                args.skipper.cache( child, parents );
376            }
377        }
378        else
379        {
380            child.setChildren( children );
381        }
382    }
383
384    private boolean filter( DependencyProcessingContext context )
385    {
386        return context.depSelector != null && !context.depSelector.selectDependency( context.dependency );
387    }
388
389
390    private void resolveArtifactDescriptorAsync( Args args, DependencyProcessingContext context,
391                                                 Results results )
392    {
393        Dependency dependency = context.dependency;
394        args.resolver.resolveDescriptors( dependency.getArtifact(), () ->
395        {
396            VersionRangeRequest rangeRequest =
397                    createVersionRangeRequest( args.request.getRequestContext(), context.trace, context.repositories,
398                            dependency );
399            VersionRangeResult rangeResult = cachedResolveRangeResult( rangeRequest, args.pool, args.session );
400            List<? extends Version> versions = filterVersions( dependency, rangeResult, context.verFilter,
401                    args.versionContext );
402
403            //resolve newer version first to maximize benefits of skipper
404            Collections.reverse( versions );
405
406            Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>( versions.size() );
407            Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream();
408            stream.forEach( version ->
409                    Optional.ofNullable( resolveDescriptorForVersion( args, context, results, dependency, version ) )
410                            .ifPresent( r -> descriptors.put( version, r ) )
411            );
412
413            DescriptorResolutionResult resolutionResult =
414                    new DescriptorResolutionResult( dependency.getArtifact(), rangeResult );
415            //keep original sequence
416            versions.forEach( version -> resolutionResult.descriptors.put( version, descriptors.get( version ) ) );
417            //populate for versions in version range
418            resolutionResult.flatten().forEach( dr -> args.resolver.cacheVersionRangeDescriptor( dr.artifact, dr ) );
419
420            return resolutionResult;
421        } );
422    }
423
424    private ArtifactDescriptorResult resolveDescriptorForVersion( Args args, DependencyProcessingContext context,
425                                                                  Results results, Dependency dependency,
426                                                                  Version version )
427    {
428        Artifact original = dependency.getArtifact();
429        Artifact newArtifact = new DefaultArtifact( original.getGroupId(),
430                original.getArtifactId(), original.getClassifier(), original.getExtension(),
431                version.toString(), original.getProperties(), (ArtifactType) null );
432        Dependency newDependency = new Dependency( newArtifact, dependency.getScope(), dependency.isOptional(),
433                dependency.getExclusions() );
434        DependencyProcessingContext newContext = context.copy();
435
436        ArtifactDescriptorRequest descriptorRequest =
437                createArtifactDescriptorRequest( args.request.getRequestContext(), context.trace,
438                        newContext.repositories, newDependency );
439        return isLackingDescriptor( newArtifact )
440                ? new ArtifactDescriptorResult( descriptorRequest )
441                : resolveCachedArtifactDescriptor( args.pool, descriptorRequest, args.session,
442                newContext.withDependency( newDependency ), results );
443    }
444
445    private ArtifactDescriptorResult resolveCachedArtifactDescriptor( DataPool pool,
446                                                                      ArtifactDescriptorRequest descriptorRequest,
447                                                                      RepositorySystemSession session,
448                                                                      DependencyProcessingContext context,
449                                                                      Results results )
450    {
451        Object key = pool.toKey( descriptorRequest );
452        ArtifactDescriptorResult descriptorResult = pool.getDescriptor( key, descriptorRequest );
453        if ( descriptorResult == null )
454        {
455            try
456            {
457                descriptorResult = descriptorReader.readArtifactDescriptor( session, descriptorRequest );
458                pool.putDescriptor( key, descriptorResult );
459            }
460            catch ( ArtifactDescriptorException e )
461            {
462                results.addException( context.dependency, e, context.parents );
463                pool.putDescriptor( key, e );
464                return null;
465            }
466
467        }
468        else if ( descriptorResult == DataPool.NO_DESCRIPTOR )
469        {
470            return null;
471        }
472
473        return descriptorResult;
474    }
475
476    static class ParallelDescriptorResolver
477    {
478        final ExecutorService executorService;
479
480        /**
481         * Artifact ID -> Future of DescriptorResolutionResult
482         */
483        final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>( 256 );
484        final Logger logger = LoggerFactory.getLogger( getClass() );
485
486        ParallelDescriptorResolver( RepositorySystemSession session )
487        {
488            this.executorService = getExecutorService( session );
489        }
490
491        void resolveDescriptors( Artifact artifact, Callable<DescriptorResolutionResult> callable )
492        {
493            results.computeIfAbsent( ArtifactIdUtils.toId( artifact ),
494                    key -> this.executorService.submit( callable ) );
495        }
496
497        void cacheVersionRangeDescriptor( Artifact artifact, DescriptorResolutionResult resolutionResult )
498        {
499            results.computeIfAbsent( ArtifactIdUtils.toId( artifact ),
500                    key -> ConcurrentUtils.constantFuture( resolutionResult ) );
501        }
502
503        Future<DescriptorResolutionResult> find( Artifact artifact )
504        {
505            return results.get( ArtifactIdUtils.toId( artifact ) );
506        }
507
508        void shutdown()
509        {
510            executorService.shutdown();
511        }
512
513        private ExecutorService getExecutorService( RepositorySystemSession session )
514        {
515            int nThreads = ConfigUtils.getInteger( session, 5, CONFIG_PROP_THREADS, "maven.artifact.threads" );
516            logger.debug( "Created thread pool with {} threads to resolve descriptors.", nThreads );
517            return new ThreadPoolExecutor( nThreads, nThreads, 3L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(),
518                    new WorkerThreadFactory( getClass().getSimpleName() ) );
519        }
520    }
521
522    static class DescriptorResolutionResult
523    {
524        Artifact artifact;
525
526        VersionRangeResult rangeResult;
527
528        Map<Version, ArtifactDescriptorResult> descriptors;
529
530        DescriptorResolutionResult( Artifact artifact, VersionRangeResult rangeResult )
531        {
532            this.artifact = artifact;
533            this.rangeResult = rangeResult;
534            this.descriptors = new LinkedHashMap<>( rangeResult.getVersions().size() );
535        }
536
537        DescriptorResolutionResult( VersionRangeResult rangeResult,
538                                    Version version, ArtifactDescriptorResult descriptor )
539        {
540            this( descriptor.getArtifact(), rangeResult );
541            this.descriptors.put( version, descriptor );
542        }
543
544        List<DescriptorResolutionResult> flatten()
545        {
546            if ( descriptors.size() > 1 )
547            {
548                return descriptors.entrySet().stream()
549                        .map( e -> new DescriptorResolutionResult( rangeResult, e.getKey(), e.getValue() ) )
550                        .collect( Collectors.toList() );
551            }
552            else
553            {
554                return Collections.emptyList();
555            }
556        }
557    }
558
559    static class Args
560    {
561
562        final RepositorySystemSession session;
563
564        final boolean ignoreRepos;
565
566        final boolean premanagedState;
567
568        final DataPool pool;
569
570        final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>( 128 );
571
572        final DefaultDependencyCollectionContext collectionContext;
573
574        final DefaultVersionFilterContext versionContext;
575
576        final CollectRequest request;
577
578        final DependencyResolutionSkipper skipper;
579
580        final ParallelDescriptorResolver resolver;
581
582        Args( RepositorySystemSession session, DataPool pool,
583              DefaultDependencyCollectionContext collectionContext, DefaultVersionFilterContext versionContext,
584              CollectRequest request, DependencyResolutionSkipper skipper,
585              ParallelDescriptorResolver resolver )
586        {
587            this.session = session;
588            this.request = request;
589            this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories();
590            this.premanagedState = ConfigUtils.getBoolean( session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE );
591            this.pool = pool;
592            this.collectionContext = collectionContext;
593            this.versionContext = versionContext;
594            this.skipper = skipper;
595            this.resolver = resolver;
596        }
597
598    }
599
600}