001package org.eclipse.aether.internal.impl.collect.bf; 002 003/* 004 * Licensed to the Apache Software Foundation (ASF) under one 005 * or more contributor license agreements. See the NOTICE file 006 * distributed with this work for additional information 007 * regarding copyright ownership. The ASF licenses this file 008 * to you under the Apache License, Version 2.0 (the 009 * "License"); you may not use this file except in compliance 010 * with the License. You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, 015 * software distributed under the License is distributed on an 016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 017 * KIND, either express or implied. See the License for the 018 * specific language governing permissions and limitations 019 * under the License. 020 */ 021 022import javax.inject.Inject; 023import javax.inject.Named; 024import javax.inject.Singleton; 025 026import java.util.ArrayDeque; 027import java.util.ArrayList; 028import java.util.Collections; 029import java.util.LinkedHashMap; 030import java.util.List; 031import java.util.Map; 032import java.util.Optional; 033import java.util.Queue; 034import java.util.Set; 035import java.util.concurrent.Callable; 036import java.util.concurrent.ConcurrentHashMap; 037import java.util.concurrent.ExecutorService; 038import java.util.concurrent.Future; 039import java.util.concurrent.LinkedBlockingQueue; 040import java.util.concurrent.ThreadPoolExecutor; 041import java.util.concurrent.TimeUnit; 042import java.util.stream.Collectors; 043import java.util.stream.Stream; 044 045import org.apache.commons.lang3.concurrent.ConcurrentUtils; 046import org.eclipse.aether.RepositorySystemSession; 047import org.eclipse.aether.RequestTrace; 048import org.eclipse.aether.artifact.Artifact; 049import org.eclipse.aether.artifact.ArtifactType; 050import org.eclipse.aether.artifact.DefaultArtifact; 051import org.eclipse.aether.collection.CollectRequest; 052import org.eclipse.aether.collection.DependencyManager; 053import org.eclipse.aether.collection.DependencySelector; 054import org.eclipse.aether.collection.DependencyTraverser; 055import org.eclipse.aether.collection.VersionFilter; 056import org.eclipse.aether.graph.DefaultDependencyNode; 057import org.eclipse.aether.graph.Dependency; 058import org.eclipse.aether.graph.DependencyNode; 059import org.eclipse.aether.impl.ArtifactDescriptorReader; 060import org.eclipse.aether.impl.RemoteRepositoryManager; 061import org.eclipse.aether.impl.VersionRangeResolver; 062import org.eclipse.aether.internal.impl.collect.DataPool; 063import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext; 064import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext; 065import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate; 066import org.eclipse.aether.internal.impl.collect.PremanagedDependency; 067import org.eclipse.aether.repository.RemoteRepository; 068import org.eclipse.aether.resolution.ArtifactDescriptorException; 069import org.eclipse.aether.resolution.ArtifactDescriptorRequest; 070import org.eclipse.aether.resolution.ArtifactDescriptorResult; 071import org.eclipse.aether.resolution.VersionRangeRequest; 072import org.eclipse.aether.resolution.VersionRangeResult; 073import org.eclipse.aether.spi.locator.Service; 074import org.eclipse.aether.util.ConfigUtils; 075import org.eclipse.aether.util.artifact.ArtifactIdUtils; 076import org.eclipse.aether.util.concurrency.WorkerThreadFactory; 077import org.eclipse.aether.util.graph.manager.DependencyManagerUtils; 078import org.eclipse.aether.version.Version; 079import org.slf4j.Logger; 080import org.slf4j.LoggerFactory; 081 082import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find; 083 084/** 085 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector} 086 * 087 * @since 1.8.0 088 */ 089@Singleton 090@Named( BfDependencyCollector.NAME ) 091public class BfDependencyCollector 092 extends DependencyCollectorDelegate implements Service 093{ 094 public static final String NAME = "bf"; 095 096 /** 097 * The key in the repository session's {@link RepositorySystemSession#getConfigProperties() 098 * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode. 099 * 100 * @since 1.8.0 101 */ 102 static final String CONFIG_PROP_SKIPPER = "aether.dependencyCollector.bf.skipper"; 103 104 /** 105 * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}. 106 * 107 * @since 1.8.0 108 */ 109 static final boolean CONFIG_PROP_SKIPPER_DEFAULT = true; 110 111 /** 112 * The count of threads to be used when collecting POMs in parallel, default value 5. 113 * 114 * @since 1.9.0 115 */ 116 static final String CONFIG_PROP_THREADS = "aether.dependencyCollector.bf.threads"; 117 118 /** 119 * Default ctor for SL. 120 * 121 * @deprecated Will be dropped once SL gone. 122 */ 123 @Deprecated 124 public BfDependencyCollector() 125 { 126 // enables default constructor 127 } 128 129 @Inject 130 BfDependencyCollector( RemoteRepositoryManager remoteRepositoryManager, 131 ArtifactDescriptorReader artifactDescriptorReader, 132 VersionRangeResolver versionRangeResolver ) 133 { 134 super( remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver ); 135 } 136 137 @SuppressWarnings( "checkstyle:parameternumber" ) 138 @Override 139 protected void doCollectDependencies( RepositorySystemSession session, RequestTrace trace, DataPool pool, 140 DefaultDependencyCollectionContext context, 141 DefaultVersionFilterContext versionContext, 142 CollectRequest request, DependencyNode node, 143 List<RemoteRepository> repositories, List<Dependency> dependencies, 144 List<Dependency> managedDependencies, Results results ) 145 { 146 boolean useSkip = ConfigUtils.getBoolean( 147 session, CONFIG_PROP_SKIPPER_DEFAULT, CONFIG_PROP_SKIPPER 148 ); 149 if ( useSkip ) 150 { 151 logger.debug( "Collector skip mode enabled" ); 152 } 153 154 Args args = 155 new Args( session, pool, context, versionContext, request, 156 useSkip ? DependencyResolutionSkipper.defaultSkipper() 157 : DependencyResolutionSkipper.neverSkipper(), 158 new ParallelDescriptorResolver( session ) ); 159 160 DependencySelector rootDepSelector = session.getDependencySelector() != null 161 ? session.getDependencySelector().deriveChildSelector( context ) : null; 162 DependencyManager rootDepManager = session.getDependencyManager() != null 163 ? session.getDependencyManager().deriveChildManager( context ) : null; 164 DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null 165 ? session.getDependencyTraverser().deriveChildTraverser( context ) : null; 166 VersionFilter rootVerFilter = session.getVersionFilter() != null 167 ? session.getVersionFilter().deriveChildFilter( context ) : null; 168 169 List<DependencyNode> parents = Collections.singletonList( node ); 170 for ( Dependency dependency : dependencies ) 171 { 172 RequestTrace childTrace = collectStepTrace( trace, args.request.getRequestContext(), parents, 173 dependency ); 174 DependencyProcessingContext processingContext = 175 new DependencyProcessingContext( rootDepSelector, rootDepManager, rootDepTraverser, 176 rootVerFilter, childTrace, repositories, managedDependencies, parents, dependency, 177 PremanagedDependency.create( rootDepManager, dependency, 178 false, args.premanagedState ) ); 179 if ( !filter( processingContext ) ) 180 { 181 processingContext.withDependency( processingContext.premanagedDependency.getManagedDependency() ); 182 resolveArtifactDescriptorAsync( args, processingContext, results ); 183 args.dependencyProcessingQueue.add( processingContext ); 184 } 185 } 186 187 while ( !args.dependencyProcessingQueue.isEmpty() ) 188 { 189 processDependency( args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(), 190 false ); 191 } 192 193 args.resolver.shutdown(); 194 args.skipper.report(); 195 } 196 197 @SuppressWarnings( "checkstyle:parameternumber" ) 198 private void processDependency( Args args, Results results, 199 DependencyProcessingContext context, List<Artifact> relocations, 200 boolean disableVersionManagement ) 201 { 202 Dependency dependency = context.dependency; 203 PremanagedDependency preManaged = context.premanagedDependency; 204 205 boolean noDescriptor = isLackingDescriptor( dependency.getArtifact() ); 206 boolean traverse = 207 !noDescriptor && ( context.depTraverser == null || context.depTraverser.traverseDependency( 208 dependency ) ); 209 210 Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find( dependency.getArtifact() ); 211 DescriptorResolutionResult resolutionResult; 212 VersionRangeResult rangeResult; 213 try 214 { 215 resolutionResult = resolutionResultFuture.get(); 216 rangeResult = resolutionResult.rangeResult; 217 } 218 catch ( Exception e ) 219 { 220 results.addException( dependency, e, context.parents ); 221 return; 222 } 223 224 Set<Version> versions = resolutionResult.descriptors.keySet(); 225 for ( Version version : versions ) 226 { 227 Artifact originalArtifact = dependency.getArtifact().setVersion( version.toString() ); 228 Dependency d = dependency.setArtifact( originalArtifact ); 229 230 final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get( version ); 231 if ( descriptorResult != null ) 232 { 233 d = d.setArtifact( descriptorResult.getArtifact() ); 234 235 int cycleEntry = find( context.parents, d.getArtifact() ); 236 if ( cycleEntry >= 0 ) 237 { 238 results.addCycle( context.parents, cycleEntry, d ); 239 DependencyNode cycleNode = context.parents.get( cycleEntry ); 240 if ( cycleNode.getDependency() != null ) 241 { 242 DefaultDependencyNode child = 243 createDependencyNode( relocations, preManaged, rangeResult, version, d, 244 descriptorResult, cycleNode ); 245 context.getParent().getChildren().add( child ); 246 continue; 247 } 248 } 249 250 if ( !descriptorResult.getRelocations().isEmpty() ) 251 { 252 boolean disableVersionManagementSubsequently = 253 originalArtifact.getGroupId().equals( d.getArtifact().getGroupId() ) 254 && originalArtifact.getArtifactId().equals( d.getArtifact().getArtifactId() ); 255 256 PremanagedDependency premanagedDependency = 257 PremanagedDependency.create( context.depManager, d, disableVersionManagementSubsequently, 258 args.premanagedState ); 259 DependencyProcessingContext relocatedContext = 260 new DependencyProcessingContext( context.depSelector, context.depManager, 261 context.depTraverser, context.verFilter, 262 context.trace, context.repositories, descriptorResult.getManagedDependencies(), 263 context.parents, 264 d, premanagedDependency ); 265 266 if ( !filter( relocatedContext ) ) 267 { 268 relocatedContext.withDependency( premanagedDependency.getManagedDependency() ); 269 resolveArtifactDescriptorAsync( args, relocatedContext, results ); 270 processDependency( args, results, relocatedContext, descriptorResult.getRelocations(), 271 disableVersionManagementSubsequently ); 272 } 273 274 return; 275 } 276 else 277 { 278 d = args.pool.intern( d.setArtifact( args.pool.intern( d.getArtifact() ) ) ); 279 280 List<RemoteRepository> repos = 281 getRemoteRepositories( rangeResult.getRepository( version ), context.repositories ); 282 283 DefaultDependencyNode child = 284 createDependencyNode( relocations, preManaged, rangeResult, version, d, 285 descriptorResult.getAliases(), repos, args.request.getRequestContext() ); 286 287 context.getParent().getChildren().add( child ); 288 289 boolean recurse = traverse && !descriptorResult.getDependencies().isEmpty(); 290 DependencyProcessingContext parentContext = context.withDependency( d ); 291 if ( recurse ) 292 { 293 doRecurse( args, parentContext, descriptorResult, child, results, 294 disableVersionManagement ); 295 } 296 else if ( !args.skipper.skipResolution( child, parentContext.parents ) ) 297 { 298 List<DependencyNode> parents = new ArrayList<>( parentContext.parents.size() + 1 ); 299 parents.addAll( parentContext.parents ); 300 parents.add( child ); 301 args.skipper.cache( child, parents ); 302 } 303 } 304 } 305 else 306 { 307 List<RemoteRepository> repos = 308 getRemoteRepositories( rangeResult.getRepository( version ), context.repositories ); 309 DefaultDependencyNode child = 310 createDependencyNode( relocations, preManaged, rangeResult, version, d, null, repos, 311 args.request.getRequestContext() ); 312 context.getParent().getChildren().add( child ); 313 } 314 } 315 } 316 317 @SuppressWarnings( "checkstyle:parameternumber" ) 318 private void doRecurse( Args args, DependencyProcessingContext parentContext, 319 ArtifactDescriptorResult descriptorResult, DefaultDependencyNode child, Results results, 320 boolean disableVersionManagement ) 321 { 322 DefaultDependencyCollectionContext context = args.collectionContext; 323 context.set( parentContext.dependency, descriptorResult.getManagedDependencies() ); 324 325 DependencySelector childSelector = 326 parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector( context ) : null; 327 DependencyManager childManager = 328 parentContext.depManager != null ? parentContext.depManager.deriveChildManager( context ) : null; 329 DependencyTraverser childTraverser = 330 parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser( context ) : null; 331 VersionFilter childFilter = 332 parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter( context ) : null; 333 334 final List<RemoteRepository> childRepos = 335 args.ignoreRepos 336 ? parentContext.repositories 337 : remoteRepositoryManager.aggregateRepositories( args.session, parentContext.repositories, 338 descriptorResult.getRepositories(), true ); 339 340 Object key = 341 args.pool.toKey( parentContext.dependency.getArtifact(), childRepos, childSelector, childManager, 342 childTraverser, childFilter ); 343 344 List<DependencyNode> children = args.pool.getChildren( key ); 345 if ( children == null ) 346 { 347 boolean skipResolution = args.skipper.skipResolution( child, parentContext.parents ); 348 if ( !skipResolution ) 349 { 350 List<DependencyNode> parents = new ArrayList<>( parentContext.parents.size() + 1 ); 351 parents.addAll( parentContext.parents ); 352 parents.add( child ); 353 for ( Dependency dependency : descriptorResult.getDependencies() ) 354 { 355 RequestTrace childTrace = 356 collectStepTrace( parentContext.trace, args.request.getRequestContext(), parents, 357 dependency ); 358 PremanagedDependency premanagedDependency = 359 PremanagedDependency.create( childManager, dependency, disableVersionManagement, 360 args.premanagedState ); 361 DependencyProcessingContext processingContext = 362 new DependencyProcessingContext( childSelector, childManager, childTraverser, childFilter, 363 childTrace, childRepos, descriptorResult.getManagedDependencies(), parents, 364 dependency, premanagedDependency ); 365 if ( !filter( processingContext ) ) 366 { 367 //resolve descriptors ahead for managed dependency 368 processingContext.withDependency( 369 processingContext.premanagedDependency.getManagedDependency() ); 370 resolveArtifactDescriptorAsync( args, processingContext, results ); 371 args.dependencyProcessingQueue.add( processingContext ); 372 } 373 } 374 args.pool.putChildren( key, child.getChildren() ); 375 args.skipper.cache( child, parents ); 376 } 377 } 378 else 379 { 380 child.setChildren( children ); 381 } 382 } 383 384 private boolean filter( DependencyProcessingContext context ) 385 { 386 return context.depSelector != null && !context.depSelector.selectDependency( context.dependency ); 387 } 388 389 390 private void resolveArtifactDescriptorAsync( Args args, DependencyProcessingContext context, 391 Results results ) 392 { 393 Dependency dependency = context.dependency; 394 args.resolver.resolveDescriptors( dependency.getArtifact(), () -> 395 { 396 VersionRangeRequest rangeRequest = 397 createVersionRangeRequest( args.request.getRequestContext(), context.trace, context.repositories, 398 dependency ); 399 VersionRangeResult rangeResult = cachedResolveRangeResult( rangeRequest, args.pool, args.session ); 400 List<? extends Version> versions = filterVersions( dependency, rangeResult, context.verFilter, 401 args.versionContext ); 402 403 //resolve newer version first to maximize benefits of skipper 404 Collections.reverse( versions ); 405 406 Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>( versions.size() ); 407 Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream(); 408 stream.forEach( version -> 409 Optional.ofNullable( resolveDescriptorForVersion( args, context, results, dependency, version ) ) 410 .ifPresent( r -> descriptors.put( version, r ) ) 411 ); 412 413 DescriptorResolutionResult resolutionResult = 414 new DescriptorResolutionResult( dependency.getArtifact(), rangeResult ); 415 //keep original sequence 416 versions.forEach( version -> resolutionResult.descriptors.put( version, descriptors.get( version ) ) ); 417 //populate for versions in version range 418 resolutionResult.flatten().forEach( dr -> args.resolver.cacheVersionRangeDescriptor( dr.artifact, dr ) ); 419 420 return resolutionResult; 421 } ); 422 } 423 424 private ArtifactDescriptorResult resolveDescriptorForVersion( Args args, DependencyProcessingContext context, 425 Results results, Dependency dependency, 426 Version version ) 427 { 428 Artifact original = dependency.getArtifact(); 429 Artifact newArtifact = new DefaultArtifact( original.getGroupId(), 430 original.getArtifactId(), original.getClassifier(), original.getExtension(), 431 version.toString(), original.getProperties(), (ArtifactType) null ); 432 Dependency newDependency = new Dependency( newArtifact, dependency.getScope(), dependency.isOptional(), 433 dependency.getExclusions() ); 434 DependencyProcessingContext newContext = context.copy(); 435 436 ArtifactDescriptorRequest descriptorRequest = 437 createArtifactDescriptorRequest( args.request.getRequestContext(), context.trace, 438 newContext.repositories, newDependency ); 439 return isLackingDescriptor( newArtifact ) 440 ? new ArtifactDescriptorResult( descriptorRequest ) 441 : resolveCachedArtifactDescriptor( args.pool, descriptorRequest, args.session, 442 newContext.withDependency( newDependency ), results ); 443 } 444 445 private ArtifactDescriptorResult resolveCachedArtifactDescriptor( DataPool pool, 446 ArtifactDescriptorRequest descriptorRequest, 447 RepositorySystemSession session, 448 DependencyProcessingContext context, 449 Results results ) 450 { 451 Object key = pool.toKey( descriptorRequest ); 452 ArtifactDescriptorResult descriptorResult = pool.getDescriptor( key, descriptorRequest ); 453 if ( descriptorResult == null ) 454 { 455 try 456 { 457 descriptorResult = descriptorReader.readArtifactDescriptor( session, descriptorRequest ); 458 pool.putDescriptor( key, descriptorResult ); 459 } 460 catch ( ArtifactDescriptorException e ) 461 { 462 results.addException( context.dependency, e, context.parents ); 463 pool.putDescriptor( key, e ); 464 return null; 465 } 466 467 } 468 else if ( descriptorResult == DataPool.NO_DESCRIPTOR ) 469 { 470 return null; 471 } 472 473 return descriptorResult; 474 } 475 476 static class ParallelDescriptorResolver 477 { 478 final ExecutorService executorService; 479 480 /** 481 * Artifact ID -> Future of DescriptorResolutionResult 482 */ 483 final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>( 256 ); 484 final Logger logger = LoggerFactory.getLogger( getClass() ); 485 486 ParallelDescriptorResolver( RepositorySystemSession session ) 487 { 488 this.executorService = getExecutorService( session ); 489 } 490 491 void resolveDescriptors( Artifact artifact, Callable<DescriptorResolutionResult> callable ) 492 { 493 results.computeIfAbsent( ArtifactIdUtils.toId( artifact ), 494 key -> this.executorService.submit( callable ) ); 495 } 496 497 void cacheVersionRangeDescriptor( Artifact artifact, DescriptorResolutionResult resolutionResult ) 498 { 499 results.computeIfAbsent( ArtifactIdUtils.toId( artifact ), 500 key -> ConcurrentUtils.constantFuture( resolutionResult ) ); 501 } 502 503 Future<DescriptorResolutionResult> find( Artifact artifact ) 504 { 505 return results.get( ArtifactIdUtils.toId( artifact ) ); 506 } 507 508 void shutdown() 509 { 510 executorService.shutdown(); 511 } 512 513 private ExecutorService getExecutorService( RepositorySystemSession session ) 514 { 515 int nThreads = ConfigUtils.getInteger( session, 5, CONFIG_PROP_THREADS, "maven.artifact.threads" ); 516 logger.debug( "Created thread pool with {} threads to resolve descriptors.", nThreads ); 517 return new ThreadPoolExecutor( nThreads, nThreads, 3L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), 518 new WorkerThreadFactory( getClass().getSimpleName() ) ); 519 } 520 } 521 522 static class DescriptorResolutionResult 523 { 524 Artifact artifact; 525 526 VersionRangeResult rangeResult; 527 528 Map<Version, ArtifactDescriptorResult> descriptors; 529 530 DescriptorResolutionResult( Artifact artifact, VersionRangeResult rangeResult ) 531 { 532 this.artifact = artifact; 533 this.rangeResult = rangeResult; 534 this.descriptors = new LinkedHashMap<>( rangeResult.getVersions().size() ); 535 } 536 537 DescriptorResolutionResult( VersionRangeResult rangeResult, 538 Version version, ArtifactDescriptorResult descriptor ) 539 { 540 this( descriptor.getArtifact(), rangeResult ); 541 this.descriptors.put( version, descriptor ); 542 } 543 544 List<DescriptorResolutionResult> flatten() 545 { 546 if ( descriptors.size() > 1 ) 547 { 548 return descriptors.entrySet().stream() 549 .map( e -> new DescriptorResolutionResult( rangeResult, e.getKey(), e.getValue() ) ) 550 .collect( Collectors.toList() ); 551 } 552 else 553 { 554 return Collections.emptyList(); 555 } 556 } 557 } 558 559 static class Args 560 { 561 562 final RepositorySystemSession session; 563 564 final boolean ignoreRepos; 565 566 final boolean premanagedState; 567 568 final DataPool pool; 569 570 final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>( 128 ); 571 572 final DefaultDependencyCollectionContext collectionContext; 573 574 final DefaultVersionFilterContext versionContext; 575 576 final CollectRequest request; 577 578 final DependencyResolutionSkipper skipper; 579 580 final ParallelDescriptorResolver resolver; 581 582 Args( RepositorySystemSession session, DataPool pool, 583 DefaultDependencyCollectionContext collectionContext, DefaultVersionFilterContext versionContext, 584 CollectRequest request, DependencyResolutionSkipper skipper, 585 ParallelDescriptorResolver resolver ) 586 { 587 this.session = session; 588 this.request = request; 589 this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories(); 590 this.premanagedState = ConfigUtils.getBoolean( session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE ); 591 this.pool = pool; 592 this.collectionContext = collectionContext; 593 this.versionContext = versionContext; 594 this.skipper = skipper; 595 this.resolver = resolver; 596 } 597 598 } 599 600}