001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, 013 * software distributed under the License is distributed on an 014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 * KIND, either express or implied. See the License for the 016 * specific language governing permissions and limitations 017 * under the License. 018 */ 019package org.eclipse.aether.internal.impl.collect.bf; 020 021import javax.inject.Inject; 022import javax.inject.Named; 023import javax.inject.Singleton; 024 025import java.io.Closeable; 026import java.util.ArrayDeque; 027import java.util.ArrayList; 028import java.util.Collections; 029import java.util.LinkedHashMap; 030import java.util.List; 031import java.util.Map; 032import java.util.Optional; 033import java.util.Queue; 034import java.util.Set; 035import java.util.concurrent.Callable; 036import java.util.concurrent.ConcurrentHashMap; 037import java.util.concurrent.ExecutionException; 038import java.util.concurrent.ExecutorService; 039import java.util.concurrent.Future; 040import java.util.concurrent.TimeUnit; 041import java.util.concurrent.TimeoutException; 042import java.util.stream.Collectors; 043import java.util.stream.Stream; 044 045import org.eclipse.aether.RepositorySystemSession; 046import org.eclipse.aether.RequestTrace; 047import org.eclipse.aether.artifact.Artifact; 048import org.eclipse.aether.artifact.ArtifactType; 049import org.eclipse.aether.artifact.DefaultArtifact; 050import org.eclipse.aether.collection.CollectRequest; 051import org.eclipse.aether.collection.DependencyManager; 052import org.eclipse.aether.collection.DependencySelector; 053import org.eclipse.aether.collection.DependencyTraverser; 054import org.eclipse.aether.collection.VersionFilter; 055import org.eclipse.aether.graph.DefaultDependencyNode; 056import org.eclipse.aether.graph.Dependency; 057import org.eclipse.aether.graph.DependencyNode; 058import org.eclipse.aether.impl.ArtifactDescriptorReader; 059import org.eclipse.aether.impl.RemoteRepositoryManager; 060import org.eclipse.aether.impl.VersionRangeResolver; 061import org.eclipse.aether.internal.impl.collect.DataPool; 062import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext; 063import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext; 064import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate; 065import org.eclipse.aether.internal.impl.collect.PremanagedDependency; 066import org.eclipse.aether.repository.RemoteRepository; 067import org.eclipse.aether.resolution.ArtifactDescriptorException; 068import org.eclipse.aether.resolution.ArtifactDescriptorRequest; 069import org.eclipse.aether.resolution.ArtifactDescriptorResult; 070import org.eclipse.aether.resolution.VersionRangeRequest; 071import org.eclipse.aether.resolution.VersionRangeResult; 072import org.eclipse.aether.spi.locator.Service; 073import org.eclipse.aether.util.ConfigUtils; 074import org.eclipse.aether.util.artifact.ArtifactIdUtils; 075import org.eclipse.aether.util.concurrency.ExecutorUtils; 076import org.eclipse.aether.util.graph.manager.DependencyManagerUtils; 077import org.eclipse.aether.version.Version; 078 079import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find; 080 081/** 082 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector} 083 * 084 * @since 1.8.0 085 */ 086@Singleton 087@Named(BfDependencyCollector.NAME) 088public class BfDependencyCollector extends DependencyCollectorDelegate implements Service { 089 public static final String NAME = "bf"; 090 091 /** 092 * The key in the repository session's {@link RepositorySystemSession#getConfigProperties() 093 * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode. 094 * 095 * @since 1.8.0 096 */ 097 static final String CONFIG_PROP_SKIPPER = "aether.dependencyCollector.bf.skipper"; 098 099 /** 100 * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}. 101 * 102 * @since 1.8.0 103 */ 104 static final boolean CONFIG_PROP_SKIPPER_DEFAULT = true; 105 106 /** 107 * The count of threads to be used when collecting POMs in parallel, default value 5. 108 * 109 * @since 1.9.0 110 */ 111 static final String CONFIG_PROP_THREADS = "aether.dependencyCollector.bf.threads"; 112 113 /** 114 * Default ctor for SL. 115 * 116 * @deprecated Will be dropped once SL gone. 117 */ 118 @Deprecated 119 public BfDependencyCollector() { 120 // enables default constructor 121 } 122 123 @Inject 124 public BfDependencyCollector( 125 RemoteRepositoryManager remoteRepositoryManager, 126 ArtifactDescriptorReader artifactDescriptorReader, 127 VersionRangeResolver versionRangeResolver) { 128 super(remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver); 129 } 130 131 @SuppressWarnings("checkstyle:parameternumber") 132 @Override 133 protected void doCollectDependencies( 134 RepositorySystemSession session, 135 RequestTrace trace, 136 DataPool pool, 137 DefaultDependencyCollectionContext context, 138 DefaultVersionFilterContext versionContext, 139 CollectRequest request, 140 DependencyNode node, 141 List<RemoteRepository> repositories, 142 List<Dependency> dependencies, 143 List<Dependency> managedDependencies, 144 Results results) { 145 boolean useSkip = ConfigUtils.getBoolean(session, CONFIG_PROP_SKIPPER_DEFAULT, CONFIG_PROP_SKIPPER); 146 int nThreads = ExecutorUtils.threadCount(session, 5, CONFIG_PROP_THREADS, "maven.artifact.threads"); 147 logger.debug("Using thread pool with {} threads to resolve descriptors.", nThreads); 148 149 if (useSkip) { 150 logger.debug("Collector skip mode enabled"); 151 } 152 153 try (DependencyResolutionSkipper skipper = useSkip 154 ? DependencyResolutionSkipper.defaultSkipper() 155 : DependencyResolutionSkipper.neverSkipper(); 156 ParallelDescriptorResolver parallelDescriptorResolver = new ParallelDescriptorResolver(nThreads)) { 157 Args args = new Args(session, pool, context, versionContext, request, skipper, parallelDescriptorResolver); 158 159 DependencySelector rootDepSelector = session.getDependencySelector() != null 160 ? session.getDependencySelector().deriveChildSelector(context) 161 : null; 162 DependencyManager rootDepManager = session.getDependencyManager() != null 163 ? session.getDependencyManager().deriveChildManager(context) 164 : null; 165 DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null 166 ? session.getDependencyTraverser().deriveChildTraverser(context) 167 : null; 168 VersionFilter rootVerFilter = session.getVersionFilter() != null 169 ? session.getVersionFilter().deriveChildFilter(context) 170 : null; 171 172 List<DependencyNode> parents = Collections.singletonList(node); 173 for (Dependency dependency : dependencies) { 174 RequestTrace childTrace = 175 collectStepTrace(trace, args.request.getRequestContext(), parents, dependency); 176 DependencyProcessingContext processingContext = new DependencyProcessingContext( 177 rootDepSelector, 178 rootDepManager, 179 rootDepTraverser, 180 rootVerFilter, 181 childTrace, 182 repositories, 183 managedDependencies, 184 parents, 185 dependency, 186 PremanagedDependency.create(rootDepManager, dependency, false, args.premanagedState)); 187 if (!filter(processingContext)) { 188 processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency()); 189 resolveArtifactDescriptorAsync(args, processingContext, results); 190 args.dependencyProcessingQueue.add(processingContext); 191 } 192 } 193 194 while (!args.dependencyProcessingQueue.isEmpty()) { 195 processDependency( 196 args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(), false); 197 } 198 } 199 } 200 201 @SuppressWarnings("checkstyle:parameternumber") 202 private void processDependency( 203 Args args, 204 Results results, 205 DependencyProcessingContext context, 206 List<Artifact> relocations, 207 boolean disableVersionManagement) { 208 Dependency dependency = context.dependency; 209 PremanagedDependency preManaged = context.premanagedDependency; 210 211 boolean noDescriptor = isLackingDescriptor(dependency.getArtifact()); 212 boolean traverse = 213 !noDescriptor && (context.depTraverser == null || context.depTraverser.traverseDependency(dependency)); 214 215 Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find(dependency.getArtifact()); 216 DescriptorResolutionResult resolutionResult; 217 VersionRangeResult rangeResult; 218 try { 219 resolutionResult = resolutionResultFuture.get(); 220 rangeResult = resolutionResult.rangeResult; 221 } catch (Exception e) { 222 results.addException(dependency, e, context.parents); 223 return; 224 } 225 226 Set<Version> versions = resolutionResult.descriptors.keySet(); 227 for (Version version : versions) { 228 Artifact originalArtifact = dependency.getArtifact().setVersion(version.toString()); 229 Dependency d = dependency.setArtifact(originalArtifact); 230 231 final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get(version); 232 if (descriptorResult != null) { 233 d = d.setArtifact(descriptorResult.getArtifact()); 234 235 int cycleEntry = find(context.parents, d.getArtifact()); 236 if (cycleEntry >= 0) { 237 results.addCycle(context.parents, cycleEntry, d); 238 DependencyNode cycleNode = context.parents.get(cycleEntry); 239 if (cycleNode.getDependency() != null) { 240 DefaultDependencyNode child = createDependencyNode( 241 relocations, preManaged, rangeResult, version, d, descriptorResult, cycleNode); 242 context.getParent().getChildren().add(child); 243 continue; 244 } 245 } 246 247 if (!descriptorResult.getRelocations().isEmpty()) { 248 boolean disableVersionManagementSubsequently = 249 originalArtifact.getGroupId().equals(d.getArtifact().getGroupId()) 250 && originalArtifact 251 .getArtifactId() 252 .equals(d.getArtifact().getArtifactId()); 253 254 PremanagedDependency premanagedDependency = PremanagedDependency.create( 255 context.depManager, d, disableVersionManagementSubsequently, args.premanagedState); 256 DependencyProcessingContext relocatedContext = new DependencyProcessingContext( 257 context.depSelector, 258 context.depManager, 259 context.depTraverser, 260 context.verFilter, 261 context.trace, 262 context.repositories, 263 descriptorResult.getManagedDependencies(), 264 context.parents, 265 d, 266 premanagedDependency); 267 268 if (!filter(relocatedContext)) { 269 relocatedContext.withDependency(premanagedDependency.getManagedDependency()); 270 resolveArtifactDescriptorAsync(args, relocatedContext, results); 271 processDependency( 272 args, 273 results, 274 relocatedContext, 275 descriptorResult.getRelocations(), 276 disableVersionManagementSubsequently); 277 } 278 279 return; 280 } else { 281 d = args.pool.intern(d.setArtifact(args.pool.intern(d.getArtifact()))); 282 283 List<RemoteRepository> repos = 284 getRemoteRepositories(rangeResult.getRepository(version), context.repositories); 285 286 DefaultDependencyNode child = createDependencyNode( 287 relocations, 288 preManaged, 289 rangeResult, 290 version, 291 d, 292 descriptorResult.getAliases(), 293 repos, 294 args.request.getRequestContext()); 295 296 context.getParent().getChildren().add(child); 297 298 boolean recurse = 299 traverse && !descriptorResult.getDependencies().isEmpty(); 300 DependencyProcessingContext parentContext = context.withDependency(d); 301 if (recurse) { 302 doRecurse(args, parentContext, descriptorResult, child, results, disableVersionManagement); 303 } else if (!args.skipper.skipResolution(child, parentContext.parents)) { 304 List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1); 305 parents.addAll(parentContext.parents); 306 parents.add(child); 307 args.skipper.cache(child, parents); 308 } 309 } 310 } else { 311 List<RemoteRepository> repos = 312 getRemoteRepositories(rangeResult.getRepository(version), context.repositories); 313 DefaultDependencyNode child = createDependencyNode( 314 relocations, 315 preManaged, 316 rangeResult, 317 version, 318 d, 319 null, 320 repos, 321 args.request.getRequestContext()); 322 context.getParent().getChildren().add(child); 323 } 324 } 325 } 326 327 @SuppressWarnings("checkstyle:parameternumber") 328 private void doRecurse( 329 Args args, 330 DependencyProcessingContext parentContext, 331 ArtifactDescriptorResult descriptorResult, 332 DefaultDependencyNode child, 333 Results results, 334 boolean disableVersionManagement) { 335 DefaultDependencyCollectionContext context = args.collectionContext; 336 context.set(parentContext.dependency, descriptorResult.getManagedDependencies()); 337 338 DependencySelector childSelector = 339 parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector(context) : null; 340 DependencyManager childManager = 341 parentContext.depManager != null ? parentContext.depManager.deriveChildManager(context) : null; 342 DependencyTraverser childTraverser = 343 parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser(context) : null; 344 VersionFilter childFilter = 345 parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter(context) : null; 346 347 final List<RemoteRepository> childRepos = args.ignoreRepos 348 ? parentContext.repositories 349 : remoteRepositoryManager.aggregateRepositories( 350 args.session, parentContext.repositories, descriptorResult.getRepositories(), true); 351 352 Object key = args.pool.toKey( 353 parentContext.dependency.getArtifact(), 354 childRepos, 355 childSelector, 356 childManager, 357 childTraverser, 358 childFilter); 359 360 List<DependencyNode> children = args.pool.getChildren(key); 361 if (children == null) { 362 boolean skipResolution = args.skipper.skipResolution(child, parentContext.parents); 363 if (!skipResolution) { 364 List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1); 365 parents.addAll(parentContext.parents); 366 parents.add(child); 367 for (Dependency dependency : descriptorResult.getDependencies()) { 368 RequestTrace childTrace = collectStepTrace( 369 parentContext.trace, args.request.getRequestContext(), parents, dependency); 370 PremanagedDependency premanagedDependency = PremanagedDependency.create( 371 childManager, dependency, disableVersionManagement, args.premanagedState); 372 DependencyProcessingContext processingContext = new DependencyProcessingContext( 373 childSelector, 374 childManager, 375 childTraverser, 376 childFilter, 377 childTrace, 378 childRepos, 379 descriptorResult.getManagedDependencies(), 380 parents, 381 dependency, 382 premanagedDependency); 383 if (!filter(processingContext)) { 384 // resolve descriptors ahead for managed dependency 385 processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency()); 386 resolveArtifactDescriptorAsync(args, processingContext, results); 387 args.dependencyProcessingQueue.add(processingContext); 388 } 389 } 390 args.pool.putChildren(key, child.getChildren()); 391 args.skipper.cache(child, parents); 392 } 393 } else { 394 child.setChildren(children); 395 } 396 } 397 398 private boolean filter(DependencyProcessingContext context) { 399 return context.depSelector != null && !context.depSelector.selectDependency(context.dependency); 400 } 401 402 private void resolveArtifactDescriptorAsync(Args args, DependencyProcessingContext context, Results results) { 403 Dependency dependency = context.dependency; 404 args.resolver.resolveDescriptors(dependency.getArtifact(), () -> { 405 VersionRangeRequest rangeRequest = createVersionRangeRequest( 406 args.request.getRequestContext(), context.trace, context.repositories, dependency); 407 VersionRangeResult rangeResult = cachedResolveRangeResult(rangeRequest, args.pool, args.session); 408 List<? extends Version> versions = 409 filterVersions(dependency, rangeResult, context.verFilter, args.versionContext); 410 411 // resolve newer version first to maximize benefits of skipper 412 Collections.reverse(versions); 413 414 Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>(versions.size()); 415 Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream(); 416 stream.forEach(version -> Optional.ofNullable( 417 resolveDescriptorForVersion(args, context, results, dependency, version)) 418 .ifPresent(r -> descriptors.put(version, r))); 419 420 DescriptorResolutionResult resolutionResult = 421 new DescriptorResolutionResult(dependency.getArtifact(), rangeResult); 422 // keep original sequence 423 versions.forEach(version -> resolutionResult.descriptors.put(version, descriptors.get(version))); 424 // populate for versions in version range 425 resolutionResult.flatten().forEach(dr -> args.resolver.cacheVersionRangeDescriptor(dr.artifact, dr)); 426 427 return resolutionResult; 428 }); 429 } 430 431 private ArtifactDescriptorResult resolveDescriptorForVersion( 432 Args args, DependencyProcessingContext context, Results results, Dependency dependency, Version version) { 433 Artifact original = dependency.getArtifact(); 434 Artifact newArtifact = new DefaultArtifact( 435 original.getGroupId(), 436 original.getArtifactId(), 437 original.getClassifier(), 438 original.getExtension(), 439 version.toString(), 440 original.getProperties(), 441 (ArtifactType) null); 442 Dependency newDependency = 443 new Dependency(newArtifact, dependency.getScope(), dependency.isOptional(), dependency.getExclusions()); 444 DependencyProcessingContext newContext = context.copy(); 445 446 ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest( 447 args.request.getRequestContext(), context.trace, newContext.repositories, newDependency); 448 return isLackingDescriptor(newArtifact) 449 ? new ArtifactDescriptorResult(descriptorRequest) 450 : resolveCachedArtifactDescriptor( 451 args.pool, descriptorRequest, args.session, newContext.withDependency(newDependency), results); 452 } 453 454 private ArtifactDescriptorResult resolveCachedArtifactDescriptor( 455 DataPool pool, 456 ArtifactDescriptorRequest descriptorRequest, 457 RepositorySystemSession session, 458 DependencyProcessingContext context, 459 Results results) { 460 Object key = pool.toKey(descriptorRequest); 461 ArtifactDescriptorResult descriptorResult = pool.getDescriptor(key, descriptorRequest); 462 if (descriptorResult == null) { 463 try { 464 descriptorResult = descriptorReader.readArtifactDescriptor(session, descriptorRequest); 465 pool.putDescriptor(key, descriptorResult); 466 } catch (ArtifactDescriptorException e) { 467 results.addException(context.dependency, e, context.parents); 468 pool.putDescriptor(key, e); 469 return null; 470 } 471 472 } else if (descriptorResult == DataPool.NO_DESCRIPTOR) { 473 return null; 474 } 475 476 return descriptorResult; 477 } 478 479 static class ParallelDescriptorResolver implements Closeable { 480 private final ExecutorService executorService; 481 482 /** 483 * Artifact ID -> Future of DescriptorResolutionResult 484 */ 485 private final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>(256); 486 487 ParallelDescriptorResolver(int threads) { 488 this.executorService = ExecutorUtils.threadPool(threads, getClass().getSimpleName() + "-"); 489 } 490 491 void resolveDescriptors(Artifact artifact, Callable<DescriptorResolutionResult> callable) { 492 results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> this.executorService.submit(callable)); 493 } 494 495 void cacheVersionRangeDescriptor(Artifact artifact, DescriptorResolutionResult resolutionResult) { 496 results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> new DoneFuture<>(resolutionResult)); 497 } 498 499 Future<DescriptorResolutionResult> find(Artifact artifact) { 500 return results.get(ArtifactIdUtils.toId(artifact)); 501 } 502 503 @Override 504 public void close() { 505 executorService.shutdown(); 506 } 507 } 508 509 static class DoneFuture<V> implements Future<V> { 510 private final V v; 511 512 DoneFuture(V v) { 513 this.v = v; 514 } 515 516 @Override 517 public boolean cancel(boolean mayInterruptIfRunning) { 518 return false; 519 } 520 521 @Override 522 public boolean isCancelled() { 523 return false; 524 } 525 526 @Override 527 public boolean isDone() { 528 return true; 529 } 530 531 @Override 532 public V get() throws InterruptedException, ExecutionException { 533 return v; 534 } 535 536 @Override 537 public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { 538 return v; 539 } 540 } 541 542 static class DescriptorResolutionResult { 543 Artifact artifact; 544 545 VersionRangeResult rangeResult; 546 547 Map<Version, ArtifactDescriptorResult> descriptors; 548 549 DescriptorResolutionResult(Artifact artifact, VersionRangeResult rangeResult) { 550 this.artifact = artifact; 551 this.rangeResult = rangeResult; 552 this.descriptors = new LinkedHashMap<>(rangeResult.getVersions().size()); 553 } 554 555 DescriptorResolutionResult( 556 VersionRangeResult rangeResult, Version version, ArtifactDescriptorResult descriptor) { 557 // NOTE: In case of A1 -> A2 relocation this happens: 558 // ArtifactDescriptorResult read by ArtifactDescriptorResultReader for A1 559 // will return instance that will have artifact = A2 (as RelocatedArtifact). 560 // So to properly "key" this instance, we need to use "originally requested" A1 instead! 561 // In short: 562 // ArtifactDescriptorRequest.artifact != ArtifactDescriptorResult.artifact WHEN relocation in play 563 // otherwise (no relocation), they are EQUAL. 564 this(descriptor.getRequest().getArtifact(), rangeResult); 565 this.descriptors.put(version, descriptor); 566 } 567 568 List<DescriptorResolutionResult> flatten() { 569 if (descriptors.size() > 1) { 570 return descriptors.entrySet().stream() 571 .map(e -> new DescriptorResolutionResult(rangeResult, e.getKey(), e.getValue())) 572 .collect(Collectors.toList()); 573 } else { 574 return Collections.emptyList(); 575 } 576 } 577 } 578 579 static class Args { 580 581 final RepositorySystemSession session; 582 583 final boolean ignoreRepos; 584 585 final boolean premanagedState; 586 587 final DataPool pool; 588 589 final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>(128); 590 591 final DefaultDependencyCollectionContext collectionContext; 592 593 final DefaultVersionFilterContext versionContext; 594 595 final CollectRequest request; 596 597 final DependencyResolutionSkipper skipper; 598 599 final ParallelDescriptorResolver resolver; 600 601 Args( 602 RepositorySystemSession session, 603 DataPool pool, 604 DefaultDependencyCollectionContext collectionContext, 605 DefaultVersionFilterContext versionContext, 606 CollectRequest request, 607 DependencyResolutionSkipper skipper, 608 ParallelDescriptorResolver resolver) { 609 this.session = session; 610 this.request = request; 611 this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories(); 612 this.premanagedState = ConfigUtils.getBoolean(session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE); 613 this.pool = pool; 614 this.collectionContext = collectionContext; 615 this.versionContext = versionContext; 616 this.skipper = skipper; 617 this.resolver = resolver; 618 } 619 } 620}