001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, 013 * software distributed under the License is distributed on an 014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 * KIND, either express or implied. See the License for the 016 * specific language governing permissions and limitations 017 * under the License. 018 */ 019package org.eclipse.aether.internal.impl.collect.bf; 020 021import javax.inject.Inject; 022import javax.inject.Named; 023import javax.inject.Singleton; 024 025import java.io.Closeable; 026import java.util.ArrayDeque; 027import java.util.ArrayList; 028import java.util.Collections; 029import java.util.LinkedHashMap; 030import java.util.List; 031import java.util.Map; 032import java.util.Optional; 033import java.util.Queue; 034import java.util.Set; 035import java.util.concurrent.Callable; 036import java.util.concurrent.ConcurrentHashMap; 037import java.util.concurrent.ExecutionException; 038import java.util.concurrent.ExecutorService; 039import java.util.concurrent.Future; 040import java.util.concurrent.TimeUnit; 041import java.util.concurrent.TimeoutException; 042import java.util.concurrent.atomic.AtomicReference; 043import java.util.stream.Collectors; 044import java.util.stream.Stream; 045 046import org.eclipse.aether.RepositorySystemSession; 047import org.eclipse.aether.RequestTrace; 048import org.eclipse.aether.artifact.Artifact; 049import org.eclipse.aether.collection.CollectRequest; 050import org.eclipse.aether.collection.DependencyCollectionException; 051import org.eclipse.aether.collection.DependencyManager; 052import org.eclipse.aether.collection.DependencySelector; 053import org.eclipse.aether.collection.DependencyTraverser; 054import org.eclipse.aether.collection.VersionFilter; 055import org.eclipse.aether.graph.DefaultDependencyNode; 056import org.eclipse.aether.graph.Dependency; 057import org.eclipse.aether.graph.DependencyNode; 058import org.eclipse.aether.impl.ArtifactDescriptorReader; 059import org.eclipse.aether.impl.RemoteRepositoryManager; 060import org.eclipse.aether.impl.VersionRangeResolver; 061import org.eclipse.aether.internal.impl.collect.DataPool; 062import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext; 063import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollector; 064import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext; 065import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate; 066import org.eclipse.aether.internal.impl.collect.PremanagedDependency; 067import org.eclipse.aether.repository.RemoteRepository; 068import org.eclipse.aether.resolution.ArtifactDescriptorRequest; 069import org.eclipse.aether.resolution.ArtifactDescriptorResult; 070import org.eclipse.aether.resolution.VersionRangeRequest; 071import org.eclipse.aether.resolution.VersionRangeResult; 072import org.eclipse.aether.spi.artifact.decorator.ArtifactDecoratorFactory; 073import org.eclipse.aether.util.ConfigUtils; 074import org.eclipse.aether.util.artifact.ArtifactIdUtils; 075import org.eclipse.aether.util.concurrency.ExecutorUtils; 076import org.eclipse.aether.util.graph.manager.DependencyManagerUtils; 077import org.eclipse.aether.version.Version; 078 079import static org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle.find; 080 081/** 082 * Breadth-first {@link org.eclipse.aether.impl.DependencyCollector} 083 * 084 * @since 1.8.0 085 */ 086@Singleton 087@Named(BfDependencyCollector.NAME) 088public class BfDependencyCollector extends DependencyCollectorDelegate { 089 public static final String NAME = "bf"; 090 091 private static final String CONFIG_PROPS_PREFIX = DefaultDependencyCollector.CONFIG_PROPS_PREFIX + NAME + "."; 092 093 /** 094 * The key in the repository session's {@link RepositorySystemSession#getConfigProperties() 095 * configuration properties} used to store a {@link Boolean} flag controlling the resolver's skip mode. 096 * 097 * @since 1.8.0 098 * @configurationSource {@link RepositorySystemSession#getConfigProperties()} 099 * @configurationType {@link java.lang.Boolean} 100 * @configurationDefaultValue {@link #DEFAULT_SKIPPER} 101 */ 102 public static final String CONFIG_PROP_SKIPPER = CONFIG_PROPS_PREFIX + "skipper"; 103 104 /** 105 * The default value for {@link #CONFIG_PROP_SKIPPER}, {@code true}. 106 * 107 * @since 1.8.0 108 */ 109 public static final boolean DEFAULT_SKIPPER = true; 110 111 /** 112 * The count of threads to be used when collecting POMs in parallel. 113 * 114 * @since 1.9.0 115 * @configurationSource {@link RepositorySystemSession#getConfigProperties()} 116 * @configurationType {@link java.lang.Integer} 117 * @configurationDefaultValue {@link #DEFAULT_THREADS} 118 */ 119 public static final String CONFIG_PROP_THREADS = CONFIG_PROPS_PREFIX + "threads"; 120 121 /** 122 * The default value for {@link #CONFIG_PROP_THREADS}, default value 5. 123 * 124 * @since 1.9.0 125 */ 126 public static final int DEFAULT_THREADS = 5; 127 128 @Inject 129 public BfDependencyCollector( 130 RemoteRepositoryManager remoteRepositoryManager, 131 ArtifactDescriptorReader artifactDescriptorReader, 132 VersionRangeResolver versionRangeResolver, 133 Map<String, ArtifactDecoratorFactory> artifactDecoratorFactories) { 134 super(remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver, artifactDecoratorFactories); 135 } 136 137 @SuppressWarnings("checkstyle:parameternumber") 138 @Override 139 protected void doCollectDependencies( 140 RepositorySystemSession session, 141 RequestTrace trace, 142 DataPool pool, 143 DefaultDependencyCollectionContext context, 144 DefaultVersionFilterContext versionContext, 145 CollectRequest request, 146 DependencyNode node, 147 List<RemoteRepository> repositories, 148 List<Dependency> dependencies, 149 List<Dependency> managedDependencies, 150 Results results) 151 throws DependencyCollectionException { 152 boolean useSkip = ConfigUtils.getBoolean(session, DEFAULT_SKIPPER, CONFIG_PROP_SKIPPER); 153 int nThreads = ExecutorUtils.threadCount(session, DEFAULT_THREADS, CONFIG_PROP_THREADS); 154 logger.debug("Using thread pool with {} threads to resolve descriptors.", nThreads); 155 156 if (useSkip) { 157 logger.debug("Collector skip mode enabled"); 158 } 159 160 try (DependencyResolutionSkipper skipper = useSkip 161 ? DependencyResolutionSkipper.defaultSkipper() 162 : DependencyResolutionSkipper.neverSkipper(); 163 ParallelDescriptorResolver parallelDescriptorResolver = new ParallelDescriptorResolver(nThreads)) { 164 Args args = new Args(session, pool, context, versionContext, request, skipper, parallelDescriptorResolver); 165 166 DependencySelector rootDepSelector = session.getDependencySelector() != null 167 ? session.getDependencySelector().deriveChildSelector(context) 168 : null; 169 DependencyManager rootDepManager = session.getDependencyManager() != null 170 ? session.getDependencyManager().deriveChildManager(context) 171 : null; 172 DependencyTraverser rootDepTraverser = session.getDependencyTraverser() != null 173 ? session.getDependencyTraverser().deriveChildTraverser(context) 174 : null; 175 VersionFilter rootVerFilter = session.getVersionFilter() != null 176 ? session.getVersionFilter().deriveChildFilter(context) 177 : null; 178 179 List<DependencyNode> parents = Collections.singletonList(node); 180 for (Dependency dependency : dependencies) { 181 RequestTrace childTrace = 182 collectStepTrace(trace, args.request.getRequestContext(), parents, dependency); 183 DependencyProcessingContext processingContext = new DependencyProcessingContext( 184 rootDepSelector, 185 rootDepManager, 186 rootDepTraverser, 187 rootVerFilter, 188 childTrace, 189 repositories, 190 managedDependencies, 191 parents, 192 dependency, 193 PremanagedDependency.create(rootDepManager, dependency, false, args.premanagedState)); 194 if (!filter(processingContext)) { 195 processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency()); 196 resolveArtifactDescriptorAsync(args, processingContext, results); 197 args.dependencyProcessingQueue.add(processingContext); 198 } 199 } 200 201 while (!args.dependencyProcessingQueue.isEmpty()) { 202 processDependency( 203 args, results, args.dependencyProcessingQueue.remove(), Collections.emptyList(), false); 204 } 205 206 if (args.interruptedException.get() != null) { 207 throw new DependencyCollectionException( 208 results.getResult(), "Collection interrupted", args.interruptedException.get()); 209 } 210 } 211 } 212 213 @SuppressWarnings("checkstyle:parameternumber") 214 private void processDependency( 215 Args args, 216 Results results, 217 DependencyProcessingContext context, 218 List<Artifact> relocations, 219 boolean disableVersionManagement) { 220 if (Thread.interrupted()) { 221 args.interruptedException.set(new InterruptedException()); 222 } 223 if (args.interruptedException.get() != null) { 224 return; 225 } 226 Dependency dependency = context.dependency; 227 PremanagedDependency preManaged = context.premanagedDependency; 228 229 boolean noDescriptor = isLackingDescriptor(args.session, dependency.getArtifact()); 230 boolean traverse = 231 !noDescriptor && (context.depTraverser == null || context.depTraverser.traverseDependency(dependency)); 232 233 Future<DescriptorResolutionResult> resolutionResultFuture = args.resolver.find(dependency.getArtifact()); 234 DescriptorResolutionResult resolutionResult; 235 VersionRangeResult rangeResult; 236 try { 237 resolutionResult = resolutionResultFuture.get(); 238 rangeResult = resolutionResult.rangeResult; 239 } catch (Exception e) { 240 results.addException(dependency, e, context.parents); 241 return; 242 } 243 244 Set<Version> versions = resolutionResult.descriptors.keySet(); 245 for (Version version : versions) { 246 Artifact originalArtifact = dependency.getArtifact().setVersion(version.toString()); 247 Dependency d = dependency.setArtifact(originalArtifact); 248 249 final ArtifactDescriptorResult descriptorResult = resolutionResult.descriptors.get(version); 250 if (descriptorResult != null) { 251 d = d.setArtifact(descriptorResult.getArtifact()); 252 253 int cycleEntry = find(context.parents, d.getArtifact()); 254 if (cycleEntry >= 0) { 255 results.addCycle(context.parents, cycleEntry, d); 256 DependencyNode cycleNode = context.parents.get(cycleEntry); 257 if (cycleNode.getDependency() != null) { 258 DefaultDependencyNode child = createDependencyNode( 259 relocations, preManaged, rangeResult, version, d, descriptorResult, cycleNode); 260 context.getParent().getChildren().add(child); 261 continue; 262 } 263 } 264 265 if (!descriptorResult.getRelocations().isEmpty()) { 266 boolean disableVersionManagementSubsequently = 267 originalArtifact.getGroupId().equals(d.getArtifact().getGroupId()) 268 && originalArtifact 269 .getArtifactId() 270 .equals(d.getArtifact().getArtifactId()); 271 272 PremanagedDependency premanagedDependency = PremanagedDependency.create( 273 context.depManager, d, disableVersionManagementSubsequently, args.premanagedState); 274 DependencyProcessingContext relocatedContext = new DependencyProcessingContext( 275 context.depSelector, 276 context.depManager, 277 context.depTraverser, 278 context.verFilter, 279 context.trace, 280 context.repositories, 281 descriptorResult.getManagedDependencies(), 282 context.parents, 283 d, 284 premanagedDependency); 285 286 if (!filter(relocatedContext)) { 287 relocatedContext.withDependency(premanagedDependency.getManagedDependency()); 288 resolveArtifactDescriptorAsync(args, relocatedContext, results); 289 processDependency( 290 args, 291 results, 292 relocatedContext, 293 descriptorResult.getRelocations(), 294 disableVersionManagementSubsequently); 295 } 296 297 return; 298 } else { 299 d = args.pool.intern(d.setArtifact(args.pool.intern(d.getArtifact()))); 300 301 List<RemoteRepository> repos = 302 getRemoteRepositories(rangeResult.getRepository(version), context.repositories); 303 304 DefaultDependencyNode child = createDependencyNode( 305 relocations, 306 preManaged, 307 rangeResult, 308 version, 309 d, 310 descriptorResult.getAliases(), 311 repos, 312 args.request.getRequestContext()); 313 314 context.getParent().getChildren().add(child); 315 316 boolean recurse = 317 traverse && !descriptorResult.getDependencies().isEmpty(); 318 DependencyProcessingContext parentContext = context.withDependency(d); 319 if (recurse) { 320 doRecurse(args, parentContext, descriptorResult, child, results, disableVersionManagement); 321 } else if (!args.skipper.skipResolution(child, parentContext.parents)) { 322 List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1); 323 parents.addAll(parentContext.parents); 324 parents.add(child); 325 args.skipper.cache(child, parents); 326 } 327 } 328 } else { 329 List<RemoteRepository> repos = 330 getRemoteRepositories(rangeResult.getRepository(version), context.repositories); 331 DefaultDependencyNode child = createDependencyNode( 332 relocations, 333 preManaged, 334 rangeResult, 335 version, 336 d, 337 null, 338 repos, 339 args.request.getRequestContext()); 340 context.getParent().getChildren().add(child); 341 } 342 } 343 } 344 345 @SuppressWarnings("checkstyle:parameternumber") 346 private void doRecurse( 347 Args args, 348 DependencyProcessingContext parentContext, 349 ArtifactDescriptorResult descriptorResult, 350 DefaultDependencyNode child, 351 Results results, 352 boolean disableVersionManagement) { 353 DefaultDependencyCollectionContext context = args.collectionContext.get(); 354 args.collectionContext.compareAndSet( 355 context, context.set(parentContext.dependency, descriptorResult.getManagedDependencies())); 356 context = args.collectionContext.get(); 357 358 DependencySelector childSelector = 359 parentContext.depSelector != null ? parentContext.depSelector.deriveChildSelector(context) : null; 360 DependencyManager childManager = 361 parentContext.depManager != null ? parentContext.depManager.deriveChildManager(context) : null; 362 DependencyTraverser childTraverser = 363 parentContext.depTraverser != null ? parentContext.depTraverser.deriveChildTraverser(context) : null; 364 VersionFilter childFilter = 365 parentContext.verFilter != null ? parentContext.verFilter.deriveChildFilter(context) : null; 366 367 final List<RemoteRepository> childRepos = args.ignoreRepos 368 ? parentContext.repositories 369 : remoteRepositoryManager.aggregateRepositories( 370 args.session, parentContext.repositories, descriptorResult.getRepositories(), true); 371 372 Object key = args.pool.toKey( 373 parentContext.dependency.getArtifact(), 374 childRepos, 375 childSelector, 376 childManager, 377 childTraverser, 378 childFilter); 379 380 List<DependencyNode> children = args.pool.getChildren(key); 381 if (children == null) { 382 boolean skipResolution = args.skipper.skipResolution(child, parentContext.parents); 383 if (!skipResolution) { 384 List<DependencyNode> parents = new ArrayList<>(parentContext.parents.size() + 1); 385 parents.addAll(parentContext.parents); 386 parents.add(child); 387 for (Dependency dependency : descriptorResult.getDependencies()) { 388 RequestTrace childTrace = collectStepTrace( 389 parentContext.trace, args.request.getRequestContext(), parents, dependency); 390 PremanagedDependency premanagedDependency = PremanagedDependency.create( 391 childManager, dependency, disableVersionManagement, args.premanagedState); 392 DependencyProcessingContext processingContext = new DependencyProcessingContext( 393 childSelector, 394 childManager, 395 childTraverser, 396 childFilter, 397 childTrace, 398 childRepos, 399 descriptorResult.getManagedDependencies(), 400 parents, 401 dependency, 402 premanagedDependency); 403 if (!filter(processingContext)) { 404 // resolve descriptors ahead for managed dependency 405 processingContext.withDependency(processingContext.premanagedDependency.getManagedDependency()); 406 resolveArtifactDescriptorAsync(args, processingContext, results); 407 args.dependencyProcessingQueue.add(processingContext); 408 } 409 } 410 args.pool.putChildren(key, child.getChildren()); 411 args.skipper.cache(child, parents); 412 } 413 } else { 414 child.setChildren(children); 415 } 416 } 417 418 private boolean filter(DependencyProcessingContext context) { 419 return context.depSelector != null && !context.depSelector.selectDependency(context.dependency); 420 } 421 422 private void resolveArtifactDescriptorAsync(Args args, DependencyProcessingContext context, Results results) { 423 Dependency dependency = context.dependency; 424 args.resolver.resolveDescriptors(dependency.getArtifact(), () -> { 425 VersionRangeRequest rangeRequest = createVersionRangeRequest( 426 args.request.getRequestContext(), context.trace, context.repositories, dependency); 427 VersionRangeResult rangeResult = cachedResolveRangeResult(rangeRequest, args.pool, args.session); 428 List<? extends Version> versions = 429 filterVersions(dependency, rangeResult, context.verFilter, args.versionContext); 430 431 // resolve newer version first to maximize benefits of skipper 432 Collections.reverse(versions); 433 434 Map<Version, ArtifactDescriptorResult> descriptors = new ConcurrentHashMap<>(versions.size()); 435 Stream<? extends Version> stream = versions.size() > 1 ? versions.parallelStream() : versions.stream(); 436 stream.forEach(version -> Optional.ofNullable( 437 resolveDescriptorForVersion(args, context, results, dependency, version)) 438 .ifPresent(r -> descriptors.put(version, r))); 439 440 DescriptorResolutionResult resolutionResult = 441 new DescriptorResolutionResult(dependency.getArtifact(), rangeResult); 442 // keep original sequence 443 versions.forEach(version -> resolutionResult.descriptors.put(version, descriptors.get(version))); 444 // populate for versions in version range 445 resolutionResult.flatten().forEach(dr -> args.resolver.cacheVersionRangeDescriptor(dr.artifact, dr)); 446 447 return resolutionResult; 448 }); 449 } 450 451 private ArtifactDescriptorResult resolveDescriptorForVersion( 452 Args args, DependencyProcessingContext context, Results results, Dependency dependency, Version version) { 453 Artifact original = dependency.getArtifact(); 454 Artifact newArtifact = original.setVersion(version.toString()); 455 Dependency newDependency = 456 new Dependency(newArtifact, dependency.getScope(), dependency.isOptional(), dependency.getExclusions()); 457 DependencyProcessingContext newContext = context.copy(); 458 459 ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest( 460 args.request.getRequestContext(), context.trace, newContext.repositories, newDependency); 461 return isLackingDescriptor(args.session, newArtifact) 462 ? new ArtifactDescriptorResult(descriptorRequest) 463 : resolveCachedArtifactDescriptor( 464 args.pool, 465 descriptorRequest, 466 args.session, 467 newContext.withDependency(newDependency).dependency, 468 results, 469 context.parents); 470 } 471 472 static class ParallelDescriptorResolver implements Closeable { 473 private final ExecutorService executorService; 474 475 /** 476 * Artifact ID -> Future of DescriptorResolutionResult 477 */ 478 private final Map<String, Future<DescriptorResolutionResult>> results = new ConcurrentHashMap<>(256); 479 480 ParallelDescriptorResolver(int threads) { 481 this.executorService = ExecutorUtils.threadPool(threads, getClass().getSimpleName() + "-"); 482 } 483 484 void resolveDescriptors(Artifact artifact, Callable<DescriptorResolutionResult> callable) { 485 results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> this.executorService.submit(callable)); 486 } 487 488 void cacheVersionRangeDescriptor(Artifact artifact, DescriptorResolutionResult resolutionResult) { 489 results.computeIfAbsent(ArtifactIdUtils.toId(artifact), key -> new DoneFuture<>(resolutionResult)); 490 } 491 492 Future<DescriptorResolutionResult> find(Artifact artifact) { 493 return results.get(ArtifactIdUtils.toId(artifact)); 494 } 495 496 @Override 497 public void close() { 498 executorService.shutdown(); 499 } 500 } 501 502 static class DoneFuture<V> implements Future<V> { 503 private final V v; 504 505 DoneFuture(V v) { 506 this.v = v; 507 } 508 509 @Override 510 public boolean cancel(boolean mayInterruptIfRunning) { 511 return false; 512 } 513 514 @Override 515 public boolean isCancelled() { 516 return false; 517 } 518 519 @Override 520 public boolean isDone() { 521 return true; 522 } 523 524 @Override 525 public V get() throws InterruptedException, ExecutionException { 526 return v; 527 } 528 529 @Override 530 public V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { 531 return v; 532 } 533 } 534 535 static class DescriptorResolutionResult { 536 Artifact artifact; 537 538 VersionRangeResult rangeResult; 539 540 Map<Version, ArtifactDescriptorResult> descriptors; 541 542 DescriptorResolutionResult(Artifact artifact, VersionRangeResult rangeResult) { 543 this.artifact = artifact; 544 this.rangeResult = rangeResult; 545 this.descriptors = new LinkedHashMap<>(rangeResult.getVersions().size()); 546 } 547 548 DescriptorResolutionResult( 549 VersionRangeResult rangeResult, Version version, ArtifactDescriptorResult descriptor) { 550 // NOTE: In case of A1 -> A2 relocation this happens: 551 // ArtifactDescriptorResult read by ArtifactDescriptorResultReader for A1 552 // will return instance that will have artifact = A2 (as RelocatedArtifact). 553 // So to properly "key" this instance, we need to use "originally requested" A1 instead! 554 // In short: 555 // ArtifactDescriptorRequest.artifact != ArtifactDescriptorResult.artifact WHEN relocation in play 556 // otherwise (no relocation), they are EQUAL. 557 this(descriptor.getRequest().getArtifact(), rangeResult); 558 this.descriptors.put(version, descriptor); 559 } 560 561 List<DescriptorResolutionResult> flatten() { 562 if (descriptors.size() > 1) { 563 return descriptors.entrySet().stream() 564 .map(e -> new DescriptorResolutionResult(rangeResult, e.getKey(), e.getValue())) 565 .collect(Collectors.toList()); 566 } else { 567 return Collections.emptyList(); 568 } 569 } 570 } 571 572 static class Args { 573 574 final RepositorySystemSession session; 575 576 final boolean ignoreRepos; 577 578 final boolean premanagedState; 579 580 final DataPool pool; 581 582 final Queue<DependencyProcessingContext> dependencyProcessingQueue = new ArrayDeque<>(128); 583 584 final AtomicReference<DefaultDependencyCollectionContext> collectionContext; 585 586 final DefaultVersionFilterContext versionContext; 587 588 final CollectRequest request; 589 590 final DependencyResolutionSkipper skipper; 591 592 final ParallelDescriptorResolver resolver; 593 594 final AtomicReference<InterruptedException> interruptedException; 595 596 Args( 597 RepositorySystemSession session, 598 DataPool pool, 599 DefaultDependencyCollectionContext collectionContext, 600 DefaultVersionFilterContext versionContext, 601 CollectRequest request, 602 DependencyResolutionSkipper skipper, 603 ParallelDescriptorResolver resolver) { 604 this.session = session; 605 this.request = request; 606 this.ignoreRepos = session.isIgnoreArtifactDescriptorRepositories(); 607 this.premanagedState = ConfigUtils.getBoolean(session, false, DependencyManagerUtils.CONFIG_PROP_VERBOSE); 608 this.pool = pool; 609 this.collectionContext = new AtomicReference<>(collectionContext); 610 this.versionContext = versionContext; 611 this.skipper = skipper; 612 this.resolver = resolver; 613 this.interruptedException = new AtomicReference<>(null); 614 } 615 } 616}