1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.buildcache;
20
21 import javax.annotation.Nonnull;
22 import javax.inject.Inject;
23 import javax.inject.Named;
24 import javax.inject.Provider;
25
26 import java.io.File;
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.lang.reflect.Field;
30 import java.lang.reflect.InvocationTargetException;
31 import java.lang.reflect.Method;
32 import java.nio.charset.StandardCharsets;
33 import java.nio.file.FileVisitResult;
34 import java.nio.file.Files;
35 import java.nio.file.Path;
36 import java.nio.file.Paths;
37 import java.nio.file.SimpleFileVisitor;
38 import java.nio.file.StandardCopyOption;
39 import java.nio.file.attribute.BasicFileAttributes;
40 import java.util.ArrayList;
41 import java.util.Collections;
42 import java.util.HashMap;
43 import java.util.List;
44 import java.util.Map;
45 import java.util.Objects;
46 import java.util.Optional;
47 import java.util.Set;
48 import java.util.TreeSet;
49 import java.util.UUID;
50 import java.util.concurrent.ConcurrentHashMap;
51 import java.util.concurrent.ConcurrentMap;
52 import java.util.concurrent.Future;
53 import java.util.concurrent.FutureTask;
54 import java.util.concurrent.atomic.AtomicBoolean;
55 import java.util.function.UnaryOperator;
56 import java.util.regex.Pattern;
57
58 import org.apache.commons.io.FilenameUtils;
59 import org.apache.commons.lang3.StringUtils;
60 import org.apache.commons.lang3.Strings;
61 import org.apache.commons.lang3.mutable.MutableBoolean;
62 import org.apache.maven.SessionScoped;
63 import org.apache.maven.artifact.handler.ArtifactHandler;
64 import org.apache.maven.artifact.handler.manager.ArtifactHandlerManager;
65 import org.apache.maven.buildcache.artifact.ArtifactRestorationReport;
66 import org.apache.maven.buildcache.artifact.OutputType;
67 import org.apache.maven.buildcache.artifact.RestoredArtifact;
68 import org.apache.maven.buildcache.checksum.MavenProjectInput;
69 import org.apache.maven.buildcache.hash.HashAlgorithm;
70 import org.apache.maven.buildcache.hash.HashFactory;
71 import org.apache.maven.buildcache.xml.Build;
72 import org.apache.maven.buildcache.xml.CacheConfig;
73 import org.apache.maven.buildcache.xml.CacheSource;
74 import org.apache.maven.buildcache.xml.DtoUtils;
75 import org.apache.maven.buildcache.xml.XmlService;
76 import org.apache.maven.buildcache.xml.build.Artifact;
77 import org.apache.maven.buildcache.xml.build.CompletedExecution;
78 import org.apache.maven.buildcache.xml.build.DigestItem;
79 import org.apache.maven.buildcache.xml.build.ProjectsInputInfo;
80 import org.apache.maven.buildcache.xml.build.Scm;
81 import org.apache.maven.buildcache.xml.config.DirName;
82 import org.apache.maven.buildcache.xml.config.PropertyName;
83 import org.apache.maven.buildcache.xml.config.TrackedProperty;
84 import org.apache.maven.buildcache.xml.diff.Diff;
85 import org.apache.maven.buildcache.xml.report.CacheReport;
86 import org.apache.maven.buildcache.xml.report.ProjectReport;
87 import org.apache.maven.execution.MavenSession;
88 import org.apache.maven.execution.MojoExecutionEvent;
89 import org.apache.maven.plugin.MojoExecution;
90 import org.apache.maven.plugin.descriptor.Parameter;
91 import org.apache.maven.project.MavenProject;
92 import org.apache.maven.project.MavenProjectHelper;
93 import org.codehaus.plexus.util.ReflectionUtils;
94 import org.eclipse.aether.RepositorySystem;
95 import org.slf4j.Logger;
96 import org.slf4j.LoggerFactory;
97
98 import static java.nio.file.StandardOpenOption.CREATE;
99 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
100 import static org.apache.commons.lang3.StringUtils.isNotBlank;
101 import static org.apache.commons.lang3.StringUtils.split;
102 import static org.apache.maven.buildcache.CacheResult.empty;
103 import static org.apache.maven.buildcache.CacheResult.failure;
104 import static org.apache.maven.buildcache.CacheResult.partialSuccess;
105 import static org.apache.maven.buildcache.CacheResult.rebuilded;
106 import static org.apache.maven.buildcache.CacheResult.success;
107 import static org.apache.maven.buildcache.RemoteCacheRepository.BUILDINFO_XML;
108 import static org.apache.maven.buildcache.checksum.KeyUtils.getVersionlessProjectKey;
109 import static org.apache.maven.buildcache.checksum.MavenProjectInput.CACHE_IMPLEMENTATION_VERSION;
110
111
112
113
114 @SessionScoped
115 @Named
116 @SuppressWarnings("unused")
117 public class CacheControllerImpl implements CacheController {
118
119 private static final Logger LOGGER = LoggerFactory.getLogger(CacheControllerImpl.class);
120 private static final String DEFAULT_FILE_GLOB = "*";
121 public static final String ERROR_MSG_RESTORATION_OUTSIDE_PROJECT =
122 "Blocked an attempt to restore files outside of a project directory: ";
123
124 private final MavenProjectHelper projectHelper;
125 private final ArtifactHandlerManager artifactHandlerManager;
126 private final XmlService xmlService;
127 private final CacheConfig cacheConfig;
128 private final LocalCacheRepository localCache;
129 private final RemoteCacheRepository remoteCache;
130 private final ConcurrentMap<String, CacheResult> cacheResults = new ConcurrentHashMap<>();
131 private final Provider<LifecyclePhasesHelper> providerLifecyclePhasesHelper;
132 private volatile Map<String, MavenProject> projectIndex;
133 private final ProjectInputCalculator projectInputCalculator;
134 private final RestoredArtifactHandler restoreArtifactHandler;
135 private volatile Scm scm;
136
137
138
139
140
141
142 private final Map<String, Path> attachedResourcesPathsById = new HashMap<>();
143
144 private int attachedResourceCounter = 0;
145
146 @Inject
147 public CacheControllerImpl(
148 MavenProjectHelper projectHelper,
149 RepositorySystem repoSystem,
150 ArtifactHandlerManager artifactHandlerManager,
151 XmlService xmlService,
152 LocalCacheRepository localCache,
153 RemoteCacheRepository remoteCache,
154 CacheConfig cacheConfig,
155 ProjectInputCalculator projectInputCalculator,
156 RestoredArtifactHandler restoreArtifactHandler,
157 Provider<LifecyclePhasesHelper> providerLifecyclePhasesHelper) {
158
159 this.projectHelper = projectHelper;
160 this.localCache = localCache;
161 this.remoteCache = remoteCache;
162 this.cacheConfig = cacheConfig;
163 this.artifactHandlerManager = artifactHandlerManager;
164 this.xmlService = xmlService;
165 this.providerLifecyclePhasesHelper = providerLifecyclePhasesHelper;
166 this.projectInputCalculator = projectInputCalculator;
167 this.restoreArtifactHandler = restoreArtifactHandler;
168 }
169
170 @Override
171 @Nonnull
172 public CacheResult findCachedBuild(
173 MavenSession session, MavenProject project, List<MojoExecution> mojoExecutions, boolean skipCache) {
174 final LifecyclePhasesHelper lifecyclePhasesHelper = providerLifecyclePhasesHelper.get();
175 final String highestPhase = lifecyclePhasesHelper.resolveHighestLifecyclePhase(project, mojoExecutions);
176
177 if (!lifecyclePhasesHelper.isLaterPhaseThanClean(highestPhase)) {
178 return empty();
179 }
180
181 String projectName = getVersionlessProjectKey(project);
182
183 ProjectsInputInfo inputInfo = projectInputCalculator.calculateInput(project);
184
185 final CacheContext context = new CacheContext(project, inputInfo, session);
186
187 CacheResult result = empty(context);
188 if (!skipCache) {
189
190 LOGGER.info("Attempting to restore project {} from build cache", projectName);
191
192
193 if (cacheConfig.isRemoteCacheEnabled()) {
194 result = findCachedBuild(mojoExecutions, context);
195 if (!result.isSuccess() && result.getContext() != null) {
196 LOGGER.info("Remote cache is incomplete or missing, trying local build for {}", projectName);
197 }
198 }
199
200 if (!result.isSuccess() && result.getContext() != null) {
201 CacheResult localBuild = findLocalBuild(mojoExecutions, context);
202 if (localBuild.isSuccess() || (localBuild.isPartialSuccess() && !result.isPartialSuccess())) {
203 result = localBuild;
204 } else {
205 LOGGER.info(
206 "Local build was not found by checksum {} for {}", inputInfo.getChecksum(), projectName);
207 }
208 }
209 } else {
210 LOGGER.info(
211 "Project {} is marked as requiring force rebuild, will skip lookup in build cache", projectName);
212 }
213 cacheResults.put(getVersionlessProjectKey(project), result);
214
215 return result;
216 }
217
218 private CacheResult findCachedBuild(List<MojoExecution> mojoExecutions, CacheContext context) {
219 Optional<Build> cachedBuild = Optional.empty();
220 try {
221 cachedBuild = localCache.findBuild(context);
222 if (cachedBuild.isPresent()) {
223 return analyzeResult(context, mojoExecutions, cachedBuild.get());
224 }
225 } catch (Exception e) {
226 LOGGER.error("Cannot read cached remote build", e);
227 }
228 return cachedBuild.map(build -> failure(build, context)).orElseGet(() -> empty(context));
229 }
230
231 private CacheResult findLocalBuild(List<MojoExecution> mojoExecutions, CacheContext context) {
232 Optional<Build> localBuild = Optional.empty();
233 try {
234 localBuild = localCache.findLocalBuild(context);
235 if (localBuild.isPresent()) {
236 return analyzeResult(context, mojoExecutions, localBuild.get());
237 }
238 } catch (Exception e) {
239 LOGGER.error("Cannot read local build", e);
240 }
241 return localBuild.map(build -> failure(build, context)).orElseGet(() -> empty(context));
242 }
243
244 private CacheResult analyzeResult(CacheContext context, List<MojoExecution> mojoExecutions, Build build) {
245 try {
246 final ProjectsInputInfo inputInfo = context.getInputInfo();
247 String projectName = getVersionlessProjectKey(context.getProject());
248
249 LOGGER.info(
250 "Found cached build, restoring {} from cache by checksum {}", projectName, inputInfo.getChecksum());
251 LOGGER.debug("Cached build details: {}", build);
252
253 final String cacheImplementationVersion = build.getCacheImplementationVersion();
254 if (!CACHE_IMPLEMENTATION_VERSION.equals(cacheImplementationVersion)) {
255 LOGGER.warn(
256 "Maven and cached build implementations mismatch, caching might not work correctly. "
257 + "Implementation version: " + CACHE_IMPLEMENTATION_VERSION + ", cached build: {}",
258 build.getCacheImplementationVersion());
259 }
260
261 final LifecyclePhasesHelper lifecyclePhasesHelper = providerLifecyclePhasesHelper.get();
262 List<MojoExecution> cachedSegment =
263 lifecyclePhasesHelper.getCachedSegment(context.getProject(), mojoExecutions, build);
264 List<MojoExecution> missingMojos = build.getMissingExecutions(cachedSegment);
265 if (!missingMojos.isEmpty()) {
266 LOGGER.warn(
267 "Cached build doesn't contains all requested plugin executions "
268 + "(missing: {}), cannot restore",
269 missingMojos);
270 return failure(build, context);
271 }
272
273 if (!isCachedSegmentPropertiesPresent(context.getProject(), build, cachedSegment)) {
274 LOGGER.info("Cached build violates cache rules, cannot restore");
275 return failure(build, context);
276 }
277
278 final String highestRequestPhase =
279 lifecyclePhasesHelper.resolveHighestLifecyclePhase(context.getProject(), mojoExecutions);
280
281 if (lifecyclePhasesHelper.isLaterPhaseThanBuild(highestRequestPhase, build)
282 && !canIgnoreMissingSegment(context.getProject(), build, mojoExecutions)) {
283 LOGGER.info(
284 "Project {} restored partially. Highest cached goal: {}, requested: {}",
285 projectName,
286 build.getHighestCompletedGoal(),
287 highestRequestPhase);
288 return partialSuccess(build, context);
289 }
290
291 return success(build, context);
292
293 } catch (Exception e) {
294 LOGGER.error("Failed to restore project", e);
295 localCache.clearCache(context);
296 return failure(build, context);
297 }
298 }
299
300 private boolean canIgnoreMissingSegment(MavenProject project, Build info, List<MojoExecution> mojoExecutions) {
301 final LifecyclePhasesHelper lifecyclePhasesHelper = providerLifecyclePhasesHelper.get();
302 final List<MojoExecution> postCachedSegment =
303 lifecyclePhasesHelper.getPostCachedSegment(project, mojoExecutions, info);
304
305 for (MojoExecution mojoExecution : postCachedSegment) {
306 if (!cacheConfig.canIgnore(mojoExecution)) {
307 return false;
308 }
309 }
310 return true;
311 }
312
313 private UnaryOperator<File> createRestorationToDiskConsumer(final MavenProject project, final Artifact artifact) {
314
315 if (cacheConfig.isRestoreOnDiskArtifacts() && MavenProjectInput.isRestoreOnDiskArtifacts(project)) {
316
317 Path restorationPath = project.getBasedir().toPath().resolve(artifact.getFilePath());
318 final AtomicBoolean restored = new AtomicBoolean(false);
319 return file -> {
320
321 if (restored.compareAndSet(false, true)) {
322 verifyRestorationInsideProject(project, restorationPath);
323 try {
324 Files.createDirectories(restorationPath.getParent());
325 Files.copy(file.toPath(), restorationPath, StandardCopyOption.REPLACE_EXISTING);
326 } catch (IOException e) {
327 LOGGER.error("Cannot restore file " + artifact.getFileName(), e);
328 throw new RuntimeException(e);
329 }
330 LOGGER.debug("Restored file on disk ({} to {})", artifact.getFileName(), restorationPath);
331 }
332 return restorationPath.toFile();
333 };
334 }
335
336 return file -> file;
337 }
338
339 private boolean isPathInsideProject(final MavenProject project, Path path) {
340 Path restorationPath = path.toAbsolutePath().normalize();
341 return restorationPath.startsWith(project.getBasedir().toPath());
342 }
343
344 private void verifyRestorationInsideProject(final MavenProject project, Path path) {
345 if (!isPathInsideProject(project, path)) {
346 Path normalized = path.toAbsolutePath().normalize();
347 LOGGER.error(ERROR_MSG_RESTORATION_OUTSIDE_PROJECT + normalized);
348 throw new RuntimeException(ERROR_MSG_RESTORATION_OUTSIDE_PROJECT + normalized);
349 }
350 }
351
352 @Override
353 public ArtifactRestorationReport restoreProjectArtifacts(CacheResult cacheResult) {
354
355 LOGGER.debug("Restore project artifacts");
356 final Build build = cacheResult.getBuildInfo();
357 final CacheContext context = cacheResult.getContext();
358 final MavenProject project = context.getProject();
359 ArtifactRestorationReport restorationReport = new ArtifactRestorationReport();
360
361 try {
362 RestoredArtifact restoredProjectArtifact = null;
363 List<RestoredArtifact> restoredAttachedArtifacts = new ArrayList<>();
364
365 if (build.getArtifact() != null && isNotBlank(build.getArtifact().getFileName())) {
366 final Artifact artifactInfo = build.getArtifact();
367 String originalVersion = artifactInfo.getVersion();
368 artifactInfo.setVersion(project.getVersion());
369
370 final Future<File> downloadTask =
371 createDownloadTask(cacheResult, context, project, artifactInfo, originalVersion);
372 restoredProjectArtifact = restoredArtifact(
373 project.getArtifact(),
374 artifactInfo.getType(),
375 artifactInfo.getClassifier(),
376 downloadTask,
377 createRestorationToDiskConsumer(project, artifactInfo));
378 if (!cacheConfig.isLazyRestore()) {
379 restoredProjectArtifact.getFile();
380 }
381 }
382
383 for (Artifact attachedArtifactInfo : build.getAttachedArtifacts()) {
384 String originalVersion = attachedArtifactInfo.getVersion();
385 attachedArtifactInfo.setVersion(project.getVersion());
386 if (isNotBlank(attachedArtifactInfo.getFileName())) {
387 OutputType outputType = OutputType.fromClassifier(attachedArtifactInfo.getClassifier());
388 if (OutputType.ARTIFACT != outputType) {
389
390
391
392 if (cacheConfig.isRestoreGeneratedSources()
393 && MavenProjectInput.isRestoreGeneratedSources(project)) {
394
395 restorationReport.setRestoredFilesInProjectDirectory(true);
396
397 final Path attachedArtifactFile =
398 localCache.getArtifactFile(context, cacheResult.getSource(), attachedArtifactInfo);
399 restoreGeneratedSources(attachedArtifactInfo, attachedArtifactFile, project);
400 }
401 } else {
402 Future<File> downloadTask = createDownloadTask(
403 cacheResult, context, project, attachedArtifactInfo, originalVersion);
404 final RestoredArtifact restoredAttachedArtifact = restoredArtifact(
405 restoredProjectArtifact == null ? project.getArtifact() : restoredProjectArtifact,
406 attachedArtifactInfo.getType(),
407 attachedArtifactInfo.getClassifier(),
408 downloadTask,
409 createRestorationToDiskConsumer(project, attachedArtifactInfo));
410 if (!cacheConfig.isLazyRestore()) {
411 restoredAttachedArtifact.getFile();
412 }
413 restoredAttachedArtifacts.add(restoredAttachedArtifact);
414 }
415 }
416 }
417
418
419 if (restoredProjectArtifact != null) {
420 project.setArtifact(restoredProjectArtifact);
421
422 if (!project.hasLifecyclePhase("package")) {
423 project.addLifecyclePhase("package");
424 }
425 }
426 restoredAttachedArtifacts.forEach(project::addAttachedArtifact);
427 restorationReport.setSuccess(true);
428 } catch (Exception e) {
429 LOGGER.debug("Cannot restore cache, continuing with normal build.", e);
430 }
431 return restorationReport;
432 }
433
434
435
436
437
438 private RestoredArtifact restoredArtifact(
439 org.apache.maven.artifact.Artifact parent,
440 String artifactType,
441 String artifactClassifier,
442 Future<File> artifactFile,
443 UnaryOperator<File> restoreToDiskConsumer) {
444 ArtifactHandler handler = null;
445
446 if (artifactType != null) {
447 handler = artifactHandlerManager.getArtifactHandler(artifactType);
448 }
449
450 if (handler == null) {
451 handler = artifactHandlerManager.getArtifactHandler("jar");
452 }
453
454
455 RestoredArtifact artifact = new RestoredArtifact(
456 parent, artifactFile, artifactType, artifactClassifier, handler, restoreToDiskConsumer);
457 artifact.setResolved(true);
458
459 return artifact;
460 }
461
462 private Future<File> createDownloadTask(
463 CacheResult cacheResult,
464 CacheContext context,
465 MavenProject project,
466 Artifact artifact,
467 String originalVersion) {
468 final FutureTask<File> downloadTask = new FutureTask<>(() -> {
469 LOGGER.debug("Downloading artifact {}", artifact.getArtifactId());
470 final Path artifactFile = localCache.getArtifactFile(context, cacheResult.getSource(), artifact);
471
472 if (!Files.exists(artifactFile)) {
473 throw new FileNotFoundException("Missing file for cached build, cannot restore. File: " + artifactFile);
474 }
475 LOGGER.debug("Downloaded artifact {} to: {}", artifact.getArtifactId(), artifactFile);
476 return restoreArtifactHandler
477 .adjustArchiveArtifactVersion(project, originalVersion, artifactFile)
478 .toFile();
479 });
480 if (!cacheConfig.isLazyRestore()) {
481 downloadTask.run();
482 }
483 return downloadTask;
484 }
485
486 @Override
487 public void save(
488 CacheResult cacheResult,
489 List<MojoExecution> mojoExecutions,
490 Map<String, MojoExecutionEvent> executionEvents) {
491 CacheContext context = cacheResult.getContext();
492
493 if (context == null || context.getInputInfo() == null) {
494 LOGGER.info("Cannot save project in cache, skipping");
495 return;
496 }
497
498 final MavenProject project = context.getProject();
499 final MavenSession session = context.getSession();
500 try {
501 final HashFactory hashFactory = cacheConfig.getHashFactory();
502 final org.apache.maven.artifact.Artifact projectArtifact = project.getArtifact();
503 final List<org.apache.maven.artifact.Artifact> attachedArtifacts;
504 final List<Artifact> attachedArtifactDtos;
505 final Artifact projectArtifactDto;
506 if (project.hasLifecyclePhase("package")) {
507 final HashAlgorithm algorithm = hashFactory.createAlgorithm();
508 attachGeneratedSources(project);
509 attachOutputs(project);
510 attachedArtifacts = project.getAttachedArtifacts() != null
511 ? project.getAttachedArtifacts()
512 : Collections.emptyList();
513 attachedArtifactDtos = artifactDtos(attachedArtifacts, algorithm, project);
514 projectArtifactDto = artifactDto(project.getArtifact(), algorithm, project);
515 } else {
516 attachedArtifacts = Collections.emptyList();
517 attachedArtifactDtos = new ArrayList<>();
518 projectArtifactDto = null;
519 }
520
521 List<CompletedExecution> completedExecution = buildExecutionInfo(mojoExecutions, executionEvents);
522
523 final Build build = new Build(
524 session.getGoals(),
525 projectArtifactDto,
526 attachedArtifactDtos,
527 context.getInputInfo(),
528 completedExecution,
529 hashFactory.getAlgorithm());
530 populateGitInfo(build, session);
531 build.getDto().set_final(cacheConfig.isSaveToRemoteFinal());
532 cacheResults.put(getVersionlessProjectKey(project), rebuilded(cacheResult, build));
533
534 localCache.beforeSave(context);
535
536
537 if (project.hasLifecyclePhase("package")) {
538 if (projectArtifact.getFile() != null) {
539 localCache.saveArtifactFile(cacheResult, projectArtifact);
540 }
541 for (org.apache.maven.artifact.Artifact attachedArtifact : attachedArtifacts) {
542 if (attachedArtifact.getFile() != null) {
543 boolean storeArtifact =
544 isOutputArtifact(attachedArtifact.getFile().getName());
545 if (storeArtifact) {
546 localCache.saveArtifactFile(cacheResult, attachedArtifact);
547 } else {
548 LOGGER.debug(
549 "Skipping attached project artifact '{}' = "
550 + " it is marked for exclusion from caching",
551 attachedArtifact.getFile().getName());
552 }
553 }
554 }
555 }
556
557 localCache.saveBuildInfo(cacheResult, build);
558
559 if (cacheConfig.isBaselineDiffEnabled()) {
560 produceDiffReport(cacheResult, build);
561 }
562
563 } catch (Exception e) {
564 LOGGER.error("Failed to save project, cleaning cache. Project: {}", project, e);
565 try {
566 localCache.clearCache(context);
567 } catch (Exception ex) {
568 LOGGER.error("Failed to clean cache due to unexpected error:", ex);
569 }
570 }
571 }
572
573 public void produceDiffReport(CacheResult cacheResult, Build build) {
574 MavenProject project = cacheResult.getContext().getProject();
575 Optional<Build> baselineHolder = remoteCache.findBaselineBuild(project);
576 if (baselineHolder.isPresent()) {
577 Build baseline = baselineHolder.get();
578 String outputDirectory = project.getBuild().getDirectory();
579 Path reportOutputDir = Paths.get(outputDirectory, "incremental-maven");
580 LOGGER.info("Saving cache builds diff to: {}", reportOutputDir);
581 Diff diff = new CacheDiff(build.getDto(), baseline.getDto(), cacheConfig).compare();
582 try {
583 Files.createDirectories(reportOutputDir);
584 final ProjectsInputInfo baselineInputs = baseline.getDto().getProjectsInputInfo();
585 final String checksum = baselineInputs.getChecksum();
586 Files.write(
587 reportOutputDir.resolve("buildinfo-baseline-" + checksum + ".xml"),
588 xmlService.toBytes(baseline.getDto()),
589 TRUNCATE_EXISTING,
590 CREATE);
591 Files.write(
592 reportOutputDir.resolve("buildinfo-" + checksum + ".xml"),
593 xmlService.toBytes(build.getDto()),
594 TRUNCATE_EXISTING,
595 CREATE);
596 Files.write(
597 reportOutputDir.resolve("buildsdiff-" + checksum + ".xml"),
598 xmlService.toBytes(diff),
599 TRUNCATE_EXISTING,
600 CREATE);
601 final Optional<DigestItem> pom =
602 CacheDiff.findPom(build.getDto().getProjectsInputInfo());
603 if (pom.isPresent()) {
604 Files.write(
605 reportOutputDir.resolve("effective-pom-" + checksum + ".xml"),
606 pom.get().getValue().getBytes(StandardCharsets.UTF_8),
607 TRUNCATE_EXISTING,
608 CREATE);
609 }
610 final Optional<DigestItem> baselinePom = CacheDiff.findPom(baselineInputs);
611 if (baselinePom.isPresent()) {
612 Files.write(
613 reportOutputDir.resolve("effective-pom-baseline-" + baselineInputs.getChecksum() + ".xml"),
614 baselinePom.get().getValue().getBytes(StandardCharsets.UTF_8),
615 TRUNCATE_EXISTING,
616 CREATE);
617 }
618 } catch (IOException e) {
619 LOGGER.error("Cannot produce build diff for project", e);
620 }
621 } else {
622 LOGGER.info("Cannot find project in baseline build, skipping diff");
623 }
624 }
625
626 private List<Artifact> artifactDtos(
627 List<org.apache.maven.artifact.Artifact> attachedArtifacts, HashAlgorithm digest, MavenProject project)
628 throws IOException {
629 List<Artifact> result = new ArrayList<>();
630 for (org.apache.maven.artifact.Artifact attachedArtifact : attachedArtifacts) {
631 if (attachedArtifact.getFile() != null
632 && isOutputArtifact(attachedArtifact.getFile().getName())) {
633 result.add(artifactDto(attachedArtifact, digest, project));
634 }
635 }
636 return result;
637 }
638
639 private Artifact artifactDto(
640 org.apache.maven.artifact.Artifact projectArtifact, HashAlgorithm algorithm, MavenProject project)
641 throws IOException {
642 final Artifact dto = DtoUtils.createDto(projectArtifact);
643 if (projectArtifact.getFile() != null && projectArtifact.getFile().isFile()) {
644 final Path file = projectArtifact.getFile().toPath();
645 dto.setFileHash(algorithm.hash(file));
646 dto.setFileSize(Files.size(file));
647
648
649 Path relativePath = attachedResourcesPathsById.get(projectArtifact.getClassifier());
650 if (relativePath == null) {
651
652
653 relativePath = project.getBasedir().toPath().relativize(file.toAbsolutePath());
654 }
655 dto.setFilePath(FilenameUtils.separatorsToUnix(relativePath.toString()));
656 }
657 return dto;
658 }
659
660 private List<CompletedExecution> buildExecutionInfo(
661 List<MojoExecution> mojoExecutions, Map<String, MojoExecutionEvent> executionEvents) {
662 List<CompletedExecution> list = new ArrayList<>();
663 for (MojoExecution mojoExecution : mojoExecutions) {
664 final String executionKey = CacheUtils.mojoExecutionKey(mojoExecution);
665 final MojoExecutionEvent executionEvent =
666 executionEvents != null ? executionEvents.get(executionKey) : null;
667 CompletedExecution executionInfo = new CompletedExecution();
668 executionInfo.setExecutionKey(executionKey);
669 executionInfo.setMojoClassName(mojoExecution.getMojoDescriptor().getImplementation());
670 if (executionEvent != null) {
671 recordMojoProperties(executionInfo, executionEvent);
672 }
673 list.add(executionInfo);
674 }
675 return list;
676 }
677
678 private void recordMojoProperties(CompletedExecution execution, MojoExecutionEvent executionEvent) {
679 final MojoExecution mojoExecution = executionEvent.getExecution();
680
681 final boolean logAll = cacheConfig.isLogAllProperties(mojoExecution);
682 List<TrackedProperty> trackedProperties = cacheConfig.getTrackedProperties(mojoExecution);
683 List<PropertyName> noLogProperties = cacheConfig.getNologProperties(mojoExecution);
684 List<PropertyName> forceLogProperties = cacheConfig.getLoggedProperties(mojoExecution);
685 final Object mojo = executionEvent.getMojo();
686
687 final File baseDir = executionEvent.getProject().getBasedir();
688 final String baseDirPath = FilenameUtils.normalizeNoEndSeparator(baseDir.getAbsolutePath()) + File.separator;
689
690 final List<Parameter> parameters = mojoExecution.getMojoDescriptor().getParameters();
691 for (Parameter parameter : parameters) {
692
693 if (!parameter.isEditable()) {
694 continue;
695 }
696
697 final String propertyName = parameter.getName();
698 final boolean tracked = isTracked(propertyName, trackedProperties);
699 if (!tracked && isExcluded(propertyName, logAll, noLogProperties, forceLogProperties)) {
700 continue;
701 }
702
703 try {
704 Field field = ReflectionUtils.getFieldByNameIncludingSuperclasses(propertyName, mojo.getClass());
705 if (field != null) {
706 final Object value = ReflectionUtils.getValueIncludingSuperclasses(propertyName, mojo);
707 DtoUtils.addProperty(execution, propertyName, value, baseDirPath, tracked);
708 continue;
709 }
710
711 Method getter = getGetter(propertyName, mojo.getClass());
712 if (getter != null) {
713 Object value = getter.invoke(mojo);
714 DtoUtils.addProperty(execution, propertyName, value, baseDirPath, tracked);
715 continue;
716 }
717
718 if (LOGGER.isWarnEnabled()) {
719 LOGGER.warn(
720 "Cannot find a Mojo parameter '{}' to read for Mojo {}. This parameter should be ignored.",
721 propertyName,
722 mojoExecution);
723 }
724
725 } catch (IllegalAccessException | InvocationTargetException e) {
726 LOGGER.info("Cannot get property {} value from {}: {}", propertyName, mojo, e.getMessage());
727 if (tracked) {
728 throw new IllegalArgumentException("Property configured in cache introspection config " + "for "
729 + mojo + " is not accessible: " + propertyName);
730 }
731 }
732 }
733 }
734
735 private static Method getGetter(String fieldName, Class<?> clazz) {
736 String getterMethodName = "get" + org.codehaus.plexus.util.StringUtils.capitalizeFirstLetter(fieldName);
737 Method[] methods = clazz.getMethods();
738 for (Method method : methods) {
739 if (method.getName().equals(getterMethodName)
740 && !method.getReturnType().equals(Void.TYPE)
741 && method.getParameterCount() == 0) {
742 return method;
743 }
744 }
745 return null;
746 }
747
748 private boolean isExcluded(
749 String propertyName,
750 boolean logAll,
751 List<PropertyName> excludedProperties,
752 List<PropertyName> forceLogProperties) {
753 if (!forceLogProperties.isEmpty()) {
754 for (PropertyName logProperty : forceLogProperties) {
755 if (Strings.CS.equals(propertyName, logProperty.getPropertyName())) {
756 return false;
757 }
758 }
759 return true;
760 }
761
762 if (!excludedProperties.isEmpty()) {
763 for (PropertyName excludedProperty : excludedProperties) {
764 if (Strings.CS.equals(propertyName, excludedProperty.getPropertyName())) {
765 return true;
766 }
767 }
768 return false;
769 }
770
771 return !logAll;
772 }
773
774 private boolean isTracked(String propertyName, List<TrackedProperty> trackedProperties) {
775 for (TrackedProperty trackedProperty : trackedProperties) {
776 if (Strings.CS.equals(propertyName, trackedProperty.getPropertyName())) {
777 return true;
778 }
779 }
780 return false;
781 }
782
783 private boolean isCachedSegmentPropertiesPresent(
784 MavenProject project, Build build, List<MojoExecution> mojoExecutions) {
785 for (MojoExecution mojoExecution : mojoExecutions) {
786
787 final List<TrackedProperty> trackedProperties = cacheConfig.getTrackedProperties(mojoExecution);
788 final CompletedExecution cachedExecution = build.findMojoExecutionInfo(mojoExecution);
789
790 if (cachedExecution == null) {
791 LOGGER.info(
792 "Execution is not cached. Plugin: {}, goal {}, executionId: {}",
793 mojoExecution.getPlugin(),
794 mojoExecution.getGoal(),
795 mojoExecution.getExecutionId());
796 return false;
797 }
798
799 if (!DtoUtils.containsAllProperties(cachedExecution, trackedProperties)) {
800 LOGGER.warn(
801 "Cached build record doesn't contain all tracked properties. Plugin: {}, goal: {},"
802 + " executionId: {}",
803 mojoExecution.getPlugin(),
804 mojoExecution.getGoal(),
805 mojoExecution.getExecutionId());
806 return false;
807 }
808 }
809 return true;
810 }
811
812 @Override
813 public boolean isForcedExecution(MavenProject project, MojoExecution execution) {
814 if (cacheConfig.isForcedExecution(execution)) {
815 return true;
816 }
817
818 if (StringUtils.isNotBlank(cacheConfig.getAlwaysRunPlugins())) {
819 String[] alwaysRunPluginsList = split(cacheConfig.getAlwaysRunPlugins(), ",");
820 for (String pluginAndGoal : alwaysRunPluginsList) {
821 String[] tokens = pluginAndGoal.split(":");
822 String alwaysRunPlugin = tokens[0];
823 String alwaysRunGoal = tokens.length == 1 ? "*" : tokens[1];
824 if (Objects.equals(execution.getPlugin().getArtifactId(), alwaysRunPlugin)
825 && ("*".equals(alwaysRunGoal) || Objects.equals(execution.getGoal(), alwaysRunGoal))) {
826 return true;
827 }
828 }
829 }
830 return false;
831 }
832
833 @Override
834 public void saveCacheReport(MavenSession session) {
835 try {
836 CacheReport cacheReport = new CacheReport();
837 for (CacheResult result : cacheResults.values()) {
838 ProjectReport projectReport = new ProjectReport();
839 CacheContext context = result.getContext();
840 MavenProject project = context.getProject();
841 projectReport.setGroupId(project.getGroupId());
842 projectReport.setArtifactId(project.getArtifactId());
843 projectReport.setChecksum(context.getInputInfo().getChecksum());
844 boolean checksumMatched = result.getStatus() != RestoreStatus.EMPTY;
845 projectReport.setChecksumMatched(checksumMatched);
846 projectReport.setLifecycleMatched(checksumMatched && result.isSuccess());
847 projectReport.setSource(String.valueOf(result.getSource()));
848 if (result.getSource() == CacheSource.REMOTE) {
849 projectReport.setUrl(remoteCache.getResourceUrl(context, BUILDINFO_XML));
850 } else if (result.getSource() == CacheSource.BUILD && cacheConfig.isSaveToRemote()) {
851 projectReport.setSharedToRemote(true);
852 projectReport.setUrl(remoteCache.getResourceUrl(context, BUILDINFO_XML));
853 }
854 cacheReport.addProject(projectReport);
855 }
856
857 String buildId = UUID.randomUUID().toString();
858 localCache.saveCacheReport(buildId, session, cacheReport);
859 } catch (Exception e) {
860 LOGGER.error("Cannot save incremental build aggregated report", e);
861 }
862 }
863
864 private void populateGitInfo(Build build, MavenSession session) {
865 if (scm == null) {
866 synchronized (this) {
867 if (scm == null) {
868 try {
869 scm = CacheUtils.readGitInfo(session);
870 } catch (IOException e) {
871 scm = new Scm();
872 LOGGER.error("Cannot populate git info", e);
873 }
874 }
875 }
876 }
877 build.getDto().setScm(scm);
878 }
879
880 private boolean zipAndAttachArtifact(MavenProject project, Path dir, String classifier, final String glob)
881 throws IOException {
882 Path temp = Files.createTempFile("maven-incremental-", project.getArtifactId());
883 temp.toFile().deleteOnExit();
884 boolean hasFile = CacheUtils.zip(dir, temp, glob, cacheConfig.isPreservePermissions());
885 if (hasFile) {
886 projectHelper.attachArtifact(project, "zip", classifier, temp.toFile());
887 }
888 return hasFile;
889 }
890
891 private void restoreGeneratedSources(Artifact artifact, Path artifactFilePath, MavenProject project)
892 throws IOException {
893 final Path baseDir = project.getBasedir().toPath();
894 final Path outputDir = baseDir.resolve(FilenameUtils.separatorsToSystem(artifact.getFilePath()));
895 verifyRestorationInsideProject(project, outputDir);
896 if (!Files.exists(outputDir)) {
897 Files.createDirectories(outputDir);
898 }
899 CacheUtils.unzip(artifactFilePath, outputDir, cacheConfig.isPreservePermissions());
900 }
901
902
903 public void attachGeneratedSources(MavenProject project) throws IOException {
904 final Path targetDir = Paths.get(project.getBuild().getDirectory());
905
906 final Path generatedSourcesDir = targetDir.resolve("generated-sources");
907 attachDirIfNotEmpty(generatedSourcesDir, targetDir, project, OutputType.GENERATED_SOURCE, DEFAULT_FILE_GLOB);
908
909 final Path generatedTestSourcesDir = targetDir.resolve("generated-test-sources");
910 attachDirIfNotEmpty(
911 generatedTestSourcesDir, targetDir, project, OutputType.GENERATED_SOURCE, DEFAULT_FILE_GLOB);
912
913 Set<String> sourceRoots = new TreeSet<>();
914 if (project.getCompileSourceRoots() != null) {
915 sourceRoots.addAll(project.getCompileSourceRoots());
916 }
917 if (project.getTestCompileSourceRoots() != null) {
918 sourceRoots.addAll(project.getTestCompileSourceRoots());
919 }
920
921 for (String sourceRoot : sourceRoots) {
922 final Path sourceRootPath = Paths.get(sourceRoot);
923 if (Files.isDirectory(sourceRootPath)
924 && sourceRootPath.startsWith(targetDir)
925 && !(sourceRootPath.startsWith(generatedSourcesDir)
926 || sourceRootPath.startsWith(generatedTestSourcesDir))) {
927 attachDirIfNotEmpty(sourceRootPath, targetDir, project, OutputType.GENERATED_SOURCE, DEFAULT_FILE_GLOB);
928 }
929 }
930 }
931
932 private void attachOutputs(MavenProject project) throws IOException {
933 final List<DirName> attachedDirs = cacheConfig.getAttachedOutputs();
934 for (DirName dir : attachedDirs) {
935 final Path targetDir = Paths.get(project.getBuild().getDirectory());
936 final Path outputDir = targetDir.resolve(dir.getValue());
937 if (isPathInsideProject(project, outputDir)) {
938 attachDirIfNotEmpty(outputDir, targetDir, project, OutputType.EXTRA_OUTPUT, dir.getGlob());
939 } else {
940 LOGGER.warn("Outside project output candidate directory discarded ({})", outputDir.normalize());
941 }
942 }
943 }
944
945 private void attachDirIfNotEmpty(
946 Path candidateSubDir,
947 Path parentDir,
948 MavenProject project,
949 final OutputType attachedOutputType,
950 final String glob)
951 throws IOException {
952 if (Files.isDirectory(candidateSubDir) && hasFiles(candidateSubDir)) {
953 final Path relativePath = project.getBasedir().toPath().relativize(candidateSubDir);
954 attachedResourceCounter++;
955 final String classifier = attachedOutputType.getClassifierPrefix() + attachedResourceCounter;
956 boolean success = zipAndAttachArtifact(project, candidateSubDir, classifier, glob);
957 if (success) {
958 attachedResourcesPathsById.put(classifier, relativePath);
959 LOGGER.debug("Attached directory: {}", candidateSubDir);
960 }
961 }
962 }
963
964 private boolean hasFiles(Path candidateSubDir) throws IOException {
965 final MutableBoolean hasFiles = new MutableBoolean();
966 Files.walkFileTree(candidateSubDir, new SimpleFileVisitor<Path>() {
967
968 @Override
969 public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) {
970 hasFiles.setTrue();
971 return FileVisitResult.TERMINATE;
972 }
973 });
974 return hasFiles.booleanValue();
975 }
976
977 private boolean isOutputArtifact(String name) {
978 List<Pattern> excludePatterns = cacheConfig.getExcludePatterns();
979 for (Pattern pattern : excludePatterns) {
980 if (pattern.matcher(name).matches()) {
981 return false;
982 }
983 }
984 return true;
985 }
986 }