1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.buildcache;
20
21 import javax.annotation.Nonnull;
22 import javax.inject.Inject;
23 import javax.inject.Named;
24
25 import java.io.File;
26 import java.io.IOException;
27 import java.nio.file.DirectoryStream;
28 import java.nio.file.FileVisitResult;
29 import java.nio.file.Files;
30 import java.nio.file.Path;
31 import java.nio.file.Paths;
32 import java.nio.file.SimpleFileVisitor;
33 import java.nio.file.StandardCopyOption;
34 import java.nio.file.attribute.BasicFileAttributes;
35 import java.nio.file.attribute.FileTime;
36 import java.util.ArrayList;
37 import java.util.Collection;
38 import java.util.Comparator;
39 import java.util.HashMap;
40 import java.util.LinkedList;
41 import java.util.List;
42 import java.util.Map;
43 import java.util.Optional;
44 import java.util.concurrent.ConcurrentHashMap;
45 import java.util.stream.Collectors;
46
47 import org.apache.commons.io.FileUtils;
48 import org.apache.commons.lang3.tuple.Pair;
49 import org.apache.maven.SessionScoped;
50 import org.apache.maven.buildcache.xml.Build;
51 import org.apache.maven.buildcache.xml.CacheConfig;
52 import org.apache.maven.buildcache.xml.CacheSource;
53 import org.apache.maven.buildcache.xml.XmlService;
54 import org.apache.maven.buildcache.xml.build.Artifact;
55 import org.apache.maven.buildcache.xml.build.Scm;
56 import org.apache.maven.buildcache.xml.report.CacheReport;
57 import org.apache.maven.execution.MavenSession;
58 import org.apache.maven.model.Dependency;
59 import org.apache.maven.project.MavenProject;
60 import org.slf4j.Logger;
61 import org.slf4j.LoggerFactory;
62
63 import static java.nio.file.StandardOpenOption.CREATE;
64 import static java.nio.file.StandardOpenOption.CREATE_NEW;
65 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
66 import static java.util.concurrent.TimeUnit.DAYS;
67 import static java.util.concurrent.TimeUnit.HOURS;
68 import static java.util.concurrent.TimeUnit.MINUTES;
69 import static org.apache.commons.lang3.StringUtils.isNotBlank;
70 import static org.apache.maven.buildcache.CacheUtils.getMultimoduleRoot;
71 import static org.apache.maven.buildcache.checksum.MavenProjectInput.CACHE_IMPLEMENTATION_VERSION;
72
73
74
75
76 @SessionScoped
77 @Named
78 @SuppressWarnings("unused")
79 public class LocalCacheRepositoryImpl implements LocalCacheRepository {
80
81 private static final String BUILDINFO_XML = "buildinfo.xml";
82 private static final String LOOKUPINFO_XML = "lookupinfo.xml";
83 private static final long ONE_HOUR_MILLIS = HOURS.toMillis(1);
84 private static final long ONE_MINUTE_MILLIS = MINUTES.toMillis(1);
85 private static final long ONE_DAY_MILLIS = DAYS.toMillis(1);
86 private static final String EMPTY = "";
87
88 private static final Logger LOGGER = LoggerFactory.getLogger(LocalCacheRepositoryImpl.class);
89
90 private final RemoteCacheRepository remoteRepository;
91 private final XmlService xmlService;
92 private final CacheConfig cacheConfig;
93 private final Map<Pair<MavenSession, Dependency>, Optional<Build>> bestBuildCache = new ConcurrentHashMap<>();
94
95 @Inject
96 public LocalCacheRepositoryImpl(
97 RemoteCacheRepository remoteRepository, XmlService xmlService, CacheConfig cacheConfig) {
98 this.remoteRepository = remoteRepository;
99 this.xmlService = xmlService;
100 this.cacheConfig = cacheConfig;
101 }
102
103 @Nonnull
104 @Override
105 public Optional<Build> findLocalBuild(CacheContext context) throws IOException {
106 Path localBuildInfoPath = localBuildPath(context, BUILDINFO_XML, false);
107 LOGGER.debug("Checking local build info: {}", localBuildInfoPath);
108 if (Files.exists(localBuildInfoPath)) {
109 LOGGER.info(
110 "Local build found by checksum {}", context.getInputInfo().getChecksum());
111 try {
112 org.apache.maven.buildcache.xml.build.Build dto = xmlService.loadBuild(localBuildInfoPath.toFile());
113 return Optional.of(new Build(dto, CacheSource.LOCAL));
114 } catch (Exception e) {
115 LOGGER.info("Local build info is not valid, deleting: {}", localBuildInfoPath, e);
116 Files.delete(localBuildInfoPath);
117 }
118 }
119 return Optional.empty();
120 }
121
122 @Nonnull
123 @Override
124 public Optional<Build> findBuild(CacheContext context) throws IOException {
125 Path buildInfoPath = remoteBuildPath(context, BUILDINFO_XML);
126 LOGGER.debug("Checking if build is already downloaded: {}", buildInfoPath);
127
128 if (Files.exists(buildInfoPath)) {
129 LOGGER.info(
130 "Downloaded build found by checksum {}",
131 context.getInputInfo().getChecksum());
132 try {
133 org.apache.maven.buildcache.xml.build.Build dto = xmlService.loadBuild(buildInfoPath.toFile());
134 return Optional.of(new Build(dto, CacheSource.REMOTE));
135 } catch (Exception e) {
136 LOGGER.info("Downloaded build info is not valid, deleting: {}", buildInfoPath, e);
137 Files.delete(buildInfoPath);
138 }
139 }
140
141 if (!cacheConfig.isRemoteCacheEnabled()) {
142 return Optional.empty();
143 }
144
145 try {
146 Path lookupInfoPath = remoteBuildPath(context, LOOKUPINFO_XML);
147 if (Files.exists(lookupInfoPath)) {
148 final BasicFileAttributes fileAttributes =
149 Files.readAttributes(lookupInfoPath, BasicFileAttributes.class);
150 final long lastModified = fileAttributes.lastModifiedTime().toMillis();
151 final long created = fileAttributes.creationTime().toMillis();
152 final long now = System.currentTimeMillis();
153
154 if (now < created + ONE_HOUR_MILLIS
155 && now < lastModified + ONE_MINUTE_MILLIS) {
156 LOGGER.info("Skipping remote lookup, last unsuccessful lookup less than 1m ago.");
157 return Optional.empty();
158 } else if (now < created + ONE_DAY_MILLIS
159 && now < lastModified + ONE_HOUR_MILLIS) {
160 LOGGER.info("Skipping remote lookup, last unsuccessful lookup less than 1h ago.");
161 return Optional.empty();
162 } else if (now > created + ONE_DAY_MILLIS
163 && now < lastModified + ONE_DAY_MILLIS) {
164 LOGGER.info("Skipping remote lookup, last unsuccessful lookup less than 1d ago.");
165 return Optional.empty();
166 }
167 }
168
169 final Optional<Build> build = remoteRepository.findBuild(context);
170 if (build.isPresent()) {
171 LOGGER.info("Build info downloaded from remote repo, saving to: {}", buildInfoPath);
172 Files.createDirectories(buildInfoPath.getParent());
173 Files.write(buildInfoPath, xmlService.toBytes(build.get().getDto()), CREATE_NEW);
174 } else {
175 FileUtils.touch(lookupInfoPath.toFile());
176 }
177 return build;
178 } catch (Exception e) {
179 LOGGER.error("Remote build info is not valid, cached data is not compatible", e);
180 return Optional.empty();
181 }
182 }
183
184 @Override
185 public void clearCache(CacheContext context) {
186 try {
187 final Path buildCacheDir = buildCacheDir(context);
188 Path artifactCacheDir = buildCacheDir.getParent();
189
190 if (!Files.exists(artifactCacheDir)) {
191 return;
192 }
193
194 List<Path> cacheDirs = new ArrayList<>();
195 try (DirectoryStream<Path> paths = Files.newDirectoryStream(artifactCacheDir)) {
196 for (Path dir : paths) {
197 if (Files.isDirectory(dir)) {
198 cacheDirs.add(dir);
199 }
200 }
201 }
202 int maxLocalBuildsCached = cacheConfig.getMaxLocalBuildsCached() - 1;
203 if (cacheDirs.size() > maxLocalBuildsCached) {
204 cacheDirs.sort(Comparator.comparing(LocalCacheRepositoryImpl::lastModifiedTime));
205 for (Path dir : cacheDirs.subList(0, cacheDirs.size() - maxLocalBuildsCached)) {
206 FileUtils.deleteDirectory(dir.toFile());
207 }
208 }
209 final Path path = localBuildDir(context);
210 if (Files.exists(path)) {
211 FileUtils.deleteDirectory(path.toFile());
212 }
213 } catch (IOException e) {
214 final String artifactId = context.getProject().getArtifactId();
215 throw new RuntimeException(
216 "Failed to cleanup local cache of " + artifactId + " on build failure, it might be inconsistent",
217 e);
218 }
219 }
220
221 @Nonnull
222 @Override
223 public Optional<Build> findBestMatchingBuild(MavenSession session, Dependency dependency) {
224 return bestBuildCache.computeIfAbsent(Pair.of(session, dependency), this::findBestMatchingBuildImpl);
225 }
226
227 @Nonnull
228 private Optional<Build> findBestMatchingBuildImpl(Pair<MavenSession, Dependency> dependencySession) {
229 try {
230 final MavenSession session = dependencySession.getLeft();
231 final Dependency dependency = dependencySession.getRight();
232
233 final Path artifactCacheDir =
234 artifactCacheDir(session, dependency.getGroupId(), dependency.getArtifactId());
235
236 final Map<Pair<String, String>, Collection<Pair<Build, Path>>> filesByVersion = new HashMap<>();
237
238 Files.walkFileTree(artifactCacheDir, new SimpleFileVisitor<Path>() {
239
240 @Override
241 public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) {
242 final File file = path.toFile();
243 if (file.getName().equals(BUILDINFO_XML)) {
244 try {
245 final org.apache.maven.buildcache.xml.build.Build dto = xmlService.loadBuild(file);
246 final Pair<Build, Path> buildInfoAndFile = Pair.of(new Build(dto, CacheSource.LOCAL), path);
247 final String cachedVersion = dto.getArtifact().getVersion();
248 final String cachedBranch = getScmRef(dto.getScm());
249 add(filesByVersion, Pair.of(cachedVersion, cachedBranch), buildInfoAndFile);
250 if (isNotBlank(cachedBranch)) {
251 add(filesByVersion, Pair.of(EMPTY, cachedBranch), buildInfoAndFile);
252 }
253 if (isNotBlank(cachedVersion)) {
254 add(filesByVersion, Pair.of(cachedVersion, EMPTY), buildInfoAndFile);
255 }
256 } catch (Exception e) {
257
258 LOGGER.info(
259 "Build info is not compatible to current maven " + "implementation: {}", file, e);
260 }
261 }
262 return FileVisitResult.CONTINUE;
263 }
264 });
265
266 if (filesByVersion.isEmpty()) {
267 return Optional.empty();
268 }
269
270 final String currentRef = getScmRef(CacheUtils.readGitInfo(session));
271
272 Collection<Pair<Build, Path>> bestMatched = new LinkedList<>();
273 if (isNotBlank(currentRef)) {
274 bestMatched = filesByVersion.get(Pair.of(dependency.getVersion(), currentRef));
275 }
276 if (bestMatched.isEmpty()) {
277
278 bestMatched = filesByVersion.get(Pair.of(dependency.getVersion(), EMPTY));
279 }
280 if (bestMatched.isEmpty() && isNotBlank(currentRef)) {
281
282 bestMatched = filesByVersion.get(Pair.of(EMPTY, currentRef));
283 }
284 if (bestMatched.isEmpty()) {
285
286 bestMatched = filesByVersion.values().stream()
287 .flatMap(Collection::stream)
288 .collect(Collectors.toList());
289 }
290
291 return bestMatched.stream()
292 .max(Comparator.comparing(p -> lastModifiedTime(p.getRight())))
293 .map(Pair::getLeft);
294 } catch (IOException e) {
295 LOGGER.info("Cannot find dependency in cache", e);
296 return Optional.empty();
297 }
298 }
299
300 private String getScmRef(Scm scm) {
301 if (scm != null) {
302 return scm.getSourceBranch() != null ? scm.getSourceBranch() : scm.getRevision();
303 } else {
304 return EMPTY;
305 }
306 }
307
308 @Override
309 public Path getArtifactFile(CacheContext context, CacheSource source, Artifact artifact) throws IOException {
310 if (source == CacheSource.LOCAL) {
311 return localBuildPath(context, artifact.getFileName(), false);
312 } else {
313 Path cachePath = remoteBuildPath(context, artifact.getFileName());
314 if (!Files.exists(cachePath) && cacheConfig.isRemoteCacheEnabled()) {
315 if (!remoteRepository.getArtifactContent(context, artifact, cachePath)) {
316 Files.deleteIfExists(cachePath);
317 }
318 }
319 return cachePath;
320 }
321 }
322
323 @Override
324 public void beforeSave(CacheContext environment) {
325 clearCache(environment);
326 }
327
328 @Override
329 public void saveBuildInfo(CacheResult cacheResult, Build build) throws IOException {
330 final Path path = localBuildPath(cacheResult.getContext(), BUILDINFO_XML, true);
331 Files.write(path, xmlService.toBytes(build.getDto()), TRUNCATE_EXISTING, CREATE);
332 LOGGER.info("Saved Build to local file: {}", path);
333 if (cacheConfig.isSaveToRemote() && !cacheResult.isFinal()) {
334 remoteRepository.saveBuildInfo(cacheResult, build);
335 }
336 }
337
338 @Override
339 public void saveCacheReport(String buildId, MavenSession session, CacheReport cacheReport) throws IOException {
340 Path path = getMultimoduleRoot(session).resolve("target").resolve("maven-incremental");
341 Files.createDirectories(path);
342 Path reportPath = path.resolve("cache-report." + buildId + ".xml");
343 Files.write(reportPath, xmlService.toBytes(cacheReport), TRUNCATE_EXISTING, CREATE);
344 LOGGER.debug("Save cache-report to local file: {}", reportPath);
345 if (cacheConfig.isSaveToRemote()) {
346 LOGGER.info("Saving cache report on build completion");
347 remoteRepository.saveCacheReport(buildId, session, cacheReport);
348 }
349 }
350
351 @Override
352 public void saveArtifactFile(CacheResult cacheResult, org.apache.maven.artifact.Artifact artifact)
353 throws IOException {
354
355 File artifactFile = artifact.getFile();
356 Path cachePath = localBuildPath(cacheResult.getContext(), CacheUtils.normalizedName(artifact), true);
357 Files.copy(artifactFile.toPath(), cachePath, StandardCopyOption.REPLACE_EXISTING);
358 if (cacheConfig.isSaveToRemote() && !cacheResult.isFinal()) {
359 remoteRepository.saveArtifactFile(cacheResult, artifact);
360 }
361 }
362
363 private Path buildCacheDir(CacheContext context) throws IOException {
364 final MavenProject project = context.getProject();
365 final Path artifactCacheDir =
366 artifactCacheDir(context.getSession(), project.getGroupId(), project.getArtifactId());
367 return artifactCacheDir.resolve(context.getInputInfo().getChecksum());
368 }
369
370 private Path artifactCacheDir(MavenSession session, String groupId, String artifactId) throws IOException {
371 final Path vga = Paths.get(CACHE_IMPLEMENTATION_VERSION, groupId, artifactId);
372 final Path path = baseDir(session).resolve(vga);
373 Files.createDirectories(path);
374 return path;
375 }
376
377 private Path baseDir(MavenSession session) {
378 String loc = cacheConfig.getLocalRepositoryLocation();
379 if (loc != null) {
380 return Paths.get(loc);
381 } else {
382 return Paths.get(session.getLocalRepository().getBasedir())
383 .getParent()
384 .resolve("build-cache");
385 }
386 }
387
388 private Path remoteBuildPath(CacheContext context, String filename) throws IOException {
389 return remoteBuildDir(context).resolve(filename);
390 }
391
392 private Path localBuildPath(CacheContext context, String filename, boolean createDir) throws IOException {
393 final Path localBuildDir = localBuildDir(context);
394 if (createDir) {
395 Files.createDirectories(localBuildDir);
396 }
397 return localBuildDir.resolve(filename);
398 }
399
400 private Path remoteBuildDir(CacheContext context) throws IOException {
401 return buildCacheDir(context).resolve(cacheConfig.getId());
402 }
403
404 private Path localBuildDir(CacheContext context) throws IOException {
405 return buildCacheDir(context).resolve("local");
406 }
407
408 private static FileTime lastModifiedTime(Path p) {
409 try {
410 return Files.getLastModifiedTime(p);
411 } catch (IOException e) {
412 return FileTime.fromMillis(0);
413 }
414 }
415
416 private static <K, V> void add(Map<K, Collection<V>> map, K key, V value) {
417 map.computeIfAbsent(key, k -> new ArrayList<>()).add(value);
418 }
419 }