View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.buildcache;
20  
21  import javax.annotation.Nonnull;
22  import javax.inject.Inject;
23  import javax.inject.Named;
24  
25  import java.io.File;
26  import java.io.IOException;
27  import java.nio.file.DirectoryStream;
28  import java.nio.file.FileVisitResult;
29  import java.nio.file.Files;
30  import java.nio.file.Path;
31  import java.nio.file.Paths;
32  import java.nio.file.SimpleFileVisitor;
33  import java.nio.file.StandardCopyOption;
34  import java.nio.file.attribute.BasicFileAttributes;
35  import java.nio.file.attribute.FileTime;
36  import java.util.ArrayList;
37  import java.util.Collection;
38  import java.util.Comparator;
39  import java.util.HashMap;
40  import java.util.LinkedList;
41  import java.util.List;
42  import java.util.Map;
43  import java.util.Optional;
44  import java.util.concurrent.ConcurrentHashMap;
45  import java.util.stream.Collectors;
46  
47  import org.apache.commons.io.FileUtils;
48  import org.apache.commons.lang3.tuple.Pair;
49  import org.apache.maven.SessionScoped;
50  import org.apache.maven.buildcache.xml.Build;
51  import org.apache.maven.buildcache.xml.CacheConfig;
52  import org.apache.maven.buildcache.xml.CacheSource;
53  import org.apache.maven.buildcache.xml.XmlService;
54  import org.apache.maven.buildcache.xml.build.Artifact;
55  import org.apache.maven.buildcache.xml.build.Scm;
56  import org.apache.maven.buildcache.xml.report.CacheReport;
57  import org.apache.maven.execution.MavenSession;
58  import org.apache.maven.model.Dependency;
59  import org.apache.maven.project.MavenProject;
60  import org.slf4j.Logger;
61  import org.slf4j.LoggerFactory;
62  
63  import static java.nio.file.StandardOpenOption.CREATE;
64  import static java.nio.file.StandardOpenOption.CREATE_NEW;
65  import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
66  import static java.util.concurrent.TimeUnit.DAYS;
67  import static java.util.concurrent.TimeUnit.HOURS;
68  import static java.util.concurrent.TimeUnit.MINUTES;
69  import static org.apache.commons.lang3.StringUtils.isNotBlank;
70  import static org.apache.maven.buildcache.CacheUtils.getMultimoduleRoot;
71  import static org.apache.maven.buildcache.checksum.MavenProjectInput.CACHE_IMPLEMENTATION_VERSION;
72  
73  /**
74   * Local cache repository implementation.
75   */
76  @SessionScoped
77  @Named
78  @SuppressWarnings("unused")
79  public class LocalCacheRepositoryImpl implements LocalCacheRepository {
80  
81      private static final String BUILDINFO_XML = "buildinfo.xml";
82      private static final String LOOKUPINFO_XML = "lookupinfo.xml";
83      private static final long ONE_HOUR_MILLIS = HOURS.toMillis(1);
84      private static final long ONE_MINUTE_MILLIS = MINUTES.toMillis(1);
85      private static final long ONE_DAY_MILLIS = DAYS.toMillis(1);
86      private static final String EMPTY = "";
87  
88      private static final Logger LOGGER = LoggerFactory.getLogger(LocalCacheRepositoryImpl.class);
89  
90      private final RemoteCacheRepository remoteRepository;
91      private final XmlService xmlService;
92      private final CacheConfig cacheConfig;
93      private final Map<Pair<MavenSession, Dependency>, Optional<Build>> bestBuildCache = new ConcurrentHashMap<>();
94  
95      @Inject
96      public LocalCacheRepositoryImpl(
97              RemoteCacheRepository remoteRepository, XmlService xmlService, CacheConfig cacheConfig) {
98          this.remoteRepository = remoteRepository;
99          this.xmlService = xmlService;
100         this.cacheConfig = cacheConfig;
101     }
102 
103     @Nonnull
104     @Override
105     public Optional<Build> findLocalBuild(CacheContext context) throws IOException {
106         Path localBuildInfoPath = localBuildPath(context, BUILDINFO_XML, false);
107         LOGGER.debug("Checking local build info: {}", localBuildInfoPath);
108         if (Files.exists(localBuildInfoPath)) {
109             LOGGER.info(
110                     "Local build found by checksum {}", context.getInputInfo().getChecksum());
111             try {
112                 org.apache.maven.buildcache.xml.build.Build dto = xmlService.loadBuild(localBuildInfoPath.toFile());
113                 return Optional.of(new Build(dto, CacheSource.LOCAL));
114             } catch (Exception e) {
115                 LOGGER.info("Local build info is not valid, deleting: {}", localBuildInfoPath, e);
116                 Files.delete(localBuildInfoPath);
117             }
118         }
119         return Optional.empty();
120     }
121 
122     @Nonnull
123     @Override
124     public Optional<Build> findBuild(CacheContext context) throws IOException {
125         Path buildInfoPath = remoteBuildPath(context, BUILDINFO_XML);
126         LOGGER.debug("Checking if build is already downloaded: {}", buildInfoPath);
127 
128         if (Files.exists(buildInfoPath)) {
129             LOGGER.info(
130                     "Downloaded build found by checksum {}",
131                     context.getInputInfo().getChecksum());
132             try {
133                 org.apache.maven.buildcache.xml.build.Build dto = xmlService.loadBuild(buildInfoPath.toFile());
134                 return Optional.of(new Build(dto, CacheSource.REMOTE));
135             } catch (Exception e) {
136                 LOGGER.info("Downloaded build info is not valid, deleting: {}", buildInfoPath, e);
137                 Files.delete(buildInfoPath);
138             }
139         }
140 
141         if (!cacheConfig.isRemoteCacheEnabled()) {
142             return Optional.empty();
143         }
144 
145         try {
146             Path lookupInfoPath = remoteBuildPath(context, LOOKUPINFO_XML);
147             if (Files.exists(lookupInfoPath)) {
148                 final BasicFileAttributes fileAttributes =
149                         Files.readAttributes(lookupInfoPath, BasicFileAttributes.class);
150                 final long lastModified = fileAttributes.lastModifiedTime().toMillis();
151                 final long created = fileAttributes.creationTime().toMillis();
152                 final long now = System.currentTimeMillis();
153                 //  throttle remote cache calls, maven like
154                 if (now < created + ONE_HOUR_MILLIS
155                         && now < lastModified + ONE_MINUTE_MILLIS) { // fresh file, allow lookup every minute
156                     LOGGER.info("Skipping remote lookup, last unsuccessful lookup less than 1m ago.");
157                     return Optional.empty();
158                 } else if (now < created + ONE_DAY_MILLIS
159                         && now < lastModified + ONE_HOUR_MILLIS) { // less than 1 day file, allow 1 per hour lookup
160                     LOGGER.info("Skipping remote lookup, last unsuccessful lookup less than 1h ago.");
161                     return Optional.empty();
162                 } else if (now > created + ONE_DAY_MILLIS
163                         && now < lastModified + ONE_DAY_MILLIS) { // more than 1 day file, allow 1 per day lookup
164                     LOGGER.info("Skipping remote lookup, last unsuccessful lookup less than 1d ago.");
165                     return Optional.empty();
166                 }
167             }
168 
169             final Optional<Build> build = remoteRepository.findBuild(context);
170             if (build.isPresent()) {
171                 LOGGER.info("Build info downloaded from remote repo, saving to: {}", buildInfoPath);
172                 Files.createDirectories(buildInfoPath.getParent());
173                 Files.write(buildInfoPath, xmlService.toBytes(build.get().getDto()), CREATE_NEW);
174             } else {
175                 FileUtils.touch(lookupInfoPath.toFile());
176             }
177             return build;
178         } catch (Exception e) {
179             LOGGER.error("Remote build info is not valid, cached data is not compatible", e);
180             return Optional.empty();
181         }
182     }
183 
184     @Override
185     public void clearCache(CacheContext context) {
186         try {
187             final Path buildCacheDir = buildCacheDir(context);
188             Path artifactCacheDir = buildCacheDir.getParent();
189 
190             if (!Files.exists(artifactCacheDir)) {
191                 return;
192             }
193 
194             List<Path> cacheDirs = new ArrayList<>();
195             try (DirectoryStream<Path> paths = Files.newDirectoryStream(artifactCacheDir)) {
196                 for (Path dir : paths) {
197                     if (Files.isDirectory(dir)) {
198                         cacheDirs.add(dir);
199                     }
200                 }
201             }
202             int maxLocalBuildsCached = cacheConfig.getMaxLocalBuildsCached() - 1;
203             if (cacheDirs.size() > maxLocalBuildsCached) {
204                 cacheDirs.sort(Comparator.comparing(LocalCacheRepositoryImpl::lastModifiedTime));
205                 for (Path dir : cacheDirs.subList(0, cacheDirs.size() - maxLocalBuildsCached)) {
206                     FileUtils.deleteDirectory(dir.toFile());
207                 }
208             }
209             final Path path = localBuildDir(context);
210             if (Files.exists(path)) {
211                 FileUtils.deleteDirectory(path.toFile());
212             }
213         } catch (IOException e) {
214             final String artifactId = context.getProject().getArtifactId();
215             throw new RuntimeException(
216                     "Failed to cleanup local cache of " + artifactId + " on build failure, it might be inconsistent",
217                     e);
218         }
219     }
220 
221     @Nonnull
222     @Override
223     public Optional<Build> findBestMatchingBuild(MavenSession session, Dependency dependency) {
224         return bestBuildCache.computeIfAbsent(Pair.of(session, dependency), this::findBestMatchingBuildImpl);
225     }
226 
227     @Nonnull
228     private Optional<Build> findBestMatchingBuildImpl(Pair<MavenSession, Dependency> dependencySession) {
229         try {
230             final MavenSession session = dependencySession.getLeft();
231             final Dependency dependency = dependencySession.getRight();
232 
233             final Path artifactCacheDir =
234                     artifactCacheDir(session, dependency.getGroupId(), dependency.getArtifactId());
235 
236             final Map<Pair<String, String>, Collection<Pair<Build, Path>>> filesByVersion = new HashMap<>();
237 
238             Files.walkFileTree(artifactCacheDir, new SimpleFileVisitor<Path>() {
239 
240                 @Override
241                 public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) {
242                     final File file = path.toFile();
243                     if (file.getName().equals(BUILDINFO_XML)) {
244                         try {
245                             final org.apache.maven.buildcache.xml.build.Build dto = xmlService.loadBuild(file);
246                             final Pair<Build, Path> buildInfoAndFile = Pair.of(new Build(dto, CacheSource.LOCAL), path);
247                             final String cachedVersion = dto.getArtifact().getVersion();
248                             final String cachedBranch = getScmRef(dto.getScm());
249                             add(filesByVersion, Pair.of(cachedVersion, cachedBranch), buildInfoAndFile);
250                             if (isNotBlank(cachedBranch)) {
251                                 add(filesByVersion, Pair.of(EMPTY, cachedBranch), buildInfoAndFile);
252                             }
253                             if (isNotBlank(cachedVersion)) {
254                                 add(filesByVersion, Pair.of(cachedVersion, EMPTY), buildInfoAndFile);
255                             }
256                         } catch (Exception e) {
257                             // version is unusable nothing we can do here
258                             LOGGER.info(
259                                     "Build info is not compatible to current maven " + "implementation: {}", file, e);
260                         }
261                     }
262                     return FileVisitResult.CONTINUE;
263                 }
264             });
265 
266             if (filesByVersion.isEmpty()) {
267                 return Optional.empty();
268             }
269 
270             final String currentRef = getScmRef(CacheUtils.readGitInfo(session));
271             // first lets try by branch and version
272             Collection<Pair<Build, Path>> bestMatched = new LinkedList<>();
273             if (isNotBlank(currentRef)) {
274                 bestMatched = filesByVersion.get(Pair.of(dependency.getVersion(), currentRef));
275             }
276             if (bestMatched.isEmpty()) {
277                 // then by version
278                 bestMatched = filesByVersion.get(Pair.of(dependency.getVersion(), EMPTY));
279             }
280             if (bestMatched.isEmpty() && isNotBlank(currentRef)) {
281                 // then by branch
282                 bestMatched = filesByVersion.get(Pair.of(EMPTY, currentRef));
283             }
284             if (bestMatched.isEmpty()) {
285                 // ok lets take all
286                 bestMatched = filesByVersion.values().stream()
287                         .flatMap(Collection::stream)
288                         .collect(Collectors.toList());
289             }
290 
291             return bestMatched.stream()
292                     .max(Comparator.comparing(p -> lastModifiedTime(p.getRight())))
293                     .map(Pair::getLeft);
294         } catch (IOException e) {
295             LOGGER.info("Cannot find dependency in cache", e);
296             return Optional.empty();
297         }
298     }
299 
300     private String getScmRef(Scm scm) {
301         if (scm != null) {
302             return scm.getSourceBranch() != null ? scm.getSourceBranch() : scm.getRevision();
303         } else {
304             return EMPTY;
305         }
306     }
307 
308     @Override
309     public Path getArtifactFile(CacheContext context, CacheSource source, Artifact artifact) throws IOException {
310         if (source == CacheSource.LOCAL) {
311             return localBuildPath(context, artifact.getFileName(), false);
312         } else {
313             Path cachePath = remoteBuildPath(context, artifact.getFileName());
314             if (!Files.exists(cachePath) && cacheConfig.isRemoteCacheEnabled()) {
315                 if (!remoteRepository.getArtifactContent(context, artifact, cachePath)) {
316                     Files.deleteIfExists(cachePath);
317                 }
318             }
319             return cachePath;
320         }
321     }
322 
323     @Override
324     public void beforeSave(CacheContext environment) {
325         clearCache(environment);
326     }
327 
328     @Override
329     public void saveBuildInfo(CacheResult cacheResult, Build build) throws IOException {
330         final Path path = localBuildPath(cacheResult.getContext(), BUILDINFO_XML, true);
331         Files.write(path, xmlService.toBytes(build.getDto()), TRUNCATE_EXISTING, CREATE);
332         LOGGER.info("Saved Build to local file: {}", path);
333         if (cacheConfig.isSaveToRemote() && !cacheResult.isFinal()) {
334             remoteRepository.saveBuildInfo(cacheResult, build);
335         }
336     }
337 
338     @Override
339     public void saveCacheReport(String buildId, MavenSession session, CacheReport cacheReport) throws IOException {
340         Path path = getMultimoduleRoot(session).resolve("target").resolve("maven-incremental");
341         Files.createDirectories(path);
342         Path reportPath = path.resolve("cache-report." + buildId + ".xml");
343         Files.write(reportPath, xmlService.toBytes(cacheReport), TRUNCATE_EXISTING, CREATE);
344         LOGGER.debug("Save cache-report to local file: {}", reportPath);
345         if (cacheConfig.isSaveToRemote()) {
346             LOGGER.info("Saving cache report on build completion");
347             remoteRepository.saveCacheReport(buildId, session, cacheReport);
348         }
349     }
350 
351     @Override
352     public void saveArtifactFile(CacheResult cacheResult, org.apache.maven.artifact.Artifact artifact)
353             throws IOException {
354         // safe artifacts to cache
355         File artifactFile = artifact.getFile();
356         Path cachePath = localBuildPath(cacheResult.getContext(), CacheUtils.normalizedName(artifact), true);
357         Files.copy(artifactFile.toPath(), cachePath, StandardCopyOption.REPLACE_EXISTING);
358         if (cacheConfig.isSaveToRemote() && !cacheResult.isFinal()) {
359             remoteRepository.saveArtifactFile(cacheResult, artifact);
360         }
361     }
362 
363     private Path buildCacheDir(CacheContext context) throws IOException {
364         final MavenProject project = context.getProject();
365         final Path artifactCacheDir =
366                 artifactCacheDir(context.getSession(), project.getGroupId(), project.getArtifactId());
367         return artifactCacheDir.resolve(context.getInputInfo().getChecksum());
368     }
369 
370     private Path artifactCacheDir(MavenSession session, String groupId, String artifactId) throws IOException {
371         final Path vga = Paths.get(CACHE_IMPLEMENTATION_VERSION, groupId, artifactId);
372         final Path path = baseDir(session).resolve(vga);
373         Files.createDirectories(path);
374         return path;
375     }
376 
377     private Path baseDir(MavenSession session) {
378         String loc = cacheConfig.getLocalRepositoryLocation();
379         if (loc != null) {
380             return Paths.get(loc);
381         } else {
382             return Paths.get(session.getLocalRepository().getBasedir())
383                     .getParent()
384                     .resolve("build-cache");
385         }
386     }
387 
388     private Path remoteBuildPath(CacheContext context, String filename) throws IOException {
389         return remoteBuildDir(context).resolve(filename);
390     }
391 
392     private Path localBuildPath(CacheContext context, String filename, boolean createDir) throws IOException {
393         final Path localBuildDir = localBuildDir(context);
394         if (createDir) {
395             Files.createDirectories(localBuildDir);
396         }
397         return localBuildDir.resolve(filename);
398     }
399 
400     private Path remoteBuildDir(CacheContext context) throws IOException {
401         return buildCacheDir(context).resolve(cacheConfig.getId());
402     }
403 
404     private Path localBuildDir(CacheContext context) throws IOException {
405         return buildCacheDir(context).resolve("local");
406     }
407 
408     private static FileTime lastModifiedTime(Path p) {
409         try {
410             return Files.getLastModifiedTime(p);
411         } catch (IOException e) {
412             return FileTime.fromMillis(0);
413         }
414     }
415 
416     private static <K, V> void add(Map<K, Collection<V>> map, K key, V value) {
417         map.computeIfAbsent(key, k -> new ArrayList<>()).add(value);
418     }
419 }