View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.eclipse.aether.internal.impl.checksum;
20  
21  import javax.inject.Inject;
22  import javax.inject.Named;
23  import javax.inject.Singleton;
24  
25  import java.io.BufferedReader;
26  import java.io.IOException;
27  import java.io.UncheckedIOException;
28  import java.nio.charset.StandardCharsets;
29  import java.nio.file.Files;
30  import java.nio.file.Path;
31  import java.util.ArrayList;
32  import java.util.HashMap;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Objects;
36  import java.util.concurrent.ConcurrentHashMap;
37  import java.util.concurrent.atomic.AtomicBoolean;
38  import java.util.function.Function;
39  import java.util.stream.Collectors;
40  
41  import org.eclipse.aether.MultiRuntimeException;
42  import org.eclipse.aether.RepositorySystemSession;
43  import org.eclipse.aether.artifact.Artifact;
44  import org.eclipse.aether.impl.RepositorySystemLifecycle;
45  import org.eclipse.aether.internal.impl.LocalPathComposer;
46  import org.eclipse.aether.metadata.Metadata;
47  import org.eclipse.aether.repository.ArtifactRepository;
48  import org.eclipse.aether.spi.connector.checksum.ChecksumAlgorithmFactory;
49  import org.eclipse.aether.spi.io.PathProcessor;
50  import org.eclipse.aether.spi.remoterepo.RepositoryKeyFunctionFactory;
51  import org.eclipse.aether.util.ConfigUtils;
52  import org.eclipse.aether.util.PathUtils;
53  
54  import static java.util.Objects.requireNonNull;
55  
56  /**
57   * Compact file {@link FileTrustedChecksumsSourceSupport} implementation that use specified directory as base
58   * directory, where it expects a "summary" file named as "checksums.${checksumExt}" for each checksum algorithm.
59   * File format is GNU Coreutils compatible: each line holds checksum followed by two spaces and artifact relative path
60   * (from local repository root, without leading "./"). Metadata checksums are covered as well, keyed by the metadata
61   * path composed from the origin repository key (for example {@code g/a/v/maven-metadata-central.xml}). This means that trusted checksums summary file can be used to
62   * validate artifacts or generate it using standard GNU tools like GNU {@code sha1sum} is (for BSD derivatives same
63   * file can be used with {@code -r} switch).
64   * <p>
65   * The format supports comments "#" (hash) and empty lines for easier structuring the file content, and both are
66   * ignored. Also, their presence makes the summary file incompatible with GNU Coreutils format. On save of the
67   * summary file, the comments and empty lines are lost, and file is sorted by path names for easier diffing
68   * (2nd column in file).
69   * <p>
70   * The source by default is "origin aware", and it will factor in origin repository ID as well into summary file name,
71   * for example "checksums-central.sha256".
72   * <p>
73   * Example commands for managing summary file (in examples will use repository ID "central"):
74   * <ul>
75   *     <li>To create summary file: {@code find * -not -name "checksums-central.sha256" -type f -print0 |
76   *       xargs -0 sha256sum | sort -k 2 > checksums-central.sha256}</li>
77   *     <li>To verify artifacts using summary file: {@code sha256sum --quiet -c checksums-central.sha256}</li>
78   * </ul>
79   * <p>
80   * The checksums summary file is lazily loaded and remains cached during lifetime of the component, so file changes
81   * during lifecycle of the component are not picked up. This implementation can be simultaneously used to lookup and
82   * also record checksums. The recorded checksums will become visible for every session, and will be flushed
83   * at repository system shutdown, merged with existing ones on disk.
84   * <p>
85   * The name of this implementation is "summaryFile".
86   *
87   * @see <a href="https://man7.org/linux/man-pages/man1/sha1sum.1.html">sha1sum man page</a>
88   * @see <a href="https://www.gnu.org/software/coreutils/manual/coreutils.html#md5sum-invocation">GNU Coreutils: md5sum</a>
89   * @since 1.9.0
90   */
91  @Singleton
92  @Named(SummaryFileTrustedChecksumsSource.NAME)
93  public final class SummaryFileTrustedChecksumsSource extends FileTrustedChecksumsSourceSupport {
94      public static final String NAME = "summaryFile";
95  
96      private static final String CONFIG_PROPS_PREFIX =
97              FileTrustedChecksumsSourceSupport.CONFIG_PROPS_PREFIX + NAME + ".";
98  
99      /**
100      * Is checksum source enabled?
101      *
102      * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
103      * @configurationType {@link java.lang.Boolean}
104      * @configurationDefaultValue false
105      */
106     public static final String CONFIG_PROP_ENABLED = FileTrustedChecksumsSourceSupport.CONFIG_PROPS_PREFIX + NAME;
107 
108     /**
109      * The basedir where checksums are. If relative, is resolved from local repository root.
110      *
111      * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
112      * @configurationType {@link java.lang.String}
113      * @configurationDefaultValue {@link #LOCAL_REPO_PREFIX_DIR}
114      */
115     public static final String CONFIG_PROP_BASEDIR = CONFIG_PROPS_PREFIX + "basedir";
116 
117     public static final String LOCAL_REPO_PREFIX_DIR = ".checksums";
118 
119     /**
120      * Is source origin aware?
121      *
122      * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
123      * @configurationType {@link java.lang.Boolean}
124      * @configurationDefaultValue true
125      */
126     public static final String CONFIG_PROP_ORIGIN_AWARE = CONFIG_PROPS_PREFIX + "originAware";
127 
128     public static final String CHECKSUMS_FILE_PREFIX = "checksums";
129 
130     private final LocalPathComposer localPathComposer;
131 
132     private final RepositorySystemLifecycle repositorySystemLifecycle;
133 
134     private final PathProcessor pathProcessor;
135 
136     private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> checksums;
137 
138     private final ConcurrentHashMap<Path, Boolean> changedChecksums;
139 
140     private final AtomicBoolean onShutdownHandlerRegistered;
141 
142     @Inject
143     public SummaryFileTrustedChecksumsSource(
144             RepositoryKeyFunctionFactory repoKeyFunctionFactory,
145             LocalPathComposer localPathComposer,
146             RepositorySystemLifecycle repositorySystemLifecycle,
147             PathProcessor pathProcessor) {
148         super(repoKeyFunctionFactory);
149         this.localPathComposer = requireNonNull(localPathComposer);
150         this.repositorySystemLifecycle = requireNonNull(repositorySystemLifecycle);
151         this.pathProcessor = requireNonNull(pathProcessor);
152         this.checksums = new ConcurrentHashMap<>();
153         this.changedChecksums = new ConcurrentHashMap<>();
154         this.onShutdownHandlerRegistered = new AtomicBoolean(false);
155     }
156 
157     @Override
158     protected boolean isEnabled(RepositorySystemSession session) {
159         return ConfigUtils.getBoolean(session, false, CONFIG_PROP_ENABLED);
160     }
161 
162     private boolean isOriginAware(RepositorySystemSession session) {
163         return ConfigUtils.getBoolean(session, true, CONFIG_PROP_ORIGIN_AWARE);
164     }
165 
166     @Override
167     protected Map<String, String> doGetTrustedArtifactChecksums(
168             RepositorySystemSession session,
169             Artifact artifact,
170             ArtifactRepository artifactRepository,
171             List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) {
172         return doGetTrustedPathChecksums(
173                 session,
174                 repositoryKey(session, artifactRepository).subList(0, 1),
175                 rk -> localPathComposer.getPathForArtifact(artifact, false),
176                 checksumAlgorithmFactories);
177     }
178 
179     @Override
180     protected Map<String, String> doGetTrustedMetadataChecksums(
181             RepositorySystemSession session,
182             Metadata metadata,
183             ArtifactRepository artifactRepository,
184             List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) {
185         return doGetTrustedPathChecksums(
186                 session,
187                 repositoryKey(session, artifactRepository),
188                 rk -> localPathComposer.getPathForMetadata(metadata, rk),
189                 checksumAlgorithmFactories);
190     }
191 
192     /**
193      * Looks up the trusted checksums for given local-repository style path (the summary file 2nd column), be it
194      * an artifact or a metadata path.
195      */
196     private Map<String, String> doGetTrustedPathChecksums(
197             RepositorySystemSession session,
198             List<String> repoKeys,
199             Function<String, String> pathComposer,
200             List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) {
201         final HashMap<String, String> result = new HashMap<>();
202         final Path basedir = getBasedir(session, LOCAL_REPO_PREFIX_DIR, CONFIG_PROP_BASEDIR, false);
203         if (Files.isDirectory(basedir)) {
204             final boolean originAware = isOriginAware(session);
205             for (String repoKey : repoKeys) {
206                 for (ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories) {
207                     Path summaryFile =
208                             summaryFile(basedir, originAware, repoKey, checksumAlgorithmFactory.getFileExtension());
209                     ConcurrentHashMap<String, String> algorithmChecksums =
210                             checksums.computeIfAbsent(summaryFile, f -> loadProvidedChecksums(summaryFile));
211                     String checksum = algorithmChecksums.get(pathComposer.apply(repoKey));
212                     if (checksum != null) {
213                         result.putIfAbsent(checksumAlgorithmFactory.getName(), checksum);
214                     }
215                 }
216             }
217         }
218         return result;
219     }
220 
221     @Override
222     protected Writer doGetTrustedArtifactChecksumsWriter(RepositorySystemSession session) {
223         if (onShutdownHandlerRegistered.compareAndSet(false, true)) {
224             repositorySystemLifecycle.addOnSystemEndedHandler(this::saveRecordedLines);
225         }
226         return new SummaryFileWriter(
227                 checksums,
228                 getBasedir(session, LOCAL_REPO_PREFIX_DIR, CONFIG_PROP_BASEDIR, true),
229                 isOriginAware(session),
230                 r -> repositoryKey(session, r).get(0));
231     }
232 
233     /**
234      * Returns the summary file path. The file itself and its parent directories may not exist, this method merely
235      * calculate the path.
236      */
237     private Path summaryFile(Path basedir, boolean originAware, String safeRepositoryId, String checksumExtension) {
238         String fileName = CHECKSUMS_FILE_PREFIX;
239         if (originAware) {
240             // defense in depth: the repository key is spliced into the summary file name,
241             // so it must be a safe path segment
242             PathUtils.validatePathComponent(safeRepositoryId, "repository key");
243             fileName += "-" + safeRepositoryId;
244         }
245         return basedir.resolve(fileName + "." + checksumExtension);
246     }
247 
248     private ConcurrentHashMap<String, String> loadProvidedChecksums(Path summaryFile) {
249         ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();
250         if (Files.isRegularFile(summaryFile)) {
251             try (BufferedReader reader = Files.newBufferedReader(summaryFile, StandardCharsets.UTF_8)) {
252                 String line;
253                 while ((line = reader.readLine()) != null) {
254                     if (!line.startsWith("#") && !line.isEmpty()) {
255                         String[] parts = line.split("  ", 2);
256                         if (parts.length == 2) {
257                             String newChecksum = parts[0];
258                             String artifactPath = parts[1];
259                             String oldChecksum = result.put(artifactPath, newChecksum);
260                             if (oldChecksum != null) {
261                                 if (Objects.equals(oldChecksum, newChecksum)) {
262                                     logger.warn(
263                                             "Checksums file '{}' contains duplicate checksums for artifact {}: {}",
264                                             summaryFile,
265                                             artifactPath,
266                                             oldChecksum);
267                                 } else {
268                                     logger.warn(
269                                             "Checksums file '{}' contains different checksums for artifact {}: "
270                                                     + "old '{}' replaced by new '{}'",
271                                             summaryFile,
272                                             artifactPath,
273                                             oldChecksum,
274                                             newChecksum);
275                                 }
276                             }
277                         } else {
278                             logger.warn("Checksums file '{}' ignored malformed line '{}'", summaryFile, line);
279                         }
280                     }
281                 }
282             } catch (IOException e) {
283                 throw new UncheckedIOException(e);
284             }
285             logger.info("Loaded {} trusted checksums from {}", result.size(), summaryFile);
286         }
287         return result;
288     }
289 
290     private class SummaryFileWriter implements Writer {
291         private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache;
292 
293         private final Path basedir;
294 
295         private final boolean originAware;
296 
297         private final Function<ArtifactRepository, String> repositoryKeyFunction;
298 
299         private SummaryFileWriter(
300                 ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache,
301                 Path basedir,
302                 boolean originAware,
303                 Function<ArtifactRepository, String> repositoryKeyFunction) {
304             this.cache = cache;
305             this.basedir = basedir;
306             this.originAware = originAware;
307             this.repositoryKeyFunction = repositoryKeyFunction;
308         }
309 
310         @Override
311         public void addTrustedArtifactChecksums(
312                 Artifact artifact,
313                 ArtifactRepository artifactRepository,
314                 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories,
315                 Map<String, String> trustedArtifactChecksums) {
316             addTrustedPathChecksums(
317                     artifact,
318                     localPathComposer.getPathForArtifact(artifact, false),
319                     artifactRepository,
320                     checksumAlgorithmFactories,
321                     trustedArtifactChecksums);
322         }
323 
324         @Override
325         public void addTrustedMetadataChecksums(
326                 Metadata metadata,
327                 ArtifactRepository artifactRepository,
328                 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories,
329                 Map<String, String> trustedMetadataChecksums) {
330             addTrustedPathChecksums(
331                     metadata,
332                     localPathComposer.getPathForMetadata(metadata, repositoryKeyFunction.apply(artifactRepository)),
333                     artifactRepository,
334                     checksumAlgorithmFactories,
335                     trustedMetadataChecksums);
336         }
337 
338         private void addTrustedPathChecksums(
339                 Object subject,
340                 String path,
341                 ArtifactRepository artifactRepository,
342                 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories,
343                 Map<String, String> trustedChecksums) {
344             for (ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories) {
345                 Path summaryFile = summaryFile(
346                         basedir,
347                         originAware,
348                         repositoryKeyFunction.apply(artifactRepository),
349                         checksumAlgorithmFactory.getFileExtension());
350                 String checksum = requireNonNull(trustedChecksums.get(checksumAlgorithmFactory.getName()));
351 
352                 String oldChecksum = cache.computeIfAbsent(summaryFile, k -> loadProvidedChecksums(summaryFile))
353                         .put(path, checksum);
354 
355                 if (oldChecksum == null) {
356                     changedChecksums.put(summaryFile, Boolean.TRUE); // new
357                 } else if (!Objects.equals(oldChecksum, checksum)) {
358                     changedChecksums.put(summaryFile, Boolean.TRUE); // replaced
359                     logger.info("Trusted checksum for {} replaced: old {}, new {}", subject, oldChecksum, checksum);
360                 }
361             }
362         }
363     }
364 
365     /**
366      * On-close handler that saves recorded checksums, if any.
367      */
368     private void saveRecordedLines() {
369         if (changedChecksums.isEmpty()) {
370             return;
371         }
372 
373         ArrayList<Exception> exceptions = new ArrayList<>();
374         for (Map.Entry<Path, ConcurrentHashMap<String, String>> entry : checksums.entrySet()) {
375             Path summaryFile = entry.getKey();
376             if (changedChecksums.get(summaryFile) != Boolean.TRUE) {
377                 continue;
378             }
379             ConcurrentHashMap<String, String> recordedLines = entry.getValue();
380             if (!recordedLines.isEmpty()) {
381                 try {
382                     ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();
383                     result.putAll(loadProvidedChecksums(summaryFile));
384                     result.putAll(recordedLines);
385 
386                     logger.info("Saving {} checksums to '{}'", result.size(), summaryFile);
387                     pathProcessor.writeWithBackup(
388                             summaryFile,
389                             result.entrySet().stream()
390                                     .sorted(Map.Entry.comparingByKey())
391                                     .map(e -> e.getValue() + "  " + e.getKey())
392                                     .collect(Collectors.joining(System.lineSeparator())));
393                 } catch (IOException e) {
394                     exceptions.add(e);
395                 }
396             }
397         }
398         MultiRuntimeException.mayThrow("session save checksums failure", exceptions);
399     }
400 }