001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.internal.impl.checksum;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.BufferedReader;
026import java.io.IOException;
027import java.io.UncheckedIOException;
028import java.nio.charset.StandardCharsets;
029import java.nio.file.Files;
030import java.nio.file.Path;
031import java.util.ArrayList;
032import java.util.HashMap;
033import java.util.List;
034import java.util.Map;
035import java.util.Objects;
036import java.util.concurrent.ConcurrentHashMap;
037import java.util.concurrent.atomic.AtomicBoolean;
038import java.util.function.Function;
039import java.util.stream.Collectors;
040
041import org.eclipse.aether.MultiRuntimeException;
042import org.eclipse.aether.RepositorySystemSession;
043import org.eclipse.aether.artifact.Artifact;
044import org.eclipse.aether.impl.RepositorySystemLifecycle;
045import org.eclipse.aether.internal.impl.LocalPathComposer;
046import org.eclipse.aether.metadata.Metadata;
047import org.eclipse.aether.repository.ArtifactRepository;
048import org.eclipse.aether.spi.connector.checksum.ChecksumAlgorithmFactory;
049import org.eclipse.aether.spi.io.PathProcessor;
050import org.eclipse.aether.spi.remoterepo.RepositoryKeyFunctionFactory;
051import org.eclipse.aether.util.ConfigUtils;
052import org.eclipse.aether.util.PathUtils;
053
054import static java.util.Objects.requireNonNull;
055
056/**
057 * Compact file {@link FileTrustedChecksumsSourceSupport} implementation that use specified directory as base
058 * directory, where it expects a "summary" file named as "checksums.${checksumExt}" for each checksum algorithm.
059 * File format is GNU Coreutils compatible: each line holds checksum followed by two spaces and artifact relative path
060 * (from local repository root, without leading "./"). Metadata checksums are covered as well, keyed by the metadata
061 * path composed from the origin repository key (for example {@code g/a/v/maven-metadata-central.xml}). This means that trusted checksums summary file can be used to
062 * validate artifacts or generate it using standard GNU tools like GNU {@code sha1sum} is (for BSD derivatives same
063 * file can be used with {@code -r} switch).
064 * <p>
065 * The format supports comments "#" (hash) and empty lines for easier structuring the file content, and both are
066 * ignored. Also, their presence makes the summary file incompatible with GNU Coreutils format. On save of the
067 * summary file, the comments and empty lines are lost, and file is sorted by path names for easier diffing
068 * (2nd column in file).
069 * <p>
070 * The source by default is "origin aware", and it will factor in origin repository ID as well into summary file name,
071 * for example "checksums-central.sha256".
072 * <p>
073 * Example commands for managing summary file (in examples will use repository ID "central"):
074 * <ul>
075 *     <li>To create summary file: {@code find * -not -name "checksums-central.sha256" -type f -print0 |
076 *       xargs -0 sha256sum | sort -k 2 > checksums-central.sha256}</li>
077 *     <li>To verify artifacts using summary file: {@code sha256sum --quiet -c checksums-central.sha256}</li>
078 * </ul>
079 * <p>
080 * The checksums summary file is lazily loaded and remains cached during lifetime of the component, so file changes
081 * during lifecycle of the component are not picked up. This implementation can be simultaneously used to lookup and
082 * also record checksums. The recorded checksums will become visible for every session, and will be flushed
083 * at repository system shutdown, merged with existing ones on disk.
084 * <p>
085 * The name of this implementation is "summaryFile".
086 *
087 * @see <a href="https://man7.org/linux/man-pages/man1/sha1sum.1.html">sha1sum man page</a>
088 * @see <a href="https://www.gnu.org/software/coreutils/manual/coreutils.html#md5sum-invocation">GNU Coreutils: md5sum</a>
089 * @since 1.9.0
090 */
091@Singleton
092@Named(SummaryFileTrustedChecksumsSource.NAME)
093public final class SummaryFileTrustedChecksumsSource extends FileTrustedChecksumsSourceSupport {
094    public static final String NAME = "summaryFile";
095
096    private static final String CONFIG_PROPS_PREFIX =
097            FileTrustedChecksumsSourceSupport.CONFIG_PROPS_PREFIX + NAME + ".";
098
099    /**
100     * Is checksum source enabled?
101     *
102     * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
103     * @configurationType {@link java.lang.Boolean}
104     * @configurationDefaultValue false
105     */
106    public static final String CONFIG_PROP_ENABLED = FileTrustedChecksumsSourceSupport.CONFIG_PROPS_PREFIX + NAME;
107
108    /**
109     * The basedir where checksums are. If relative, is resolved from local repository root.
110     *
111     * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
112     * @configurationType {@link java.lang.String}
113     * @configurationDefaultValue {@link #LOCAL_REPO_PREFIX_DIR}
114     */
115    public static final String CONFIG_PROP_BASEDIR = CONFIG_PROPS_PREFIX + "basedir";
116
117    public static final String LOCAL_REPO_PREFIX_DIR = ".checksums";
118
119    /**
120     * Is source origin aware?
121     *
122     * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
123     * @configurationType {@link java.lang.Boolean}
124     * @configurationDefaultValue true
125     */
126    public static final String CONFIG_PROP_ORIGIN_AWARE = CONFIG_PROPS_PREFIX + "originAware";
127
128    public static final String CHECKSUMS_FILE_PREFIX = "checksums";
129
130    private final LocalPathComposer localPathComposer;
131
132    private final RepositorySystemLifecycle repositorySystemLifecycle;
133
134    private final PathProcessor pathProcessor;
135
136    private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> checksums;
137
138    private final ConcurrentHashMap<Path, Boolean> changedChecksums;
139
140    private final AtomicBoolean onShutdownHandlerRegistered;
141
142    @Inject
143    public SummaryFileTrustedChecksumsSource(
144            RepositoryKeyFunctionFactory repoKeyFunctionFactory,
145            LocalPathComposer localPathComposer,
146            RepositorySystemLifecycle repositorySystemLifecycle,
147            PathProcessor pathProcessor) {
148        super(repoKeyFunctionFactory);
149        this.localPathComposer = requireNonNull(localPathComposer);
150        this.repositorySystemLifecycle = requireNonNull(repositorySystemLifecycle);
151        this.pathProcessor = requireNonNull(pathProcessor);
152        this.checksums = new ConcurrentHashMap<>();
153        this.changedChecksums = new ConcurrentHashMap<>();
154        this.onShutdownHandlerRegistered = new AtomicBoolean(false);
155    }
156
157    @Override
158    protected boolean isEnabled(RepositorySystemSession session) {
159        return ConfigUtils.getBoolean(session, false, CONFIG_PROP_ENABLED);
160    }
161
162    private boolean isOriginAware(RepositorySystemSession session) {
163        return ConfigUtils.getBoolean(session, true, CONFIG_PROP_ORIGIN_AWARE);
164    }
165
166    @Override
167    protected Map<String, String> doGetTrustedArtifactChecksums(
168            RepositorySystemSession session,
169            Artifact artifact,
170            ArtifactRepository artifactRepository,
171            List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) {
172        return doGetTrustedPathChecksums(
173                session,
174                repositoryKey(session, artifactRepository).subList(0, 1),
175                rk -> localPathComposer.getPathForArtifact(artifact, false),
176                checksumAlgorithmFactories);
177    }
178
179    @Override
180    protected Map<String, String> doGetTrustedMetadataChecksums(
181            RepositorySystemSession session,
182            Metadata metadata,
183            ArtifactRepository artifactRepository,
184            List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) {
185        return doGetTrustedPathChecksums(
186                session,
187                repositoryKey(session, artifactRepository),
188                rk -> localPathComposer.getPathForMetadata(metadata, rk),
189                checksumAlgorithmFactories);
190    }
191
192    /**
193     * Looks up the trusted checksums for given local-repository style path (the summary file 2nd column), be it
194     * an artifact or a metadata path.
195     */
196    private Map<String, String> doGetTrustedPathChecksums(
197            RepositorySystemSession session,
198            List<String> repoKeys,
199            Function<String, String> pathComposer,
200            List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) {
201        final HashMap<String, String> result = new HashMap<>();
202        final Path basedir = getBasedir(session, LOCAL_REPO_PREFIX_DIR, CONFIG_PROP_BASEDIR, false);
203        if (Files.isDirectory(basedir)) {
204            final boolean originAware = isOriginAware(session);
205            for (String repoKey : repoKeys) {
206                for (ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories) {
207                    Path summaryFile =
208                            summaryFile(basedir, originAware, repoKey, checksumAlgorithmFactory.getFileExtension());
209                    ConcurrentHashMap<String, String> algorithmChecksums =
210                            checksums.computeIfAbsent(summaryFile, f -> loadProvidedChecksums(summaryFile));
211                    String checksum = algorithmChecksums.get(pathComposer.apply(repoKey));
212                    if (checksum != null) {
213                        result.putIfAbsent(checksumAlgorithmFactory.getName(), checksum);
214                    }
215                }
216            }
217        }
218        return result;
219    }
220
221    @Override
222    protected Writer doGetTrustedArtifactChecksumsWriter(RepositorySystemSession session) {
223        if (onShutdownHandlerRegistered.compareAndSet(false, true)) {
224            repositorySystemLifecycle.addOnSystemEndedHandler(this::saveRecordedLines);
225        }
226        return new SummaryFileWriter(
227                checksums,
228                getBasedir(session, LOCAL_REPO_PREFIX_DIR, CONFIG_PROP_BASEDIR, true),
229                isOriginAware(session),
230                r -> repositoryKey(session, r).get(0));
231    }
232
233    /**
234     * Returns the summary file path. The file itself and its parent directories may not exist, this method merely
235     * calculate the path.
236     */
237    private Path summaryFile(Path basedir, boolean originAware, String safeRepositoryId, String checksumExtension) {
238        String fileName = CHECKSUMS_FILE_PREFIX;
239        if (originAware) {
240            // defense in depth: the repository key is spliced into the summary file name,
241            // so it must be a safe path segment
242            PathUtils.validatePathComponent(safeRepositoryId, "repository key");
243            fileName += "-" + safeRepositoryId;
244        }
245        return basedir.resolve(fileName + "." + checksumExtension);
246    }
247
248    private ConcurrentHashMap<String, String> loadProvidedChecksums(Path summaryFile) {
249        ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();
250        if (Files.isRegularFile(summaryFile)) {
251            try (BufferedReader reader = Files.newBufferedReader(summaryFile, StandardCharsets.UTF_8)) {
252                String line;
253                while ((line = reader.readLine()) != null) {
254                    if (!line.startsWith("#") && !line.isEmpty()) {
255                        String[] parts = line.split("  ", 2);
256                        if (parts.length == 2) {
257                            String newChecksum = parts[0];
258                            String artifactPath = parts[1];
259                            String oldChecksum = result.put(artifactPath, newChecksum);
260                            if (oldChecksum != null) {
261                                if (Objects.equals(oldChecksum, newChecksum)) {
262                                    logger.warn(
263                                            "Checksums file '{}' contains duplicate checksums for artifact {}: {}",
264                                            summaryFile,
265                                            artifactPath,
266                                            oldChecksum);
267                                } else {
268                                    logger.warn(
269                                            "Checksums file '{}' contains different checksums for artifact {}: "
270                                                    + "old '{}' replaced by new '{}'",
271                                            summaryFile,
272                                            artifactPath,
273                                            oldChecksum,
274                                            newChecksum);
275                                }
276                            }
277                        } else {
278                            logger.warn("Checksums file '{}' ignored malformed line '{}'", summaryFile, line);
279                        }
280                    }
281                }
282            } catch (IOException e) {
283                throw new UncheckedIOException(e);
284            }
285            logger.info("Loaded {} trusted checksums from {}", result.size(), summaryFile);
286        }
287        return result;
288    }
289
290    private class SummaryFileWriter implements Writer {
291        private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache;
292
293        private final Path basedir;
294
295        private final boolean originAware;
296
297        private final Function<ArtifactRepository, String> repositoryKeyFunction;
298
299        private SummaryFileWriter(
300                ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache,
301                Path basedir,
302                boolean originAware,
303                Function<ArtifactRepository, String> repositoryKeyFunction) {
304            this.cache = cache;
305            this.basedir = basedir;
306            this.originAware = originAware;
307            this.repositoryKeyFunction = repositoryKeyFunction;
308        }
309
310        @Override
311        public void addTrustedArtifactChecksums(
312                Artifact artifact,
313                ArtifactRepository artifactRepository,
314                List<ChecksumAlgorithmFactory> checksumAlgorithmFactories,
315                Map<String, String> trustedArtifactChecksums) {
316            addTrustedPathChecksums(
317                    artifact,
318                    localPathComposer.getPathForArtifact(artifact, false),
319                    artifactRepository,
320                    checksumAlgorithmFactories,
321                    trustedArtifactChecksums);
322        }
323
324        @Override
325        public void addTrustedMetadataChecksums(
326                Metadata metadata,
327                ArtifactRepository artifactRepository,
328                List<ChecksumAlgorithmFactory> checksumAlgorithmFactories,
329                Map<String, String> trustedMetadataChecksums) {
330            addTrustedPathChecksums(
331                    metadata,
332                    localPathComposer.getPathForMetadata(metadata, repositoryKeyFunction.apply(artifactRepository)),
333                    artifactRepository,
334                    checksumAlgorithmFactories,
335                    trustedMetadataChecksums);
336        }
337
338        private void addTrustedPathChecksums(
339                Object subject,
340                String path,
341                ArtifactRepository artifactRepository,
342                List<ChecksumAlgorithmFactory> checksumAlgorithmFactories,
343                Map<String, String> trustedChecksums) {
344            for (ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories) {
345                Path summaryFile = summaryFile(
346                        basedir,
347                        originAware,
348                        repositoryKeyFunction.apply(artifactRepository),
349                        checksumAlgorithmFactory.getFileExtension());
350                String checksum = requireNonNull(trustedChecksums.get(checksumAlgorithmFactory.getName()));
351
352                String oldChecksum = cache.computeIfAbsent(summaryFile, k -> loadProvidedChecksums(summaryFile))
353                        .put(path, checksum);
354
355                if (oldChecksum == null) {
356                    changedChecksums.put(summaryFile, Boolean.TRUE); // new
357                } else if (!Objects.equals(oldChecksum, checksum)) {
358                    changedChecksums.put(summaryFile, Boolean.TRUE); // replaced
359                    logger.info("Trusted checksum for {} replaced: old {}, new {}", subject, oldChecksum, checksum);
360                }
361            }
362        }
363    }
364
365    /**
366     * On-close handler that saves recorded checksums, if any.
367     */
368    private void saveRecordedLines() {
369        if (changedChecksums.isEmpty()) {
370            return;
371        }
372
373        ArrayList<Exception> exceptions = new ArrayList<>();
374        for (Map.Entry<Path, ConcurrentHashMap<String, String>> entry : checksums.entrySet()) {
375            Path summaryFile = entry.getKey();
376            if (changedChecksums.get(summaryFile) != Boolean.TRUE) {
377                continue;
378            }
379            ConcurrentHashMap<String, String> recordedLines = entry.getValue();
380            if (!recordedLines.isEmpty()) {
381                try {
382                    ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();
383                    result.putAll(loadProvidedChecksums(summaryFile));
384                    result.putAll(recordedLines);
385
386                    logger.info("Saving {} checksums to '{}'", result.size(), summaryFile);
387                    pathProcessor.writeWithBackup(
388                            summaryFile,
389                            result.entrySet().stream()
390                                    .sorted(Map.Entry.comparingByKey())
391                                    .map(e -> e.getValue() + "  " + e.getKey())
392                                    .collect(Collectors.joining(System.lineSeparator())));
393                } catch (IOException e) {
394                    exceptions.add(e);
395                }
396            }
397        }
398        MultiRuntimeException.mayThrow("session save checksums failure", exceptions);
399    }
400}