001/* 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, 013 * software distributed under the License is distributed on an 014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 015 * KIND, either express or implied. See the License for the 016 * specific language governing permissions and limitations 017 * under the License. 018 */ 019package org.eclipse.aether.internal.impl.checksum; 020 021import javax.inject.Inject; 022import javax.inject.Named; 023import javax.inject.Singleton; 024 025import java.io.BufferedReader; 026import java.io.IOException; 027import java.io.UncheckedIOException; 028import java.nio.charset.StandardCharsets; 029import java.nio.file.Files; 030import java.nio.file.Path; 031import java.util.ArrayList; 032import java.util.HashMap; 033import java.util.List; 034import java.util.Map; 035import java.util.Objects; 036import java.util.concurrent.ConcurrentHashMap; 037import java.util.concurrent.atomic.AtomicBoolean; 038import java.util.function.Function; 039import java.util.stream.Collectors; 040 041import org.eclipse.aether.MultiRuntimeException; 042import org.eclipse.aether.RepositorySystemSession; 043import org.eclipse.aether.artifact.Artifact; 044import org.eclipse.aether.impl.RepositorySystemLifecycle; 045import org.eclipse.aether.internal.impl.LocalPathComposer; 046import org.eclipse.aether.metadata.Metadata; 047import org.eclipse.aether.repository.ArtifactRepository; 048import org.eclipse.aether.spi.connector.checksum.ChecksumAlgorithmFactory; 049import org.eclipse.aether.spi.io.PathProcessor; 050import org.eclipse.aether.spi.remoterepo.RepositoryKeyFunctionFactory; 051import org.eclipse.aether.util.ConfigUtils; 052import org.eclipse.aether.util.PathUtils; 053 054import static java.util.Objects.requireNonNull; 055 056/** 057 * Compact file {@link FileTrustedChecksumsSourceSupport} implementation that use specified directory as base 058 * directory, where it expects a "summary" file named as "checksums.${checksumExt}" for each checksum algorithm. 059 * File format is GNU Coreutils compatible: each line holds checksum followed by two spaces and artifact relative path 060 * (from local repository root, without leading "./"). Metadata checksums are covered as well, keyed by the metadata 061 * path composed from the origin repository key (for example {@code g/a/v/maven-metadata-central.xml}). This means that trusted checksums summary file can be used to 062 * validate artifacts or generate it using standard GNU tools like GNU {@code sha1sum} is (for BSD derivatives same 063 * file can be used with {@code -r} switch). 064 * <p> 065 * The format supports comments "#" (hash) and empty lines for easier structuring the file content, and both are 066 * ignored. Also, their presence makes the summary file incompatible with GNU Coreutils format. On save of the 067 * summary file, the comments and empty lines are lost, and file is sorted by path names for easier diffing 068 * (2nd column in file). 069 * <p> 070 * The source by default is "origin aware", and it will factor in origin repository ID as well into summary file name, 071 * for example "checksums-central.sha256". 072 * <p> 073 * Example commands for managing summary file (in examples will use repository ID "central"): 074 * <ul> 075 * <li>To create summary file: {@code find * -not -name "checksums-central.sha256" -type f -print0 | 076 * xargs -0 sha256sum | sort -k 2 > checksums-central.sha256}</li> 077 * <li>To verify artifacts using summary file: {@code sha256sum --quiet -c checksums-central.sha256}</li> 078 * </ul> 079 * <p> 080 * The checksums summary file is lazily loaded and remains cached during lifetime of the component, so file changes 081 * during lifecycle of the component are not picked up. This implementation can be simultaneously used to lookup and 082 * also record checksums. The recorded checksums will become visible for every session, and will be flushed 083 * at repository system shutdown, merged with existing ones on disk. 084 * <p> 085 * The name of this implementation is "summaryFile". 086 * 087 * @see <a href="https://man7.org/linux/man-pages/man1/sha1sum.1.html">sha1sum man page</a> 088 * @see <a href="https://www.gnu.org/software/coreutils/manual/coreutils.html#md5sum-invocation">GNU Coreutils: md5sum</a> 089 * @since 1.9.0 090 */ 091@Singleton 092@Named(SummaryFileTrustedChecksumsSource.NAME) 093public final class SummaryFileTrustedChecksumsSource extends FileTrustedChecksumsSourceSupport { 094 public static final String NAME = "summaryFile"; 095 096 private static final String CONFIG_PROPS_PREFIX = 097 FileTrustedChecksumsSourceSupport.CONFIG_PROPS_PREFIX + NAME + "."; 098 099 /** 100 * Is checksum source enabled? 101 * 102 * @configurationSource {@link RepositorySystemSession#getConfigProperties()} 103 * @configurationType {@link java.lang.Boolean} 104 * @configurationDefaultValue false 105 */ 106 public static final String CONFIG_PROP_ENABLED = FileTrustedChecksumsSourceSupport.CONFIG_PROPS_PREFIX + NAME; 107 108 /** 109 * The basedir where checksums are. If relative, is resolved from local repository root. 110 * 111 * @configurationSource {@link RepositorySystemSession#getConfigProperties()} 112 * @configurationType {@link java.lang.String} 113 * @configurationDefaultValue {@link #LOCAL_REPO_PREFIX_DIR} 114 */ 115 public static final String CONFIG_PROP_BASEDIR = CONFIG_PROPS_PREFIX + "basedir"; 116 117 public static final String LOCAL_REPO_PREFIX_DIR = ".checksums"; 118 119 /** 120 * Is source origin aware? 121 * 122 * @configurationSource {@link RepositorySystemSession#getConfigProperties()} 123 * @configurationType {@link java.lang.Boolean} 124 * @configurationDefaultValue true 125 */ 126 public static final String CONFIG_PROP_ORIGIN_AWARE = CONFIG_PROPS_PREFIX + "originAware"; 127 128 public static final String CHECKSUMS_FILE_PREFIX = "checksums"; 129 130 private final LocalPathComposer localPathComposer; 131 132 private final RepositorySystemLifecycle repositorySystemLifecycle; 133 134 private final PathProcessor pathProcessor; 135 136 private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> checksums; 137 138 private final ConcurrentHashMap<Path, Boolean> changedChecksums; 139 140 private final AtomicBoolean onShutdownHandlerRegistered; 141 142 @Inject 143 public SummaryFileTrustedChecksumsSource( 144 RepositoryKeyFunctionFactory repoKeyFunctionFactory, 145 LocalPathComposer localPathComposer, 146 RepositorySystemLifecycle repositorySystemLifecycle, 147 PathProcessor pathProcessor) { 148 super(repoKeyFunctionFactory); 149 this.localPathComposer = requireNonNull(localPathComposer); 150 this.repositorySystemLifecycle = requireNonNull(repositorySystemLifecycle); 151 this.pathProcessor = requireNonNull(pathProcessor); 152 this.checksums = new ConcurrentHashMap<>(); 153 this.changedChecksums = new ConcurrentHashMap<>(); 154 this.onShutdownHandlerRegistered = new AtomicBoolean(false); 155 } 156 157 @Override 158 protected boolean isEnabled(RepositorySystemSession session) { 159 return ConfigUtils.getBoolean(session, false, CONFIG_PROP_ENABLED); 160 } 161 162 private boolean isOriginAware(RepositorySystemSession session) { 163 return ConfigUtils.getBoolean(session, true, CONFIG_PROP_ORIGIN_AWARE); 164 } 165 166 @Override 167 protected Map<String, String> doGetTrustedArtifactChecksums( 168 RepositorySystemSession session, 169 Artifact artifact, 170 ArtifactRepository artifactRepository, 171 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) { 172 return doGetTrustedPathChecksums( 173 session, 174 repositoryKey(session, artifactRepository).subList(0, 1), 175 rk -> localPathComposer.getPathForArtifact(artifact, false), 176 checksumAlgorithmFactories); 177 } 178 179 @Override 180 protected Map<String, String> doGetTrustedMetadataChecksums( 181 RepositorySystemSession session, 182 Metadata metadata, 183 ArtifactRepository artifactRepository, 184 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) { 185 return doGetTrustedPathChecksums( 186 session, 187 repositoryKey(session, artifactRepository), 188 rk -> localPathComposer.getPathForMetadata(metadata, rk), 189 checksumAlgorithmFactories); 190 } 191 192 /** 193 * Looks up the trusted checksums for given local-repository style path (the summary file 2nd column), be it 194 * an artifact or a metadata path. 195 */ 196 private Map<String, String> doGetTrustedPathChecksums( 197 RepositorySystemSession session, 198 List<String> repoKeys, 199 Function<String, String> pathComposer, 200 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) { 201 final HashMap<String, String> result = new HashMap<>(); 202 final Path basedir = getBasedir(session, LOCAL_REPO_PREFIX_DIR, CONFIG_PROP_BASEDIR, false); 203 if (Files.isDirectory(basedir)) { 204 final boolean originAware = isOriginAware(session); 205 for (String repoKey : repoKeys) { 206 for (ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories) { 207 Path summaryFile = 208 summaryFile(basedir, originAware, repoKey, checksumAlgorithmFactory.getFileExtension()); 209 ConcurrentHashMap<String, String> algorithmChecksums = 210 checksums.computeIfAbsent(summaryFile, f -> loadProvidedChecksums(summaryFile)); 211 String checksum = algorithmChecksums.get(pathComposer.apply(repoKey)); 212 if (checksum != null) { 213 result.putIfAbsent(checksumAlgorithmFactory.getName(), checksum); 214 } 215 } 216 } 217 } 218 return result; 219 } 220 221 @Override 222 protected Writer doGetTrustedArtifactChecksumsWriter(RepositorySystemSession session) { 223 if (onShutdownHandlerRegistered.compareAndSet(false, true)) { 224 repositorySystemLifecycle.addOnSystemEndedHandler(this::saveRecordedLines); 225 } 226 return new SummaryFileWriter( 227 checksums, 228 getBasedir(session, LOCAL_REPO_PREFIX_DIR, CONFIG_PROP_BASEDIR, true), 229 isOriginAware(session), 230 r -> repositoryKey(session, r).get(0)); 231 } 232 233 /** 234 * Returns the summary file path. The file itself and its parent directories may not exist, this method merely 235 * calculate the path. 236 */ 237 private Path summaryFile(Path basedir, boolean originAware, String safeRepositoryId, String checksumExtension) { 238 String fileName = CHECKSUMS_FILE_PREFIX; 239 if (originAware) { 240 // defense in depth: the repository key is spliced into the summary file name, 241 // so it must be a safe path segment 242 PathUtils.validatePathComponent(safeRepositoryId, "repository key"); 243 fileName += "-" + safeRepositoryId; 244 } 245 return basedir.resolve(fileName + "." + checksumExtension); 246 } 247 248 private ConcurrentHashMap<String, String> loadProvidedChecksums(Path summaryFile) { 249 ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>(); 250 if (Files.isRegularFile(summaryFile)) { 251 try (BufferedReader reader = Files.newBufferedReader(summaryFile, StandardCharsets.UTF_8)) { 252 String line; 253 while ((line = reader.readLine()) != null) { 254 if (!line.startsWith("#") && !line.isEmpty()) { 255 String[] parts = line.split(" ", 2); 256 if (parts.length == 2) { 257 String newChecksum = parts[0]; 258 String artifactPath = parts[1]; 259 String oldChecksum = result.put(artifactPath, newChecksum); 260 if (oldChecksum != null) { 261 if (Objects.equals(oldChecksum, newChecksum)) { 262 logger.warn( 263 "Checksums file '{}' contains duplicate checksums for artifact {}: {}", 264 summaryFile, 265 artifactPath, 266 oldChecksum); 267 } else { 268 logger.warn( 269 "Checksums file '{}' contains different checksums for artifact {}: " 270 + "old '{}' replaced by new '{}'", 271 summaryFile, 272 artifactPath, 273 oldChecksum, 274 newChecksum); 275 } 276 } 277 } else { 278 logger.warn("Checksums file '{}' ignored malformed line '{}'", summaryFile, line); 279 } 280 } 281 } 282 } catch (IOException e) { 283 throw new UncheckedIOException(e); 284 } 285 logger.info("Loaded {} trusted checksums from {}", result.size(), summaryFile); 286 } 287 return result; 288 } 289 290 private class SummaryFileWriter implements Writer { 291 private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache; 292 293 private final Path basedir; 294 295 private final boolean originAware; 296 297 private final Function<ArtifactRepository, String> repositoryKeyFunction; 298 299 private SummaryFileWriter( 300 ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache, 301 Path basedir, 302 boolean originAware, 303 Function<ArtifactRepository, String> repositoryKeyFunction) { 304 this.cache = cache; 305 this.basedir = basedir; 306 this.originAware = originAware; 307 this.repositoryKeyFunction = repositoryKeyFunction; 308 } 309 310 @Override 311 public void addTrustedArtifactChecksums( 312 Artifact artifact, 313 ArtifactRepository artifactRepository, 314 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories, 315 Map<String, String> trustedArtifactChecksums) { 316 addTrustedPathChecksums( 317 artifact, 318 localPathComposer.getPathForArtifact(artifact, false), 319 artifactRepository, 320 checksumAlgorithmFactories, 321 trustedArtifactChecksums); 322 } 323 324 @Override 325 public void addTrustedMetadataChecksums( 326 Metadata metadata, 327 ArtifactRepository artifactRepository, 328 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories, 329 Map<String, String> trustedMetadataChecksums) { 330 addTrustedPathChecksums( 331 metadata, 332 localPathComposer.getPathForMetadata(metadata, repositoryKeyFunction.apply(artifactRepository)), 333 artifactRepository, 334 checksumAlgorithmFactories, 335 trustedMetadataChecksums); 336 } 337 338 private void addTrustedPathChecksums( 339 Object subject, 340 String path, 341 ArtifactRepository artifactRepository, 342 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories, 343 Map<String, String> trustedChecksums) { 344 for (ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories) { 345 Path summaryFile = summaryFile( 346 basedir, 347 originAware, 348 repositoryKeyFunction.apply(artifactRepository), 349 checksumAlgorithmFactory.getFileExtension()); 350 String checksum = requireNonNull(trustedChecksums.get(checksumAlgorithmFactory.getName())); 351 352 String oldChecksum = cache.computeIfAbsent(summaryFile, k -> loadProvidedChecksums(summaryFile)) 353 .put(path, checksum); 354 355 if (oldChecksum == null) { 356 changedChecksums.put(summaryFile, Boolean.TRUE); // new 357 } else if (!Objects.equals(oldChecksum, checksum)) { 358 changedChecksums.put(summaryFile, Boolean.TRUE); // replaced 359 logger.info("Trusted checksum for {} replaced: old {}, new {}", subject, oldChecksum, checksum); 360 } 361 } 362 } 363 } 364 365 /** 366 * On-close handler that saves recorded checksums, if any. 367 */ 368 private void saveRecordedLines() { 369 if (changedChecksums.isEmpty()) { 370 return; 371 } 372 373 ArrayList<Exception> exceptions = new ArrayList<>(); 374 for (Map.Entry<Path, ConcurrentHashMap<String, String>> entry : checksums.entrySet()) { 375 Path summaryFile = entry.getKey(); 376 if (changedChecksums.get(summaryFile) != Boolean.TRUE) { 377 continue; 378 } 379 ConcurrentHashMap<String, String> recordedLines = entry.getValue(); 380 if (!recordedLines.isEmpty()) { 381 try { 382 ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>(); 383 result.putAll(loadProvidedChecksums(summaryFile)); 384 result.putAll(recordedLines); 385 386 logger.info("Saving {} checksums to '{}'", result.size(), summaryFile); 387 pathProcessor.writeWithBackup( 388 summaryFile, 389 result.entrySet().stream() 390 .sorted(Map.Entry.comparingByKey()) 391 .map(e -> e.getValue() + " " + e.getKey()) 392 .collect(Collectors.joining(System.lineSeparator()))); 393 } catch (IOException e) { 394 exceptions.add(e); 395 } 396 } 397 } 398 MultiRuntimeException.mayThrow("session save checksums failure", exceptions); 399 } 400}