View Javadoc
1   package org.eclipse.aether.internal.impl.checksum;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *  http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import javax.inject.Inject;
23  import javax.inject.Named;
24  import javax.inject.Singleton;
25  
26  import java.io.BufferedReader;
27  import java.io.IOException;
28  import java.io.UncheckedIOException;
29  import java.nio.charset.StandardCharsets;
30  import java.nio.file.Files;
31  import java.nio.file.Path;
32  import java.util.ArrayList;
33  import java.util.HashMap;
34  import java.util.List;
35  import java.util.Map;
36  import java.util.Objects;
37  import java.util.concurrent.ConcurrentHashMap;
38  import java.util.concurrent.atomic.AtomicBoolean;
39  
40  import org.eclipse.aether.MultiRuntimeException;
41  import org.eclipse.aether.RepositorySystemSession;
42  import org.eclipse.aether.artifact.Artifact;
43  import org.eclipse.aether.impl.RepositorySystemLifecycle;
44  import org.eclipse.aether.internal.impl.LocalPathComposer;
45  import org.eclipse.aether.repository.ArtifactRepository;
46  import org.eclipse.aether.spi.connector.checksum.ChecksumAlgorithmFactory;
47  import org.eclipse.aether.util.FileUtils;
48  import org.slf4j.Logger;
49  import org.slf4j.LoggerFactory;
50  
51  import static java.util.Objects.requireNonNull;
52  import static java.util.stream.Collectors.toList;
53  
54  /**
55   * Compact file {@link FileTrustedChecksumsSourceSupport} implementation that use specified directory as base
56   * directory, where it expects a "summary" file named as "checksums.${checksumExt}" for each checksum algorithm.
57   * File format is GNU Coreutils compatible: each line holds checksum followed by two spaces and artifact relative path
58   * (from local repository root, without leading "./"). This means that trusted checksums summary file can be used to
59   * validate artifacts or generate it using standard GNU tools like GNU {@code sha1sum} is (for BSD derivatives same
60   * file can be used with {@code -r} switch).
61   * <p>
62   * The format supports comments "#" (hash) and empty lines for easier structuring the file content, and both are
63   * ignored. Also, their presence makes the summary file incompatible with GNU Coreutils format. On save of the
64   * summary file, the comments and empty lines are lost, and file is sorted by path names for easier diffing
65   * (2nd column in file).
66   * <p>
67   * The source by default is "origin aware", and it will factor in origin repository ID as well into summary file name,
68   * for example "checksums-central.sha256".
69   * <p>
70   * Example commands for managing summary file (in examples will use repository ID "central"):
71   * <ul>
72   *     <li>To create summary file: {@code find * -not -name "checksums-central.sha256" -type f -print0 |
73   *       xargs -0 sha256sum | sort -k 2 > checksums-central.sha256}</li>
74   *     <li>To verify artifacts using summary file: {@code sha256sum --quiet -c checksums-central.sha256}</li>
75   * </ul>
76   * <p>
77   * The checksums summary file is lazily loaded and remains cached during lifetime of the component, so file changes
78   * during lifecycle of the component are not picked up. This implementation can be simultaneously used to lookup and
79   * also record checksums. The recorded checksums will become visible for every session, and will be flushed
80   * at repository system shutdown, merged with existing ones on disk.
81   * <p>
82   * The name of this implementation is "summaryFile".
83   *
84   * @see <a href="https://man7.org/linux/man-pages/man1/sha1sum.1.html">sha1sum man page</a>
85   * @see <a href="https://www.gnu.org/software/coreutils/manual/coreutils.html#md5sum-invocation">GNU Coreutils: md5sum</a>
86   * @since 1.9.0
87   */
88  @Singleton
89  @Named( SummaryFileTrustedChecksumsSource.NAME )
90  public final class SummaryFileTrustedChecksumsSource
91          extends FileTrustedChecksumsSourceSupport
92  {
93      public static final String NAME = "summaryFile";
94  
95      private static final String CHECKSUMS_FILE_PREFIX = "checksums";
96  
97      private static final Logger LOGGER = LoggerFactory.getLogger( SummaryFileTrustedChecksumsSource.class );
98  
99      private final LocalPathComposer localPathComposer;
100 
101     private final RepositorySystemLifecycle repositorySystemLifecycle;
102 
103     private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> checksums;
104 
105     private final ConcurrentHashMap<Path, Boolean> changedChecksums;
106 
107     private final AtomicBoolean onShutdownHandlerRegistered;
108 
109 
110     @Inject
111     public SummaryFileTrustedChecksumsSource( LocalPathComposer localPathComposer,
112                                               RepositorySystemLifecycle repositorySystemLifecycle )
113     {
114         super( NAME );
115         this.localPathComposer = requireNonNull( localPathComposer );
116         this.repositorySystemLifecycle = requireNonNull( repositorySystemLifecycle );
117         this.checksums = new ConcurrentHashMap<>();
118         this.changedChecksums = new ConcurrentHashMap<>();
119         this.onShutdownHandlerRegistered = new AtomicBoolean( false );
120     }
121 
122     @Override
123     protected Map<String, String> doGetTrustedArtifactChecksums(
124             RepositorySystemSession session, Artifact artifact, ArtifactRepository artifactRepository,
125             List<ChecksumAlgorithmFactory> checksumAlgorithmFactories )
126     {
127         final HashMap<String, String> result = new HashMap<>();
128         final Path basedir = getBasedir( session, false );
129         if ( Files.isDirectory( basedir ) )
130         {
131             final String artifactPath = localPathComposer.getPathForArtifact( artifact, false );
132             final boolean originAware = isOriginAware( session );
133             for ( ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories )
134             {
135                 Path summaryFile = summaryFile( basedir, originAware, artifactRepository.getId(),
136                         checksumAlgorithmFactory.getFileExtension() );
137                 ConcurrentHashMap<String, String> algorithmChecksums = checksums.computeIfAbsent( summaryFile, f ->
138                         {
139                             ConcurrentHashMap<String, String> loaded = loadProvidedChecksums( summaryFile );
140                             if ( Files.isRegularFile( summaryFile ) )
141                             {
142                                 LOGGER.info( "Loaded {} {} trusted checksums for remote repository {}",
143                                         loaded.size(), checksumAlgorithmFactory.getName(), artifactRepository.getId() );
144                             }
145                             return loaded;
146                         }
147                 );
148                 String checksum = algorithmChecksums.get( artifactPath );
149                 if ( checksum != null )
150                 {
151                     result.put( checksumAlgorithmFactory.getName(), checksum );
152                 }
153             }
154         }
155         return result;
156     }
157 
158     @Override
159     protected SummaryFileWriter doGetTrustedArtifactChecksumsWriter( RepositorySystemSession session )
160     {
161         if ( onShutdownHandlerRegistered.compareAndSet( false, true ) )
162         {
163             repositorySystemLifecycle.addOnSystemEndedHandler( this::saveRecordedLines );
164         }
165         return new SummaryFileWriter( checksums, getBasedir( session, true ), isOriginAware( session ) );
166     }
167 
168     /**
169      * Returns the summary file path. The file itself and its parent directories may not exist, this method merely
170      * calculate the path.
171      */
172     private Path summaryFile( Path basedir, boolean originAware, String repositoryId, String checksumExtension )
173     {
174         String fileName = CHECKSUMS_FILE_PREFIX;
175         if ( originAware )
176         {
177             fileName += "-" + repositoryId;
178         }
179         return basedir.resolve( fileName + "." + checksumExtension );
180     }
181 
182     private ConcurrentHashMap<String, String> loadProvidedChecksums( Path summaryFile )
183     {
184         ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();
185         if ( Files.isRegularFile( summaryFile ) )
186         {
187             try ( BufferedReader reader = Files.newBufferedReader( summaryFile, StandardCharsets.UTF_8 ) )
188             {
189                 String line;
190                 while ( ( line = reader.readLine() ) != null )
191                 {
192                     if ( !line.startsWith( "#" ) && !line.isEmpty() )
193                     {
194                         String[] parts = line.split( "  ", 2 );
195                         if ( parts.length == 2 )
196                         {
197                             String newChecksum = parts[0];
198                             String artifactPath = parts[1];
199                             String oldChecksum = result.put( artifactPath, newChecksum );
200                             if ( oldChecksum != null )
201                             {
202                                 if ( Objects.equals( oldChecksum, newChecksum ) )
203                                 {
204                                     LOGGER.warn(
205                                             "Checksums file '{}' contains duplicate checksums for artifact {}: {}",
206                                             summaryFile, artifactPath, oldChecksum );
207                                 }
208                                 else
209                                 {
210                                     LOGGER.warn(
211                                             "Checksums file '{}' contains different checksums for artifact {}: "
212                                                     + "old '{}' replaced by new '{}'", summaryFile, artifactPath,
213                                             oldChecksum, newChecksum );
214                                 }
215                             }
216                         }
217                         else
218                         {
219                             LOGGER.warn( "Checksums file '{}' ignored malformed line '{}'", summaryFile, line );
220                         }
221                     }
222                 }
223             }
224             catch ( IOException e )
225             {
226                 throw new UncheckedIOException( e );
227             }
228         }
229         return result;
230     }
231 
232     private class SummaryFileWriter implements Writer
233     {
234         private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache;
235 
236         private final Path basedir;
237 
238         private final boolean originAware;
239 
240         private SummaryFileWriter( ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache,
241                                    Path basedir,
242                                    boolean originAware )
243         {
244             this.cache = cache;
245             this.basedir = basedir;
246             this.originAware = originAware;
247         }
248 
249         @Override
250         public void addTrustedArtifactChecksums( Artifact artifact,
251                                                  ArtifactRepository artifactRepository,
252                                                  List<ChecksumAlgorithmFactory> checksumAlgorithmFactories,
253                                                  Map<String, String> trustedArtifactChecksums )
254         {
255             String artifactPath = localPathComposer.getPathForArtifact( artifact, false );
256             for ( ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories )
257             {
258                 Path summaryFile = summaryFile( basedir, originAware, artifactRepository.getId(),
259                         checksumAlgorithmFactory.getFileExtension() );
260                 String checksum = requireNonNull(
261                         trustedArtifactChecksums.get( checksumAlgorithmFactory.getName() ) );
262 
263                 String oldChecksum = cache.computeIfAbsent( summaryFile, k -> loadProvidedChecksums( summaryFile ) )
264                         .put( artifactPath, checksum );
265 
266                 if ( oldChecksum == null )
267                 {
268                     changedChecksums.put( summaryFile, Boolean.TRUE ); // new
269                 }
270                 else if ( !Objects.equals( oldChecksum, checksum ) )
271                 {
272                     changedChecksums.put( summaryFile, Boolean.TRUE ); // replaced
273                     LOGGER.info( "Trusted checksum for artifact {} replaced: old {}, new {}",
274                             artifact, oldChecksum, checksum );
275                 }
276             }
277         }
278     }
279 
280     /**
281      * On-close handler that saves recorded checksums, if any.
282      */
283     private void saveRecordedLines()
284     {
285         if ( changedChecksums.isEmpty() )
286         {
287             return;
288         }
289 
290         ArrayList<Exception> exceptions = new ArrayList<>();
291         for ( Map.Entry<Path, ConcurrentHashMap<String, String>> entry : checksums.entrySet() )
292         {
293             Path summaryFile = entry.getKey();
294             if ( changedChecksums.get( summaryFile ) != Boolean.TRUE )
295             {
296                 continue;
297             }
298             ConcurrentHashMap<String, String> recordedLines = entry.getValue();
299             if ( !recordedLines.isEmpty() )
300             {
301                 try
302                 {
303                     ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();
304                     result.putAll( loadProvidedChecksums( summaryFile ) );
305                     result.putAll( recordedLines );
306 
307                     LOGGER.info( "Saving {} checksums to '{}'", result.size(), summaryFile );
308                     FileUtils.writeFileWithBackup(
309                             summaryFile,
310                             p -> Files.write( p,
311                                     result.entrySet().stream()
312                                             .sorted( Map.Entry.comparingByValue() )
313                                             .map( e -> e.getValue() + "  " + e.getKey() )
314                                             .collect( toList() )
315                             )
316                     );
317                 }
318                 catch ( IOException e )
319                 {
320                     exceptions.add( e );
321                 }
322             }
323         }
324         MultiRuntimeException.mayThrow( "session save checksums failure", exceptions );
325     }
326 }