View Javadoc
1   package org.apache.maven.plugins.stage;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.maven.artifact.manager.WagonManager;
23  import org.apache.maven.artifact.repository.ArtifactRepository;
24  import org.apache.maven.artifact.repository.metadata.Metadata;
25  import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader;
26  import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Writer;
27  import org.apache.maven.wagon.CommandExecutor;
28  import org.apache.maven.wagon.CommandExecutionException;
29  import org.apache.maven.wagon.ConnectionException;
30  import org.apache.maven.wagon.ResourceDoesNotExistException;
31  import org.apache.maven.wagon.TransferFailedException;
32  import org.apache.maven.wagon.UnsupportedProtocolException;
33  import org.apache.maven.wagon.Wagon;
34  import org.apache.maven.wagon.WagonException;
35  import org.apache.maven.wagon.authentication.AuthenticationException;
36  import org.apache.maven.wagon.authentication.AuthenticationInfo;
37  import org.apache.maven.wagon.authorization.AuthorizationException;
38  import org.apache.maven.wagon.repository.Repository;
39  import org.codehaus.plexus.logging.LogEnabled;
40  import org.codehaus.plexus.logging.Logger;
41  import org.codehaus.plexus.util.FileUtils;
42  import org.codehaus.plexus.util.IOUtil;
43  import org.codehaus.plexus.util.StringUtils;
44  import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
45  
46  import java.io.File;
47  import java.io.FileInputStream;
48  import java.io.FileOutputStream;
49  import java.io.FileReader;
50  import java.io.FileWriter;
51  import java.io.IOException;
52  import java.io.InputStream;
53  import java.io.OutputStream;
54  import java.io.PrintWriter;
55  import java.io.Reader;
56  import java.io.Writer;
57  import java.security.MessageDigest;
58  import java.security.NoSuchAlgorithmException;
59  import java.util.ArrayList;
60  import java.util.List;
61  import java.util.Set;
62  import java.util.TreeSet;
63  import java.util.zip.ZipEntry;
64  import java.util.zip.ZipOutputStream;
65  
66  /**
67   * @author Jason van Zyl
68   * @plexus.component
69   */
70  public class DefaultRepositoryCopier
71      implements LogEnabled, RepositoryCopier
72  {
73      private MetadataXpp3Reader reader = new MetadataXpp3Reader();
74  
75      private MetadataXpp3Writer writer = new MetadataXpp3Writer();
76  
77      /** @plexus.requirement */
78      private WagonManager wagonManager;
79  
80      private Logger logger;
81  
82      public void copy( Repository sourceRepository, Repository targetRepository, String version )
83          throws WagonException, IOException
84      {
85          String prefix = "staging-plugin";
86  
87          String fileName = prefix + "-" + version + ".zip";
88  
89          String tempdir = System.getProperty( "java.io.tmpdir" );
90  
91          logger.debug( "Writing all output to " + tempdir );
92  
93          // Create the renameScript script
94  
95          String renameScriptName = prefix + "-" + version + "-rename.sh";
96  
97          File renameScript = new File( tempdir, renameScriptName );
98  
99          // Work directory
100 
101         File basedir = new File( tempdir, prefix + "-" + version );
102 
103         FileUtils.deleteDirectory( basedir );
104 
105         basedir.mkdirs();
106 
107         Wagon sourceWagon = wagonManager.getWagon( sourceRepository );
108         AuthenticationInfo sourceAuth = wagonManager.getAuthenticationInfo( sourceRepository.getId() );
109 
110         sourceWagon.connect( sourceRepository, sourceAuth );
111 
112         logger.info( "Looking for files in the source repository." );
113 
114         List<String> files = new ArrayList<String>();
115 
116         scan( sourceWagon, "", files );
117 
118         logger.info( "Downloading files from the source repository to: " + basedir );
119 
120         for ( String s : files )
121         {
122 
123             if ( s.contains( ".svn" ) )
124             {
125                 continue;
126             }
127 
128             File f = new File( basedir, s );
129 
130             FileUtils.mkdir( f.getParentFile().getAbsolutePath() );
131 
132             logger.info( "Downloading file from the source repository: " + s );
133 
134             sourceWagon.get( s, f );
135         }
136 
137         // ----------------------------------------------------------------------------
138         // Now all the files are present locally and now we are going to grab the
139         // metadata files from the targetRepositoryUrl and pull those down locally
140         // so that we can merge the metadata.
141         // ----------------------------------------------------------------------------
142 
143         logger.info( "Downloading metadata from the target repository." );
144 
145         Wagon targetWagon = wagonManager.getWagon( targetRepository );
146 
147         if ( ! ( targetWagon instanceof CommandExecutor ) )
148         {
149             throw new CommandExecutionException( "Wagon class '" + targetWagon.getClass().getName()
150                 + "' in use for target repository is not a CommandExecutor" );
151         }
152 
153         AuthenticationInfo targetAuth = wagonManager.getAuthenticationInfo( targetRepository.getId() );
154 
155         targetWagon.connect( targetRepository, targetAuth );
156 
157         PrintWriter rw = new PrintWriter( new FileWriter( renameScript ) );
158 
159         File archive = new File( tempdir, fileName );
160 
161         for ( String s : files )
162         {
163 
164             if ( s.startsWith( "/" ) )
165             {
166                 s = s.substring( 1 );
167             }
168 
169             if ( s.endsWith( MAVEN_METADATA ) )
170             {
171                 File emf = new File( basedir, s + IN_PROCESS_MARKER );
172 
173                 try
174                 {
175                     targetWagon.get( s, emf );
176                 }
177                 catch ( ResourceDoesNotExistException e )
178                 {
179                     // We don't have an equivalent on the targetRepositoryUrl side because we have something
180                     // new on the sourceRepositoryUrl side so just skip the metadata merging.
181 
182                     continue;
183                 }
184 
185                 try
186                 {
187                     mergeMetadata( emf );
188                 }
189                 catch ( XmlPullParserException e )
190                 {
191                     throw new IOException( "Metadata file is corrupt " + s + " Reason: " + e.getMessage() );
192                 }
193             }
194         }
195 
196         Set moveCommands = new TreeSet();
197 
198         // ----------------------------------------------------------------------------
199         // Create the Zip file that we will deploy to the targetRepositoryUrl stage
200         // ----------------------------------------------------------------------------
201 
202         logger.info( "Creating zip file." );
203 
204         OutputStream os = new FileOutputStream( archive );
205 
206         ZipOutputStream zos = new ZipOutputStream( os );
207 
208         scanDirectory( basedir, basedir, zos, version, moveCommands );
209 
210         // ----------------------------------------------------------------------------
211         // Create the renameScript script. This is as atomic as we can
212         // ----------------------------------------------------------------------------
213 
214         logger.info( "Creating rename script." );
215 
216         for ( Object moveCommand : moveCommands )
217         {
218             String s = (String) moveCommand;
219 
220             // We use an explicit unix '\n' line-ending here instead of using the println() method.
221             // Using println() will cause files and folders to have a '\r' at the end if the plugin is run on Windows.
222             rw.print( s + "\n" );
223         }
224 
225         IOUtil.close( rw );
226 
227         ZipEntry e = new ZipEntry( renameScript.getName() );
228 
229         zos.putNextEntry( e );
230 
231         InputStream is = new FileInputStream( renameScript );
232 
233         IOUtil.copy( is, zos );
234 
235         IOUtil.close( is );
236 
237         IOUtil.close( zos );
238 
239         sourceWagon.disconnect();
240 
241         // Push the Zip to the target system
242 
243         logger.info( "Uploading zip file to the target repository." );
244 
245         targetWagon.put( archive, fileName );
246 
247         logger.info( "Unpacking zip file on the target machine." );
248 
249         String targetRepoBaseDirectory = targetRepository.getBasedir();
250 
251         // We use the super quiet option here as all the noise seems to kill/stall the connection
252 
253         String command = "unzip -o -qq -d " + targetRepoBaseDirectory + " " + targetRepoBaseDirectory + "/" + fileName;
254 
255         ( (CommandExecutor) targetWagon ).executeCommand( command );
256 
257         logger.info( "Deleting zip file from the target repository." );
258 
259         command = "rm -f " + targetRepoBaseDirectory + "/" + fileName;
260 
261         ( (CommandExecutor) targetWagon ).executeCommand( command );
262 
263         logger.info( "Running rename script on the target machine." );
264 
265         command = "cd " + targetRepoBaseDirectory + "; sh " + renameScriptName;
266 
267         ( (CommandExecutor) targetWagon ).executeCommand( command );
268 
269         logger.info( "Deleting rename script from the target repository." );
270 
271         command = "rm -f " + targetRepoBaseDirectory + "/" + renameScriptName;
272 
273         ( (CommandExecutor) targetWagon ).executeCommand( command );
274 
275         targetWagon.disconnect();
276     }
277 
278     private void scanDirectory( File basedir, File dir, ZipOutputStream zos, String version, Set moveCommands )
279         throws IOException
280     {
281         if ( dir == null )
282         {
283             return;
284         }
285 
286         File[] files = dir.listFiles();
287 
288         for ( File f : files )
289         {
290             if ( f.isDirectory() )
291             {
292                 if ( f.getName().equals( ".svn" ) )
293                 {
294                     continue;
295                 }
296 
297                 if ( f.getName().endsWith( version ) )
298                 {
299                     String s = f.getAbsolutePath().substring( basedir.getAbsolutePath().length() + 1 );
300                     s = StringUtils.replace( s, "\\", "/" );
301 
302                     moveCommands.add( "mv " + s + IN_PROCESS_MARKER + " " + s );
303                 }
304 
305                 scanDirectory( basedir, f, zos, version, moveCommands );
306             }
307             else
308             {
309                 InputStream is = new FileInputStream( f );
310 
311                 String s = f.getAbsolutePath().substring( basedir.getAbsolutePath().length() + 1 );
312                 s = StringUtils.replace( s, "\\", "/" );
313 
314                 // We are marking any version directories with the in-process flag so that
315                 // anything being unpacked on the target side will not be recogized by Maven
316                 // and so users cannot download partially uploaded files.
317 
318                 String vtag = "/" + version;
319 
320                 s = StringUtils.replace( s, vtag + "/", vtag + IN_PROCESS_MARKER + "/" );
321 
322                 ZipEntry e = new ZipEntry( s );
323 
324                 zos.putNextEntry( e );
325 
326                 IOUtil.copy( is, zos );
327 
328                 IOUtil.close( is );
329 
330                 int idx = s.indexOf( IN_PROCESS_MARKER );
331 
332                 if ( idx > 0 )
333                 {
334                     String d = s.substring( 0, idx );
335 
336                     moveCommands.add( "mv " + d + IN_PROCESS_MARKER + " " + d );
337                 }
338             }
339         }
340     }
341 
342     private void mergeMetadata( File existingMetadata )
343         throws IOException, XmlPullParserException
344     {
345         // Existing Metadata in target stage
346 
347         Reader existingMetadataReader = new FileReader( existingMetadata );
348 
349         Metadata existing = reader.read( existingMetadataReader );
350 
351         // Staged Metadata
352 
353         File stagedMetadataFile = new File( existingMetadata.getParentFile(), MAVEN_METADATA );
354 
355         Reader stagedMetadataReader = new FileReader( stagedMetadataFile );
356 
357         Metadata staged = reader.read( stagedMetadataReader );
358 
359         // Merge
360 
361         existing.merge( staged );
362 
363         Writer writer = new FileWriter( existingMetadata );
364 
365         this.writer.write( writer, existing );
366 
367         IOUtil.close( writer );
368 
369         IOUtil.close( stagedMetadataReader );
370 
371         IOUtil.close( existingMetadataReader );
372 
373         // Mark all metadata as in-process and regenerate the checksums as they will be different
374         // after the merger
375 
376         try
377         {
378             File newMd5 = new File( existingMetadata.getParentFile(), MAVEN_METADATA + ".md5" + IN_PROCESS_MARKER );
379 
380             FileUtils.fileWrite( newMd5.getAbsolutePath(), checksum( existingMetadata, MD5 ) );
381 
382             File oldMd5 = new File( existingMetadata.getParentFile(), MAVEN_METADATA + ".md5" );
383 
384             oldMd5.delete();
385 
386             File newSha1 = new File( existingMetadata.getParentFile(), MAVEN_METADATA + ".sha1" + IN_PROCESS_MARKER );
387 
388             FileUtils.fileWrite( newSha1.getAbsolutePath(), checksum( existingMetadata, SHA1 ) );
389 
390             File oldSha1 = new File( existingMetadata.getParentFile(), MAVEN_METADATA + ".sha1" );
391 
392             oldSha1.delete();
393         }
394         catch ( NoSuchAlgorithmException e )
395         {
396             throw new RuntimeException( e );
397         }
398 
399         // We have the new merged copy so we're good
400 
401         stagedMetadataFile.delete();
402     }
403 
404     private String checksum( File file, String type )
405         throws IOException, NoSuchAlgorithmException
406     {
407         MessageDigest md5 = MessageDigest.getInstance( type );
408 
409         InputStream is = new FileInputStream( file );
410 
411         // CHECKSTYLE_OFF: MagicNumber
412         byte[] buf = new byte[8192];
413         // CHECKSTYLE_ON: MagicNumber
414 
415         int i;
416 
417         while ( ( i = is.read( buf ) ) > 0 )
418         {
419             md5.update( buf, 0, i );
420         }
421 
422         IOUtil.close( is );
423 
424         return encode( md5.digest() );
425     }
426 
427     protected String encode( byte[] binaryData )
428     {
429         // CHECKSTYLE_OFF: MagicNumber
430         if ( binaryData.length != 16 && binaryData.length != 20 )
431         {
432             int bitLength = binaryData.length * 8;
433             throw new IllegalArgumentException( "Unrecognised length for binary data: " + bitLength + " bits" );
434         }
435         // CHECKSTYLE_ON: MagicNumber
436 
437         String retValue = "";
438 
439         for ( byte aBinaryData : binaryData )
440         {
441             // CHECKSTYLE_OFF: MagicNumber
442             String t = Integer.toHexString( aBinaryData & 0xff );
443             // CHECKSTYLE_ON: MagicNumber
444 
445             if ( t.length() == 1 )
446             {
447                 retValue += ( "0" + t );
448             }
449             else
450             {
451                 retValue += t;
452             }
453         }
454 
455         return retValue.trim();
456     }
457 
458     private void scan( Wagon wagon, String basePath, List<String> collected )
459     {
460         try
461         {
462             List<String> files = wagon.getFileList( basePath );
463 
464             if ( files.isEmpty() )
465             {
466                 collected.add( basePath );
467             }
468             else
469             {
470                 basePath = basePath + "/";
471                 for ( String file : files )
472                 {
473                     logger.info( "Found file in the source repository: " + file );
474                     scan( wagon, basePath + file, collected );
475                 }
476             }
477         }
478         catch ( TransferFailedException e )
479         {
480             throw new RuntimeException( e );
481         }
482         catch ( ResourceDoesNotExistException e )
483         {
484             // is thrown when calling getFileList on a file
485             collected.add( basePath );
486         }
487         catch ( AuthorizationException e )
488         {
489             throw new RuntimeException( e );
490         }
491 
492     }
493 
494     protected List<String> scanForArtifactPaths( ArtifactRepository repository )
495     {
496         List<String> collected;
497         try
498         {
499             Wagon wagon = wagonManager.getWagon( repository.getProtocol() );
500             Repository artifactRepository = new Repository( repository.getId(), repository.getUrl() );
501             wagon.connect( artifactRepository );
502             collected = new ArrayList<String>();
503             scan( wagon, "/", collected );
504             wagon.disconnect();
505 
506             return collected;
507 
508         }
509         catch ( UnsupportedProtocolException e )
510         {
511             throw new RuntimeException( e );
512         }
513         catch ( ConnectionException e )
514         {
515             throw new RuntimeException( e );
516         }
517         catch ( AuthenticationException e )
518         {
519             throw new RuntimeException( e );
520         }
521     }
522 
523     public void enableLogging( Logger logger )
524     {
525         this.logger = logger;
526     }
527 }