View Javadoc

1   package org.apache.maven.plugins.stage;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.maven.artifact.manager.WagonManager;
23  import org.apache.maven.artifact.repository.ArtifactRepository;
24  import org.apache.maven.artifact.repository.metadata.Metadata;
25  import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader;
26  import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Writer;
27  import org.apache.maven.wagon.CommandExecutor;
28  import org.apache.maven.wagon.CommandExecutionException;
29  import org.apache.maven.wagon.ConnectionException;
30  import org.apache.maven.wagon.ResourceDoesNotExistException;
31  import org.apache.maven.wagon.TransferFailedException;
32  import org.apache.maven.wagon.UnsupportedProtocolException;
33  import org.apache.maven.wagon.Wagon;
34  import org.apache.maven.wagon.WagonException;
35  import org.apache.maven.wagon.authentication.AuthenticationException;
36  import org.apache.maven.wagon.authentication.AuthenticationInfo;
37  import org.apache.maven.wagon.authorization.AuthorizationException;
38  import org.apache.maven.wagon.repository.Repository;
39  import org.codehaus.plexus.logging.LogEnabled;
40  import org.codehaus.plexus.logging.Logger;
41  import org.codehaus.plexus.util.FileUtils;
42  import org.codehaus.plexus.util.IOUtil;
43  import org.codehaus.plexus.util.StringUtils;
44  import org.codehaus.plexus.util.xml.pull.XmlPullParserException;
45  
46  import java.io.File;
47  import java.io.FileInputStream;
48  import java.io.FileOutputStream;
49  import java.io.FileReader;
50  import java.io.FileWriter;
51  import java.io.IOException;
52  import java.io.InputStream;
53  import java.io.OutputStream;
54  import java.io.PrintWriter;
55  import java.io.Reader;
56  import java.io.Writer;
57  import java.security.MessageDigest;
58  import java.security.NoSuchAlgorithmException;
59  import java.util.ArrayList;
60  import java.util.Iterator;
61  import java.util.List;
62  import java.util.Set;
63  import java.util.TreeSet;
64  import java.util.zip.ZipEntry;
65  import java.util.zip.ZipOutputStream;
66  
67  /**
68   * @author Jason van Zyl
69   * @plexus.component
70   */
71  public class DefaultRepositoryCopier
72      implements LogEnabled, RepositoryCopier
73  {
74      private MetadataXpp3Reader reader = new MetadataXpp3Reader();
75  
76      private MetadataXpp3Writer writer = new MetadataXpp3Writer();
77  
78      /** @plexus.requirement */
79      private WagonManager wagonManager;
80  
81      private Logger logger;
82  
83      public void copy( Repository sourceRepository, Repository targetRepository, String version )
84          throws WagonException, IOException
85      {
86          String prefix = "staging-plugin";
87  
88          String fileName = prefix + "-" + version + ".zip";
89  
90          String tempdir = System.getProperty( "java.io.tmpdir" );
91  
92          logger.debug( "Writing all output to " + tempdir );
93  
94          // Create the renameScript script
95  
96          String renameScriptName = prefix + "-" + version + "-rename.sh";
97  
98          File renameScript = new File( tempdir, renameScriptName );
99  
100         // Work directory
101 
102         File basedir = new File( tempdir, prefix + "-" + version );
103 
104         FileUtils.deleteDirectory( basedir );
105 
106         basedir.mkdirs();
107 
108         String protocol = sourceRepository.getProtocol();
109 
110         Wagon sourceWagon = wagonManager.getWagon( sourceRepository );
111         AuthenticationInfo sourceAuth = wagonManager.getAuthenticationInfo( sourceRepository.getId() );
112 
113         sourceWagon.connect( sourceRepository, sourceAuth );
114 
115         logger.info( "Looking for files in the source repository." );
116 
117         List files = new ArrayList();
118 
119         scan( sourceWagon, "", files );
120 
121         logger.info( "Downloading files from the source repository to: " + basedir );
122 
123         for ( Iterator i = files.iterator(); i.hasNext(); )
124         {
125             String s = (String) i.next();
126 
127             if ( s.indexOf( ".svn" ) >= 0 )
128             {
129                 continue;
130             }
131 
132             File f = new File( basedir, s );
133 
134             FileUtils.mkdir( f.getParentFile().getAbsolutePath() );
135 
136             logger.info( "Downloading file from the source repository: " + s );
137 
138             sourceWagon.get( s, f );
139         }
140 
141         // ----------------------------------------------------------------------------
142         // Now all the files are present locally and now we are going to grab the
143         // metadata files from the targetRepositoryUrl and pull those down locally
144         // so that we can merge the metadata.
145         // ----------------------------------------------------------------------------
146 
147         logger.info( "Downloading metadata from the target repository." );
148 
149         Wagon targetWagon = wagonManager.getWagon( targetRepository );
150 
151         if ( ! ( targetWagon instanceof CommandExecutor ) )
152         {
153             throw new CommandExecutionException( "Wagon class '" + targetWagon.getClass().getName() +
154                 "' in use for target repository is not a CommandExecutor" );
155         }
156 
157         AuthenticationInfo targetAuth = wagonManager.getAuthenticationInfo( targetRepository.getId() );
158 
159         targetWagon.connect( targetRepository, targetAuth );
160 
161         PrintWriter rw = new PrintWriter( new FileWriter( renameScript ) );
162 
163         File archive = new File( tempdir, fileName );
164 
165         for ( Iterator i = files.iterator(); i.hasNext(); )
166         {
167             String s = (String) i.next();
168 
169             if ( s.startsWith( "/" ) )
170             {
171                 s = s.substring( 1 );
172             }
173 
174             if ( s.endsWith( MAVEN_METADATA ) )
175             {
176                 File emf = new File( basedir, s + IN_PROCESS_MARKER );
177 
178                 try
179                 {
180                     targetWagon.get( s, emf );
181                 }
182                 catch ( ResourceDoesNotExistException e )
183                 {
184                     // We don't have an equivalent on the targetRepositoryUrl side because we have something
185                     // new on the sourceRepositoryUrl side so just skip the metadata merging.
186 
187                     continue;
188                 }
189 
190                 try
191                 {
192                     mergeMetadata( emf );
193                 }
194                 catch ( XmlPullParserException e )
195                 {
196                     throw new IOException( "Metadata file is corrupt " + s + " Reason: " + e.getMessage() );
197                 }
198             }
199         }
200 
201         Set moveCommands = new TreeSet();
202 
203         // ----------------------------------------------------------------------------
204         // Create the Zip file that we will deploy to the targetRepositoryUrl stage
205         // ----------------------------------------------------------------------------
206 
207         logger.info( "Creating zip file." );
208 
209         OutputStream os = new FileOutputStream( archive );
210 
211         ZipOutputStream zos = new ZipOutputStream( os );
212 
213         scanDirectory( basedir, basedir, zos, version, moveCommands );
214 
215         // ----------------------------------------------------------------------------
216         // Create the renameScript script. This is as atomic as we can
217         // ----------------------------------------------------------------------------
218 
219         logger.info( "Creating rename script." );
220 
221         for ( Iterator i = moveCommands.iterator(); i.hasNext(); )
222         {
223             String s = (String) i.next();
224 
225             // We use an explicit unix '\n' line-ending here instead of using the println() method.
226             // Using println() will cause files and folders to have a '\r' at the end if the plugin is run on Windows.
227             rw.print( s + "\n" );
228         }
229 
230         IOUtil.close( rw );
231 
232         ZipEntry e = new ZipEntry( renameScript.getName() );
233 
234         zos.putNextEntry( e );
235 
236         InputStream is = new FileInputStream( renameScript );
237 
238         IOUtil.copy( is, zos );
239 
240         IOUtil.close( is );
241 
242         IOUtil.close( zos );
243 
244         sourceWagon.disconnect();
245 
246         // Push the Zip to the target system
247 
248         logger.info( "Uploading zip file to the target repository." );
249 
250         targetWagon.put( archive, fileName );
251 
252         logger.info( "Unpacking zip file on the target machine." );
253 
254         String targetRepoBaseDirectory = targetRepository.getBasedir();
255 
256         // We use the super quiet option here as all the noise seems to kill/stall the connection
257 
258         String command = "unzip -o -qq -d " + targetRepoBaseDirectory + " " + targetRepoBaseDirectory + "/" + fileName;
259 
260         ( (CommandExecutor) targetWagon ).executeCommand( command );
261 
262         logger.info( "Deleting zip file from the target repository." );
263 
264         command = "rm -f " + targetRepoBaseDirectory + "/" + fileName;
265 
266         ( (CommandExecutor) targetWagon ).executeCommand( command );
267 
268         logger.info( "Running rename script on the target machine." );
269 
270         command = "cd " + targetRepoBaseDirectory + "; sh " + renameScriptName;
271 
272         ( (CommandExecutor) targetWagon ).executeCommand( command );
273 
274         logger.info( "Deleting rename script from the target repository." );
275 
276         command = "rm -f " + targetRepoBaseDirectory + "/" + renameScriptName;
277 
278         ( (CommandExecutor) targetWagon ).executeCommand( command );
279 
280         targetWagon.disconnect();
281     }
282 
283     private void scanDirectory( File basedir, File dir, ZipOutputStream zos, String version, Set moveCommands )
284         throws IOException
285     {
286         if ( dir == null )
287         {
288             return;
289         }
290 
291         File[] files = dir.listFiles();
292 
293         for ( int i = 0; i < files.length; i++ )
294         {
295             File f = files[i];
296 
297             if ( f.isDirectory() )
298             {
299                 if ( f.getName().equals( ".svn" ) )
300                 {
301                     continue;
302                 }
303 
304                 if ( f.getName().endsWith( version ) )
305                 {
306                     String s = f.getAbsolutePath().substring( basedir.getAbsolutePath().length() + 1 );
307                     s = StringUtils.replace( s, "\\", "/" );
308 
309                     moveCommands.add( "mv " + s + IN_PROCESS_MARKER + " " + s );
310                 }
311 
312                 scanDirectory( basedir, f, zos, version, moveCommands );
313             }
314             else
315             {
316                 InputStream is = new FileInputStream( f );
317 
318                 String s = f.getAbsolutePath().substring( basedir.getAbsolutePath().length() + 1 );                
319                 s = StringUtils.replace( s, "\\", "/" );
320 
321                 // We are marking any version directories with the in-process flag so that
322                 // anything being unpacked on the target side will not be recogized by Maven
323                 // and so users cannot download partially uploaded files.
324 
325                 String vtag = "/" + version;
326 
327                 s = StringUtils.replace( s, vtag + "/", vtag + IN_PROCESS_MARKER + "/" );
328 
329                 ZipEntry e = new ZipEntry( s );
330 
331                 zos.putNextEntry( e );
332 
333                 IOUtil.copy( is, zos );
334 
335                 IOUtil.close( is );
336 
337                 int idx = s.indexOf( IN_PROCESS_MARKER );
338 
339                 if ( idx > 0 )
340                 {
341                     String d = s.substring( 0, idx );
342 
343                     moveCommands.add( "mv " + d + IN_PROCESS_MARKER + " " + d );
344                 }
345             }
346         }
347     }
348 
349     private void mergeMetadata( File existingMetadata )
350         throws IOException, XmlPullParserException
351     {
352         // Existing Metadata in target stage
353 
354         Reader existingMetadataReader = new FileReader( existingMetadata );
355 
356         Metadata existing = reader.read( existingMetadataReader );
357 
358         // Staged Metadata
359 
360         File stagedMetadataFile = new File( existingMetadata.getParentFile(), MAVEN_METADATA );
361 
362         Reader stagedMetadataReader = new FileReader( stagedMetadataFile );
363 
364         Metadata staged = reader.read( stagedMetadataReader );
365 
366         // Merge
367 
368         existing.merge( staged );
369 
370         Writer writer = new FileWriter( existingMetadata );
371 
372         this.writer.write( writer, existing );
373 
374         IOUtil.close( writer );
375 
376         IOUtil.close( stagedMetadataReader );
377 
378         IOUtil.close( existingMetadataReader );
379 
380         // Mark all metadata as in-process and regenerate the checksums as they will be different
381         // after the merger
382 
383         try
384         {
385             File newMd5 = new File( existingMetadata.getParentFile(), MAVEN_METADATA + ".md5" + IN_PROCESS_MARKER );
386 
387             FileUtils.fileWrite( newMd5.getAbsolutePath(), checksum( existingMetadata, MD5 ) );
388 
389             File oldMd5 = new File( existingMetadata.getParentFile(), MAVEN_METADATA + ".md5" );
390 
391             oldMd5.delete();
392 
393             File newSha1 = new File( existingMetadata.getParentFile(), MAVEN_METADATA + ".sha1" + IN_PROCESS_MARKER );
394 
395             FileUtils.fileWrite( newSha1.getAbsolutePath(), checksum( existingMetadata, SHA1 ) );
396 
397             File oldSha1 = new File( existingMetadata.getParentFile(), MAVEN_METADATA + ".sha1" );
398 
399             oldSha1.delete();
400         }
401         catch ( NoSuchAlgorithmException e )
402         {
403             throw new RuntimeException( e );
404         }
405 
406         // We have the new merged copy so we're good
407 
408         stagedMetadataFile.delete();
409     }
410 
411     private String checksum( File file,
412                              String type )
413         throws IOException, NoSuchAlgorithmException
414     {
415         MessageDigest md5 = MessageDigest.getInstance( type );
416 
417         InputStream is = new FileInputStream( file );
418 
419         byte[] buf = new byte[8192];
420 
421         int i;
422 
423         while ( ( i = is.read( buf ) ) > 0 )
424         {
425             md5.update( buf, 0, i );
426         }
427 
428         IOUtil.close( is );
429 
430         return encode( md5.digest() );
431     }
432 
433     protected String encode( byte[] binaryData )
434     {
435         if ( binaryData.length != 16 && binaryData.length != 20 )
436         {
437             int bitLength = binaryData.length * 8;
438             throw new IllegalArgumentException( "Unrecognised length for binary data: " + bitLength + " bits" );
439         }
440 
441         String retValue = "";
442 
443         for ( int i = 0; i < binaryData.length; i++ )
444         {
445             String t = Integer.toHexString( binaryData[i] & 0xff );
446 
447             if ( t.length() == 1 )
448             {
449                 retValue += ( "0" + t );
450             }
451             else
452             {
453                 retValue += t;
454             }
455         }
456 
457         return retValue.trim();
458     }
459 
460     private void scan( Wagon wagon,
461                        String basePath,
462                        List collected )
463     {
464         try
465         {
466             List files = wagon.getFileList( basePath );
467 
468             if ( files.isEmpty() )
469             {
470                 collected.add( basePath );
471             }
472             else
473             {
474                 basePath = basePath + "/";
475                 for ( Iterator iterator = files.iterator(); iterator.hasNext(); )
476                 {
477                     String file = (String) iterator.next();
478                     logger.info( "Found file in the source repository: " + file );
479                     scan( wagon, basePath + file, collected );
480                 }
481             }
482         }
483         catch ( TransferFailedException e )
484         {
485             throw new RuntimeException( e );
486         }
487         catch ( ResourceDoesNotExistException e )
488         {
489             // is thrown when calling getFileList on a file
490             collected.add( basePath );
491         }
492         catch ( AuthorizationException e )
493         {
494             throw new RuntimeException( e );
495         }
496 
497     }
498 
499     protected List scanForArtifactPaths( ArtifactRepository repository )
500     {
501         List collected;
502         try
503         {
504             Wagon wagon = wagonManager.getWagon( repository.getProtocol() );
505             Repository artifactRepository = new Repository( repository.getId(), repository.getUrl() );
506             wagon.connect( artifactRepository );
507             collected = new ArrayList();
508             scan( wagon, "/", collected );
509             wagon.disconnect();
510 
511             return collected;
512 
513         }
514         catch ( UnsupportedProtocolException e )
515         {
516             throw new RuntimeException( e );
517         }
518         catch ( ConnectionException e )
519         {
520             throw new RuntimeException( e );
521         }
522         catch ( AuthenticationException e )
523         {
524             throw new RuntimeException( e );
525         }
526     }
527 
528     public void enableLogging( Logger logger )
529     {
530         this.logger = logger;
531     }
532 }