View Javadoc

1   package org.apache.maven.plugins.scmpublish;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *  http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collection;
27  import java.util.HashSet;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.io.FileUtils;
34  import org.apache.commons.io.FilenameUtils;
35  import org.apache.commons.lang3.time.DurationFormatUtils;
36  import org.apache.maven.plugin.AbstractMojo;
37  import org.apache.maven.plugin.MojoExecutionException;
38  import org.apache.maven.plugin.MojoFailureException;
39  import org.apache.maven.plugins.annotations.Component;
40  import org.apache.maven.plugins.annotations.Parameter;
41  import org.apache.maven.scm.CommandParameter;
42  import org.apache.maven.scm.CommandParameters;
43  import org.apache.maven.scm.ScmBranch;
44  import org.apache.maven.scm.ScmException;
45  import org.apache.maven.scm.ScmFileSet;
46  import org.apache.maven.scm.ScmResult;
47  import org.apache.maven.scm.command.add.AddScmResult;
48  import org.apache.maven.scm.command.checkin.CheckInScmResult;
49  import org.apache.maven.scm.manager.NoSuchScmProviderException;
50  import org.apache.maven.scm.manager.ScmManager;
51  import org.apache.maven.scm.provider.ScmProvider;
52  import org.apache.maven.scm.provider.ScmUrlUtils;
53  import org.apache.maven.scm.provider.svn.AbstractSvnScmProvider;
54  import org.apache.maven.scm.provider.svn.repository.SvnScmProviderRepository;
55  import org.apache.maven.scm.repository.ScmRepository;
56  import org.apache.maven.scm.repository.ScmRepositoryException;
57  import org.apache.maven.settings.Server;
58  import org.apache.maven.settings.Settings;
59  import org.apache.maven.settings.crypto.DefaultSettingsDecryptionRequest;
60  import org.apache.maven.settings.crypto.SettingsDecrypter;
61  import org.apache.maven.settings.crypto.SettingsDecryptionRequest;
62  import org.apache.maven.settings.crypto.SettingsDecryptionResult;
63  import org.apache.maven.shared.release.config.ReleaseDescriptor;
64  import org.apache.maven.shared.release.scm.ScmRepositoryConfigurator;
65  
66  /**
67   * Base class for the scm-publish mojos.
68   */
69  public abstract class AbstractScmPublishMojo
70      extends AbstractMojo
71  {
72      /**
73       * Location of the scm publication tree:
74       * <code>scm:&lt;scm_provider&gt;&lt;delimiter&gt;&lt;provider_specific_part&gt;</code>.
75       * Example:
76       * <code>scm:svn:https://svn.apache.org/repos/infra/websites/production/maven/content/plugins/maven-scm-publish-plugin-LATEST/</code>
77       */
78      @Parameter ( property = "scmpublish.pubScmUrl", defaultValue = "${project.distributionManagement.site.url}",
79                   required = true )
80      protected String pubScmUrl;
81  
82      /**
83       * If the checkout directory exists and this flag is activated, the plugin will try an SCM-update instead
84       * of delete then checkout.
85       */
86      @Parameter ( property = "scmpublish.tryUpdate", defaultValue = "false" )
87      protected boolean tryUpdate;
88  
89     /**
90       * Location where the scm check-out is done. By default, scm checkout is done in build (target) directory,
91       * which is deleted on every <code>mvn clean</code>. To avoid this and get better performance, configure
92       * this location outside build structure and set <code>tryUpdate</code> to <code>true</code>.
93       * See <a href="http://maven.apache.org/plugins/maven-scm-publish-plugin/various-tips.html#Improving_SCM_Checkout_Performance">
94       * Improving SCM Checkout Performance</a> for more information.
95       */
96      @Parameter ( property = "scmpublish.checkoutDirectory",
97                   defaultValue = "${project.build.directory}/scmpublish-checkout" )
98      protected File checkoutDirectory;
99  
100     /**
101      * Display list of added, deleted, and changed files, but do not do any actual SCM operations.
102      */
103     @Parameter ( property = "scmpublish.dryRun" )
104     private boolean dryRun;
105 
106     /**
107      * Run add and delete commands, but leave the actually checkin for the user to run manually.
108      */
109     @Parameter ( property = "scmpublish.skipCheckin" )
110     private boolean skipCheckin;
111 
112     /**
113      * SCM log/checkin comment for this publication.
114      */
115     @Parameter ( property = "scmpublish.checkinComment", defaultValue = "Site checkin for project ${project.name}" )
116     private String checkinComment;
117 
118     /**
119      * Patterns to exclude from the scm tree.
120      */
121     @Parameter
122     protected String excludes;
123 
124     /**
125      * Patterns to include in the scm tree.
126      */
127     @Parameter
128     protected String includes;
129 
130     /**
131      * List of SCM provider implementations.
132      * Key is the provider type, eg. <code>cvs</code>.
133      * Value is the provider implementation (the role-hint of the provider), eg. <code>cvs</code> or <code>cvs_native</code>.
134      * @see ScmManager.setScmProviderImplementation
135      */
136     @Parameter
137     private Map<String, String> providerImplementations;
138 
139     /**
140      * The SCM manager.
141      */
142     @Component
143     private ScmManager scmManager;
144 
145     /**
146      * Tool that gets a configured SCM repository from release configuration.
147      */
148     @Component
149     protected ScmRepositoryConfigurator scmRepositoryConfigurator;
150     
151     /**
152      * The serverId specified in the settings.xml, which should be used for the authentication.
153      */
154     @Parameter
155     private String serverId;
156 
157     /**
158      * The SCM username to use.
159      */
160     @Parameter ( property = "username" )
161     protected String username;
162 
163     /**
164      * The SCM password to use.
165      */
166     @Parameter ( property = "password" )
167     protected String password;
168 
169     /**
170      * Use a local checkout instead of doing a checkout from the upstream repository. <b>WARNING</b>: This will only work
171      * with distributed SCMs which support the file:// protocol
172      * TODO: we should think about having the defaults for the various SCM providers provided via Modello!
173      */
174     @Parameter ( property = "localCheckout", defaultValue = "false" )
175     protected boolean localCheckout;
176 
177     /**
178      * The outputEncoding parameter of the site plugin. This plugin will corrupt your site
179      * if this does not match the value used by the site plugin.
180      */
181     @Parameter ( property = "outputEncoding", defaultValue = "${project.reporting.outputEncoding}" )
182     protected String siteOutputEncoding;
183 
184     /**
185      * Do not delete files to the scm
186      */
187     @Parameter ( property = "scmpublish.skipDeletedFiles", defaultValue = "false" )
188     protected boolean skipDeletedFiles;
189 
190     /**
191      */
192     @Parameter ( defaultValue = "${basedir}", readonly = true )
193     protected File basedir;
194 
195     /**
196      */
197     @Component
198     protected Settings settings;
199     
200     @Component
201     private SettingsDecrypter settingsDecrypter;
202  
203 
204     /**
205      * Collections of paths not to delete when checking content to delete.
206      * If your site has subdirectories published by an other mechanism/build
207      */
208     @Parameter
209     protected String[] ignorePathsToDelete;
210 
211     /**
212      * SCM branch to use. For github, you must configure with <code>gh-pages</code>.
213      */
214     @Parameter ( property = "scmpublish.scm.branch" )
215     protected String scmBranch;
216 
217     /**
218      * Configure svn automatic remote url creation.
219      */
220     @Parameter ( property = "scmpublish.automaticRemotePathCreation", defaultValue = "true" )
221     protected boolean automaticRemotePathCreation;
222 
223     /**
224      * Filename extensions of files which need new line normalization.
225      */
226     private final static String[] NORMALIZE_EXTENSIONS = { "html", "css", "js" };
227 
228     /**
229      * Extra file extensions to normalize line ending (will be added to default
230      * <code>html</code>,<code>css</code>,<code>js</code> list)
231      */
232     @Parameter
233     protected String[] extraNormalizeExtensions;
234 
235     private Set<String> normalizeExtensions;
236 
237     protected ScmProvider scmProvider;
238 
239     protected ScmRepository scmRepository;
240 
241     protected void logInfo( String format, Object... params )
242     {
243         getLog().info( String.format( format, params ) );
244     }
245 
246     protected void logWarn( String format, Object... params )
247     {
248         getLog().warn( String.format( format, params ) );
249     }
250 
251     protected void logError( String format, Object... params )
252     {
253         getLog().error( String.format( format, params ) );
254     }
255 
256     private File relativize( File base, File file )
257     {
258         return new File( base.toURI().relativize( file.toURI() ).getPath() );
259     }
260 
261     protected boolean requireNormalizeNewlines( File f )
262         throws IOException
263     {
264         if ( normalizeExtensions == null )
265         {
266             normalizeExtensions = new HashSet<String>( Arrays.asList( NORMALIZE_EXTENSIONS ) );
267             if ( extraNormalizeExtensions != null )
268             {
269                 normalizeExtensions.addAll( Arrays.asList( extraNormalizeExtensions ) );
270             }
271         }
272 
273         return FilenameUtils.isExtension( f.getName(), normalizeExtensions );
274     }
275 
276     private ReleaseDescriptor setupScm()
277         throws ScmRepositoryException, NoSuchScmProviderException
278     {
279         String scmUrl;
280         if ( localCheckout )
281         {
282             // in the release phase we have to change the checkout URL
283             // to do a local checkout instead of going over the network.
284 
285             String provider = ScmUrlUtils.getProvider( pubScmUrl );
286             String delimiter = ScmUrlUtils.getDelimiter( pubScmUrl );
287             
288             String providerPart = "scm:" + provider + delimiter;
289 
290             // X TODO: also check the information from releaseDescriptor.getScmRelativePathProjectDirectory()
291             // X TODO: in case our toplevel git directory has no pom.
292             // X TODO: fix pathname once I understand this.
293             scmUrl = providerPart + "file://" + "target/localCheckout";
294             logInfo( "Performing a LOCAL checkout from " + scmUrl );
295         }
296 
297         ReleaseDescriptor releaseDescriptor = new ReleaseDescriptor();
298         releaseDescriptor.setInteractive( settings.isInteractiveMode() );
299 
300         if ( username == null || password == null )
301         {
302             for ( Server server : settings.getServers() )
303             {
304                 if ( server.getId().equals( serverId ) )
305                 {
306                     SettingsDecryptionRequest decryptionRequest = new DefaultSettingsDecryptionRequest( server );
307 
308                     SettingsDecryptionResult decryptionResult = settingsDecrypter.decrypt( decryptionRequest );
309 
310                     if ( !decryptionResult.getProblems().isEmpty() )
311                     {
312                         // todo throw exception?
313                     }
314 
315                     if ( username == null )
316                     {
317                         username = decryptionResult.getServer().getUsername();
318                     }
319 
320                     if ( password == null )
321                     {
322                         password = decryptionResult.getServer().getPassword();
323                     }
324 
325                     break;
326                 }
327             }
328         }
329 
330         releaseDescriptor.setScmPassword( password );
331         releaseDescriptor.setScmUsername( username );
332 
333         releaseDescriptor.setWorkingDirectory( basedir.getAbsolutePath() );
334         releaseDescriptor.setLocalCheckout( localCheckout );
335         releaseDescriptor.setScmSourceUrl( pubScmUrl );
336 
337         if ( providerImplementations != null )
338         {
339             for ( Map.Entry<String, String> providerEntry : providerImplementations.entrySet() )
340             {
341                 logInfo( "Changing the default '%s' provider implementation to '%s'.", providerEntry.getKey(),
342                          providerEntry.getValue() );
343                 scmManager.setScmProviderImplementation( providerEntry.getKey(), providerEntry.getValue() );
344             }
345         }
346 
347         scmRepository = scmRepositoryConfigurator.getConfiguredRepository( releaseDescriptor, settings );
348 
349         scmProvider = scmRepositoryConfigurator.getRepositoryProvider( scmRepository );
350 
351         return releaseDescriptor;
352     }
353 
354     protected void checkoutExisting()
355         throws MojoExecutionException
356     {
357 
358         if ( scmProvider instanceof AbstractSvnScmProvider )
359         {
360             checkCreateRemoteSvnPath();
361         }
362 
363         logInfo( "%s the pub tree from %s into %s", ( tryUpdate ? "Updating" : "Checking out" ), pubScmUrl, checkoutDirectory );
364 
365         if ( checkoutDirectory.exists() && !tryUpdate )
366 
367         {
368             try
369             {
370                 FileUtils.deleteDirectory( checkoutDirectory );
371             }
372             catch ( IOException e )
373             {
374                 logError( e.getMessage() );
375 
376                 throw new MojoExecutionException( "Unable to remove old checkout directory: " + e.getMessage(), e );
377             }
378         }
379 
380         boolean forceCheckout = false;
381 
382         if ( !checkoutDirectory.exists() )
383 
384         {
385             if ( tryUpdate )
386             {
387                 logInfo( "TryUpdate is configured but no local copy currently available: forcing checkout." );
388             }
389             checkoutDirectory.mkdirs();
390             forceCheckout = true;
391         }
392 
393         try
394         {
395             ScmFileSet fileSet = new ScmFileSet( checkoutDirectory, includes, excludes );
396 
397             ScmResult scmResult = null;
398             if ( tryUpdate && !forceCheckout )
399             {
400                 scmResult = scmProvider.update( scmRepository, fileSet );
401             }
402             else
403             {
404                 int attempt = 0;
405                 while ( scmResult == null )
406                 {
407                     try
408                     {
409                         if ( scmBranch == null )
410                         {
411                             scmResult = scmProvider.checkOut( scmRepository, fileSet );
412                         }
413                         else
414                         {
415                             ScmBranch scmBranch = new ScmBranch( this.scmBranch );
416                             scmResult = scmProvider.checkOut( scmRepository, fileSet, scmBranch );
417                         }
418                     }
419                     catch ( ScmException e )
420                     {
421                         // give it max 2 times to retry
422                         if ( attempt++ < 2 )
423                         {
424                             try
425                             {
426                                 // wait 3 seconds
427                                 Thread.sleep( 3 * 1000 );
428                             }
429                             catch ( InterruptedException ie )
430                             {
431                                 // noop
432                             }
433                         }
434                         else
435                         {
436                             throw e;
437                         }
438                     }
439                 }
440             }
441             checkScmResult( scmResult, "check out from SCM" );
442         }
443         catch ( ScmException e )
444         {
445             logError( e.getMessage() );
446 
447             throw new MojoExecutionException( "An error occurred during the checkout process: " + e.getMessage(), e );
448         }
449         catch ( IOException e )
450         {
451             logError( e.getMessage() );
452 
453             throw new MojoExecutionException( "An error occurred during the checkout process: " + e.getMessage(), e );
454         }
455     }
456 
457     private void checkCreateRemoteSvnPath()
458         throws MojoExecutionException
459     {
460         getLog().debug( "AbstractSvnScmProvider used, so we can check if remote url exists and eventually create it." );
461         AbstractSvnScmProvider svnScmProvider = (AbstractSvnScmProvider) scmProvider;
462 
463         try
464         {
465             boolean remoteExists = svnScmProvider.remoteUrlExist( scmRepository.getProviderRepository(), null );
466 
467             if ( remoteExists )
468             {
469                 return;
470             }
471         }
472         catch ( ScmException e )
473         {
474             throw new MojoExecutionException( e.getMessage(), e );
475         }
476 
477         String remoteUrl = ( (SvnScmProviderRepository) scmRepository.getProviderRepository() ).getUrl();
478 
479         if ( !automaticRemotePathCreation )
480         {
481             // olamy: return ?? that will fail during checkout IMHO :-)
482             logWarn( "Remote svn url %s does not exist and automatic remote path creation disabled.",
483                      remoteUrl );
484             return;
485         }
486 
487         logInfo( "Remote svn url %s does not exist: creating.", remoteUrl );
488 
489         File baseDir = null;
490         try
491         {
492 
493             // create a temporary directory for svnexec
494             baseDir = File.createTempFile( "scm", "tmp" );
495             baseDir.delete();
496             baseDir.mkdirs();
497             // to prevent fileSet cannot be empty
498             ScmFileSet scmFileSet = new ScmFileSet( baseDir, new File( "" ) );
499 
500             CommandParameters commandParameters = new CommandParameters();
501             commandParameters.setString( CommandParameter.SCM_MKDIR_CREATE_IN_LOCAL, Boolean.FALSE.toString() );
502             commandParameters.setString( CommandParameter.MESSAGE, "Automatic svn path creation: " + remoteUrl );
503             svnScmProvider.mkdir( scmRepository.getProviderRepository(), scmFileSet, commandParameters );
504 
505             // new remote url so force checkout!
506             if ( checkoutDirectory.exists() )
507             {
508                 FileUtils.deleteDirectory( checkoutDirectory );
509             }
510         }
511         catch ( IOException e )
512         {
513             throw new MojoExecutionException( e.getMessage(), e );
514         }
515         catch ( ScmException e )
516         {
517             throw new MojoExecutionException( e.getMessage(), e );
518         }
519         finally
520         {
521             if ( baseDir != null )
522             {
523                 try
524                 {
525                     FileUtils.forceDeleteOnExit( baseDir );
526                 }
527                 catch ( IOException e )
528                 {
529                     throw new MojoExecutionException( e.getMessage(), e );
530                 }
531             }
532         }
533     }
534 
535     public void execute()
536         throws MojoExecutionException, MojoFailureException
537     {
538         // setup the scm plugin with help from release plugin utilities
539         try
540         {
541             setupScm();
542         }
543         catch ( ScmRepositoryException e )
544         {
545             throw new MojoExecutionException( e.getMessage(), e );
546         }
547         catch ( NoSuchScmProviderException e )
548         {
549             throw new MojoExecutionException( e.getMessage(), e );
550         }
551 
552         boolean tmpCheckout = false;
553 
554         if ( checkoutDirectory.getPath().contains( "${project." ) )
555         {
556             try
557             {
558                 tmpCheckout = true;
559                 checkoutDirectory = File.createTempFile( "maven-scm-publish", ".checkout" );
560                 checkoutDirectory.delete();
561                 checkoutDirectory.mkdir();
562             }
563             catch ( IOException ioe )
564             {
565                 throw new MojoExecutionException( ioe.getMessage(), ioe );
566             }
567         }
568 
569         try
570         {
571             scmPublishExecute();
572         }
573         finally
574         {
575             if ( tmpCheckout )
576             {
577                 FileUtils.deleteQuietly( checkoutDirectory );
578             }
579         }
580     }
581 
582     /**
583      * Check-in content from scm checkout.
584      *
585      * @throws MojoExecutionException
586      */
587     protected void checkinFiles()
588         throws MojoExecutionException
589     {
590         if ( skipCheckin )
591         {
592             return;
593         }
594 
595         ScmFileSet updatedFileSet = new ScmFileSet( checkoutDirectory );
596         try
597         {
598             long start = System.currentTimeMillis();
599 
600             CheckInScmResult checkinResult =
601                 checkScmResult( scmProvider.checkIn( scmRepository, updatedFileSet, new ScmBranch( scmBranch ),
602                                                      checkinComment ), "check-in files to SCM" );
603 
604             logInfo( "Checked in %d file(s) to revision %s in %s", checkinResult.getCheckedInFiles().size(),
605                      checkinResult.getScmRevision(),
606                      DurationFormatUtils.formatPeriod( start, System.currentTimeMillis(), "H' h 'm' m 's' s'" ) );
607         }
608         catch ( ScmException e )
609         {
610             throw new MojoExecutionException( "Failed to perform SCM checkin", e );
611         }
612     }
613 
614     protected void deleteFiles( Collection<File> deleted )
615         throws MojoExecutionException
616     {
617         if ( skipDeletedFiles )
618         {
619             logInfo( "Deleting files is skipped." );
620             return;
621         }
622         List<File> deletedList = new ArrayList<File>();
623         for ( File f : deleted )
624         {
625             deletedList.add( relativize( checkoutDirectory, f ) );
626         }
627         ScmFileSet deletedFileSet = new ScmFileSet( checkoutDirectory, deletedList );
628         try
629         {
630             getLog().debug( "Deleting files: " + deletedList );
631 
632             checkScmResult( scmProvider.remove( scmRepository, deletedFileSet, "Deleting obsolete site files." ),
633                             "delete files from SCM" );
634         }
635         catch ( ScmException e )
636         {
637             throw new MojoExecutionException( "Failed to delete removed files to SCM", e );
638         }
639     }
640 
641     /**
642      * Add files to scm.
643      *
644      * @param added files to be added
645      * @throws MojoFailureException
646      * @throws MojoExecutionException
647      */
648     protected void addFiles( Collection<File> added )
649         throws MojoFailureException, MojoExecutionException
650     {
651         List<File> addedList = new ArrayList<File>();
652         Set<File> createdDirs = new HashSet<File>();
653         Set<File> dirsToAdd = new TreeSet<File>();
654 
655         createdDirs.add( relativize( checkoutDirectory, checkoutDirectory ) );
656 
657         for ( File f : added )
658         {
659             for ( File dir = f.getParentFile(); !dir.equals( checkoutDirectory ); dir = dir.getParentFile() )
660             {
661                 File relativized = relativize( checkoutDirectory, dir );
662                 //  we do the best we can with the directories
663                 if ( createdDirs.add( relativized ) )
664                 {
665                     dirsToAdd.add( relativized );
666                 }
667                 else
668                 {
669                     break;
670                 }
671             }
672             addedList.add( relativize( checkoutDirectory, f ) );
673         }
674 
675         for ( File relativized : dirsToAdd )
676         {
677             try
678             {
679                 ScmFileSet fileSet = new ScmFileSet( checkoutDirectory, relativized );
680                 getLog().debug( "scm add directory: " + relativized );
681                 AddScmResult addDirResult = scmProvider.add( scmRepository, fileSet, "Adding directory" );
682                 if ( !addDirResult.isSuccess() )
683                 {
684                     getLog().debug( " Error adding directory " + relativized + ": " + addDirResult.getCommandOutput() );
685                 }
686             }
687             catch ( ScmException e )
688             {
689                 //
690             }
691         }
692 
693         // remove directories already added !
694         addedList.removeAll( dirsToAdd );
695 
696         ScmFileSet addedFileSet = new ScmFileSet( checkoutDirectory, addedList );
697         getLog().debug( "scm add files: " + addedList );
698         try
699         {
700 
701                 CommandParameters commandParameters = new CommandParameters();
702                 commandParameters.setString( CommandParameter.MESSAGE, "Adding new site files." );
703                 commandParameters.setString( CommandParameter.FORCE_ADD, Boolean.TRUE.toString() );
704                 checkScmResult( scmProvider.add( scmRepository, addedFileSet, commandParameters ),
705                                 "add new files to SCM" );
706 
707         }
708         catch ( ScmException e )
709         {
710             throw new MojoExecutionException( "Failed to add new files to SCM", e );
711         }
712     }
713 
714     private<T extends ScmResult> T checkScmResult( T result, String failure )
715         throws MojoExecutionException
716     {
717         if ( !result.isSuccess() )
718         {
719             String msg = "Failed to " + failure + ": " + result.getProviderMessage() + " " + result.getCommandOutput();
720             logError( msg );
721             throw new MojoExecutionException( msg );
722         }
723         return result;
724     }
725 
726     public boolean isDryRun()
727     {
728         return dryRun;
729     }
730 
731     public abstract void scmPublishExecute()
732         throws MojoExecutionException, MojoFailureException;
733 
734     public void setPubScmUrl( String pubScmUrl )
735     {
736         // Fix required for Windows, which fit other OS as well
737         if ( pubScmUrl.startsWith( "scm:svn:" ) )
738         {
739             pubScmUrl = pubScmUrl.replaceFirst( "file:/[/]*", "file:///" );
740         }
741 
742         this.pubScmUrl = pubScmUrl;
743     }
744 
745 }