View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.plugins.scmpublish;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.nio.file.Files;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collection;
27  import java.util.HashSet;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.io.FileUtils;
34  import org.apache.commons.io.FilenameUtils;
35  import org.apache.commons.lang3.time.DurationFormatUtils;
36  import org.apache.maven.plugin.AbstractMojo;
37  import org.apache.maven.plugin.MojoExecutionException;
38  import org.apache.maven.plugin.MojoFailureException;
39  import org.apache.maven.plugins.annotations.Component;
40  import org.apache.maven.plugins.annotations.Parameter;
41  import org.apache.maven.scm.CommandParameter;
42  import org.apache.maven.scm.CommandParameters;
43  import org.apache.maven.scm.ScmBranch;
44  import org.apache.maven.scm.ScmException;
45  import org.apache.maven.scm.ScmFileSet;
46  import org.apache.maven.scm.ScmResult;
47  import org.apache.maven.scm.command.add.AddScmResult;
48  import org.apache.maven.scm.command.checkin.CheckInScmResult;
49  import org.apache.maven.scm.manager.NoSuchScmProviderException;
50  import org.apache.maven.scm.manager.ScmManager;
51  import org.apache.maven.scm.provider.ScmProvider;
52  import org.apache.maven.scm.provider.ScmUrlUtils;
53  import org.apache.maven.scm.provider.svn.AbstractSvnScmProvider;
54  import org.apache.maven.scm.provider.svn.repository.SvnScmProviderRepository;
55  import org.apache.maven.scm.repository.ScmRepository;
56  import org.apache.maven.scm.repository.ScmRepositoryException;
57  import org.apache.maven.settings.Settings;
58  import org.apache.maven.shared.release.config.ReleaseDescriptor;
59  import org.apache.maven.shared.release.config.ReleaseDescriptorBuilder;
60  import org.apache.maven.shared.release.scm.ScmRepositoryConfigurator;
61  import org.apache.maven.shared.utils.logging.MessageUtils;
62  
63  /**
64   * Base class for the scm-publish mojos.
65   */
66  public abstract class AbstractScmPublishMojo extends AbstractMojo {
67      // CHECKSTYLE_OFF: LineLength
68      /**
69       * Location of the scm publication tree:
70       * <code>scm:&lt;scm_provider&gt;&lt;delimiter&gt;&lt;provider_specific_part&gt;</code>.
71       * Example:
72       * <code>scm:svn:https://svn.apache.org/repos/infra/websites/production/maven/content/plugins/maven-scm-publish-plugin-LATEST/</code>
73       */
74      // CHECKSTYLE_ON: LineLength
75      @Parameter(
76              property = "scmpublish.pubScmUrl",
77              defaultValue = "${project.distributionManagement.site.url}",
78              required = true)
79      protected String pubScmUrl;
80  
81      /**
82       * If the checkout directory exists and this flag is activated, the plugin will try an SCM-update instead
83       * of delete then checkout.
84       */
85      @Parameter(property = "scmpublish.tryUpdate", defaultValue = "false")
86      protected boolean tryUpdate;
87  
88      // CHECKSTYLE_OFF: LineLength
89      /**
90       * Location where the scm check-out is done. By default, scm checkout is done in build (target) directory,
91       * which is deleted on every <code>mvn clean</code>. To avoid this and get better performance, configure
92       * this location outside build structure and set <code>tryUpdate</code> to <code>true</code>.
93       * See <a href="http://maven.apache.org/plugins/maven-scm-publish-plugin/various-tips.html#Improving_SCM_Checkout_Performance">
94       * Improving SCM Checkout Performance</a> for more information.
95       */
96      // CHECKSTYLE_ON: LineLength
97      @Parameter(
98              property = "scmpublish.checkoutDirectory",
99              defaultValue = "${project.build.directory}/scmpublish-checkout")
100     protected File checkoutDirectory;
101 
102     /**
103      * Location where the content is published inside the <code>${checkoutDirectory}</code>.
104      * By default, content is copyed at the root of <code>${checkoutDirectory}</code>.
105      */
106     @Parameter(property = "scmpublish.subDirectory")
107     protected String subDirectory;
108 
109     /**
110      * Display list of added, deleted, and changed files, but do not do any actual SCM operations.
111      */
112     @Parameter(property = "scmpublish.dryRun")
113     private boolean dryRun;
114 
115     /**
116      * Set this to 'true' to skip site deployment.
117      *
118      * @deprecated Please use {@link #skipDeployment}.
119      */
120     @Deprecated
121     @Parameter(defaultValue = "false")
122     private boolean skipDeployement;
123 
124     /**
125      * Set this to 'true' to skip site deployment.
126      */
127     @Parameter(property = "scmpublish.skipDeploy", alias = "maven.site.deploy.skip", defaultValue = "false")
128     private boolean skipDeployment;
129 
130     /**
131      * Run add and delete commands, but leave the actually checkin for the user to run manually.
132      */
133     @Parameter(property = "scmpublish.skipCheckin")
134     private boolean skipCheckin;
135 
136     /**
137      * SCM log/checkin comment for this publication.
138      */
139     @Parameter(property = "scmpublish.checkinComment", defaultValue = "Site checkin for project ${project.name}")
140     private String checkinComment;
141 
142     /**
143      * Patterns to exclude from the scm tree.
144      */
145     @Parameter
146     protected String excludes;
147 
148     /**
149      * Patterns to include in the scm tree.
150      */
151     @Parameter
152     protected String includes;
153 
154     /**
155      * List of SCM provider implementations.
156      * Key is the provider type, eg. <code>cvs</code>.
157      * Value is the provider implementation (the role-hint of the provider), eg. <code>cvs</code> or
158      * <code>cvs_native</code>.
159      * @see ScmManager.setScmProviderImplementation
160      */
161     @Parameter
162     private Map<String, String> providerImplementations;
163 
164     /**
165      * The SCM manager.
166      */
167     @Component
168     private ScmManager scmManager;
169 
170     /**
171      * Tool that gets a configured SCM repository from release configuration.
172      */
173     @Component
174     protected ScmRepositoryConfigurator scmRepositoryConfigurator;
175 
176     /**
177      * The server id specified in the {@code settings.xml}, which should be used for the authentication.
178      * @see <a href="https://maven.apache.org/settings.html#servers">Settings Reference</a>
179      */
180     @Parameter(property = "scmpublish.serverId", defaultValue = "${project.distributionManagement.site.id}")
181     private String serverId;
182 
183     /**
184      * The SCM username to use.
185      * This value takes precedence over the username derived from {@link #serverId}.
186      * @see #serverId
187      */
188     @Parameter(property = "username")
189     protected String username;
190 
191     /**
192      * The SCM password to use.
193      * This value takes precedence over the password derived from {@link #serverId}.
194      * @see #serverId
195      */
196     @Parameter(property = "password")
197     protected String password;
198 
199     /**
200      * Use a local checkout instead of doing a checkout from the upstream repository.
201      * <b>WARNING</b>: This will only work with distributed SCMs which support the file:// protocol.
202      * TODO: we should think about having the defaults for the various SCM providers provided via Modello!
203      */
204     @Parameter(property = "localCheckout", defaultValue = "false")
205     protected boolean localCheckout;
206 
207     /**
208      * The outputEncoding parameter of the site plugin. This plugin will corrupt your site
209      * if this does not match the value used by the site plugin.
210      */
211     @Parameter(property = "outputEncoding", defaultValue = "${project.reporting.outputEncoding}")
212     protected String siteOutputEncoding;
213 
214     /**
215      * Do not delete files to the scm
216      */
217     @Parameter(property = "scmpublish.skipDeletedFiles", defaultValue = "false")
218     protected boolean skipDeletedFiles;
219 
220     /**
221      * Add each directory in a separated SCM command: this can be necessary if SCM does not support
222      * adding subdirectories in one command.
223      */
224     @Parameter(defaultValue = "false")
225     protected boolean addUniqueDirectory;
226 
227     /**
228      */
229     @Parameter(defaultValue = "${basedir}", readonly = true)
230     protected File basedir;
231 
232     /**
233      */
234     @Parameter(defaultValue = "${settings}", readonly = true, required = true)
235     protected Settings settings;
236 
237     /**
238      * Collections of paths not to delete when checking content to delete.
239      * If your site has subdirectories published by an other mechanism/build
240      */
241     @Parameter
242     protected String[] ignorePathsToDelete;
243 
244     /**
245      * SCM branch to use. For github, you must configure with <code>gh-pages</code>.
246      */
247     @Parameter(property = "scmpublish.scm.branch")
248     protected String scmBranch;
249 
250     /**
251      * Configure svn automatic remote url creation.
252      */
253     @Parameter(property = "scmpublish.automaticRemotePathCreation", defaultValue = "true")
254     protected boolean automaticRemotePathCreation;
255 
256     /**
257      * Filename extensions of files which need new line normalization.
258      */
259     private static final String[] NORMALIZE_EXTENSIONS = {"html", "css", "js"};
260 
261     /**
262      * Extra file extensions to normalize line ending (will be added to default
263      * <code>html</code>,<code>css</code>,<code>js</code> list)
264      */
265     @Parameter
266     protected String[] extraNormalizeExtensions;
267 
268     private Set<String> normalizeExtensions;
269 
270     protected ScmProvider scmProvider;
271 
272     protected ScmRepository scmRepository;
273 
274     protected void logInfo(String format, Object... params) {
275         getLog().info(String.format(format, params));
276     }
277 
278     protected void logWarn(String format, Object... params) {
279         getLog().warn(String.format(format, params));
280     }
281 
282     protected void logError(String format, Object... params) {
283         getLog().error(String.format(format, params));
284     }
285 
286     private File relativize(File base, File file) {
287         return new File(base.toURI().relativize(file.toURI()).getPath());
288     }
289 
290     protected boolean requireNormalizeNewlines(File f) throws IOException {
291         if (normalizeExtensions == null) {
292             normalizeExtensions = new HashSet<>(Arrays.asList(NORMALIZE_EXTENSIONS));
293             if (extraNormalizeExtensions != null) {
294                 normalizeExtensions.addAll(Arrays.asList(extraNormalizeExtensions));
295             }
296         }
297 
298         return FilenameUtils.isExtension(f.getName(), normalizeExtensions);
299     }
300 
301     private void setupScm() throws ScmRepositoryException, NoSuchScmProviderException {
302         String scmUrl;
303         if (localCheckout) {
304             // in the release phase we have to change the checkout URL
305             // to do a local checkout instead of going over the network.
306 
307             String provider = ScmUrlUtils.getProvider(pubScmUrl);
308             String delimiter = ScmUrlUtils.getDelimiter(pubScmUrl);
309 
310             String providerPart = "scm:" + provider + delimiter;
311 
312             // X TODO: also check the information from releaseDescriptor.getScmRelativePathProjectDirectory()
313             // X TODO: in case our toplevel git directory has no pom.
314             // X TODO: fix pathname once I understand this.
315             scmUrl = providerPart + "file://" + "target/localCheckout";
316             logInfo("Performing a LOCAL checkout from " + scmUrl);
317         }
318 
319         ReleaseDescriptorBuilder descriptorBuilder = new ReleaseDescriptorBuilder();
320         descriptorBuilder.setInteractive(settings.isInteractiveMode());
321 
322         descriptorBuilder.setScmPassword(password);
323         descriptorBuilder.setScmUsername(username);
324         // used for lookup of credentials from settings.xml in DefaultScmRepositoryConfigurator
325         descriptorBuilder.setScmId(serverId);
326         descriptorBuilder.setWorkingDirectory(basedir.getAbsolutePath());
327         descriptorBuilder.setLocalCheckout(localCheckout);
328         descriptorBuilder.setScmSourceUrl(pubScmUrl);
329 
330         if (providerImplementations != null) {
331             for (Map.Entry<String, String> providerEntry : providerImplementations.entrySet()) {
332                 logInfo(
333                         "Changing the default '%s' provider implementation to '%s'.",
334                         providerEntry.getKey(), providerEntry.getValue());
335                 scmManager.setScmProviderImplementation(providerEntry.getKey(), providerEntry.getValue());
336             }
337         }
338 
339         ReleaseDescriptor releaseDescriptor = descriptorBuilder.build();
340         scmRepository = scmRepositoryConfigurator.getConfiguredRepository(releaseDescriptor, settings);
341 
342         scmProvider = scmRepositoryConfigurator.getRepositoryProvider(scmRepository);
343     }
344 
345     protected void checkoutExisting() throws MojoExecutionException {
346 
347         if (scmProvider instanceof AbstractSvnScmProvider) {
348             checkCreateRemoteSvnPath();
349         }
350 
351         logInfo(
352                 MessageUtils.buffer().strong("%s") + " the pub tree from "
353                         + MessageUtils.buffer().strong("%s") + " into %s",
354                 (tryUpdate ? "Updating" : "Checking out"),
355                 pubScmUrl,
356                 checkoutDirectory);
357 
358         if (checkoutDirectory.exists() && !tryUpdate) {
359 
360             try {
361                 FileUtils.deleteDirectory(checkoutDirectory);
362             } catch (IOException e) {
363                 logError(e.getMessage());
364 
365                 throw new MojoExecutionException("Unable to remove old checkout directory: " + e.getMessage(), e);
366             }
367         }
368 
369         boolean forceCheckout = false;
370 
371         if (!checkoutDirectory.exists()) {
372 
373             if (tryUpdate) {
374                 logInfo("TryUpdate is configured but no local copy currently available: forcing checkout.");
375             }
376             checkoutDirectory.mkdirs();
377             forceCheckout = true;
378         }
379 
380         try {
381             ScmFileSet fileSet = new ScmFileSet(checkoutDirectory, includes, excludes);
382 
383             ScmBranch branch = (scmBranch == null) ? null : new ScmBranch(scmBranch);
384 
385             ScmResult scmResult = null;
386             if (tryUpdate && !forceCheckout) {
387                 scmResult = scmProvider.update(scmRepository, fileSet, branch);
388             } else {
389                 int attempt = 0;
390                 while (scmResult == null) {
391                     try {
392                         scmResult = scmProvider.checkOut(scmRepository, fileSet, branch);
393                     } catch (ScmException e) {
394                         // give it max 2 times to retry
395                         if (attempt++ < 2) {
396                             try {
397                                 // wait 3 seconds
398                                 Thread.sleep(3 * 1000);
399                             } catch (InterruptedException ie) {
400                                 // noop
401                             }
402                         } else {
403                             throw e;
404                         }
405                     }
406                 }
407             }
408             checkScmResult(scmResult, "check out from SCM");
409         } catch (ScmException | IOException e) {
410             logError(e.getMessage());
411 
412             throw new MojoExecutionException("An error occurred during the checkout process: " + e.getMessage(), e);
413         }
414     }
415 
416     private void checkCreateRemoteSvnPath() throws MojoExecutionException {
417         getLog().debug("AbstractSvnScmProvider used, so we can check if remote url exists and eventually create it.");
418         AbstractSvnScmProvider svnScmProvider = (AbstractSvnScmProvider) scmProvider;
419 
420         try {
421             boolean remoteExists = svnScmProvider.remoteUrlExist(scmRepository.getProviderRepository(), null);
422 
423             if (remoteExists) {
424                 return;
425             }
426         } catch (ScmException e) {
427             throw new MojoExecutionException(e.getMessage(), e);
428         }
429 
430         String remoteUrl = ((SvnScmProviderRepository) scmRepository.getProviderRepository()).getUrl();
431 
432         if (!automaticRemotePathCreation) {
433             // olamy: return ?? that will fail during checkout IMHO :-)
434             logWarn("Remote svn url %s does not exist and automatic remote path creation disabled.", remoteUrl);
435             return;
436         }
437 
438         logInfo("Remote svn url %s does not exist: creating.", remoteUrl);
439 
440         File baseDir = null;
441         try {
442 
443             // create a temporary directory for svnexec
444             baseDir = Files.createTempDirectory("scm").toFile();
445 
446             // to prevent fileSet cannot be empty
447             ScmFileSet scmFileSet = new ScmFileSet(baseDir, new File(""));
448 
449             CommandParameters commandParameters = new CommandParameters();
450             commandParameters.setString(CommandParameter.SCM_MKDIR_CREATE_IN_LOCAL, Boolean.FALSE.toString());
451             commandParameters.setString(CommandParameter.MESSAGE, "Automatic svn path creation: " + remoteUrl);
452             svnScmProvider.mkdir(scmRepository.getProviderRepository(), scmFileSet, commandParameters);
453 
454             // new remote url so force checkout!
455             if (checkoutDirectory.exists()) {
456                 FileUtils.deleteDirectory(checkoutDirectory);
457             }
458         } catch (IOException | ScmException e) {
459             throw new MojoExecutionException(e.getMessage(), e);
460         } finally {
461             if (baseDir != null) {
462                 try {
463                     FileUtils.forceDeleteOnExit(baseDir);
464                 } catch (IOException e) {
465                     throw new MojoExecutionException(e.getMessage(), e);
466                 }
467             }
468         }
469     }
470 
471     public void execute() throws MojoExecutionException, MojoFailureException {
472         if (skipDeployment || skipDeployement) {
473             getLog().info("scmpublish.skipDeploy = true: Skipping site deployment");
474             return;
475         }
476 
477         // setup the scm plugin with help from release plugin utilities
478         try {
479             setupScm();
480         } catch (ScmRepositoryException | NoSuchScmProviderException e) {
481             throw new MojoExecutionException(e.getMessage(), e);
482         }
483 
484         boolean tmpCheckout = false;
485 
486         if (checkoutDirectory.getPath().contains("${project.")) {
487             try {
488                 tmpCheckout = true;
489                 checkoutDirectory = Files.createTempDirectory("maven-scm-publish" + ".checkout")
490                         .toFile();
491             } catch (IOException ioe) {
492                 throw new MojoExecutionException(ioe.getMessage(), ioe);
493             }
494         }
495 
496         try {
497             scmPublishExecute();
498         } finally {
499             if (tmpCheckout) {
500                 FileUtils.deleteQuietly(checkoutDirectory);
501             }
502         }
503     }
504 
505     /**
506      * Check-in content from scm checkout.
507      *
508      * @throws MojoExecutionException in case of issue
509      */
510     protected void checkinFiles() throws MojoExecutionException {
511         if (skipCheckin) {
512             return;
513         }
514 
515         ScmFileSet updatedFileSet = new ScmFileSet(checkoutDirectory);
516         try {
517             long start = System.currentTimeMillis();
518 
519             CheckInScmResult checkinResult = checkScmResult(
520                     scmProvider.checkIn(scmRepository, updatedFileSet, new ScmBranch(scmBranch), checkinComment),
521                     "check-in files to SCM");
522 
523             logInfo(
524                     "Checked in %d file(s) to revision %s in %s",
525                     checkinResult.getCheckedInFiles().size(),
526                     checkinResult.getScmRevision(),
527                     DurationFormatUtils.formatPeriod(start, System.currentTimeMillis(), "H' h 'm' m 's' s'"));
528         } catch (ScmException e) {
529             throw new MojoExecutionException("Failed to perform SCM checkin", e);
530         }
531     }
532 
533     protected void deleteFiles(Collection<File> deleted) throws MojoExecutionException {
534         if (skipDeletedFiles) {
535             logInfo("Deleting files is skipped.");
536             return;
537         }
538         List<File> deletedList = new ArrayList<>();
539         for (File f : deleted) {
540             deletedList.add(relativize(checkoutDirectory, f));
541         }
542         ScmFileSet deletedFileSet = new ScmFileSet(checkoutDirectory, deletedList);
543         try {
544             getLog().info("Deleting files: " + deletedList);
545 
546             checkScmResult(
547                     scmProvider.remove(scmRepository, deletedFileSet, "Deleting obsolete site files."),
548                     "delete files from SCM");
549         } catch (ScmException e) {
550             throw new MojoExecutionException("Failed to delete removed files to SCM", e);
551         }
552     }
553 
554     /**
555      * Add files to scm.
556      *
557      * @param added files to be added
558      * @throws MojoFailureException in case of issue
559      * @throws MojoExecutionException in case of issue
560      */
561     protected void addFiles(Collection<File> added) throws MojoFailureException, MojoExecutionException {
562         List<File> addedList = new ArrayList<>();
563         Set<File> createdDirs = new HashSet<>();
564         Set<File> dirsToAdd = new TreeSet<>();
565 
566         createdDirs.add(relativize(checkoutDirectory, checkoutDirectory));
567 
568         for (File f : added) {
569             for (File dir = f.getParentFile(); !dir.equals(checkoutDirectory); dir = dir.getParentFile()) {
570                 File relativized = relativize(checkoutDirectory, dir);
571                 //  we do the best we can with the directories
572                 if (createdDirs.add(relativized)) {
573                     dirsToAdd.add(relativized);
574                 } else {
575                     break;
576                 }
577             }
578             addedList.add(relativize(checkoutDirectory, f));
579         }
580 
581         if (addUniqueDirectory) { // add one directory at a time
582             for (File relativized : dirsToAdd) {
583                 try {
584                     ScmFileSet fileSet = new ScmFileSet(checkoutDirectory, relativized);
585                     getLog().info("scm add directory: " + relativized);
586                     AddScmResult addDirResult = scmProvider.add(scmRepository, fileSet, "Adding directory");
587                     if (!addDirResult.isSuccess()) {
588                         getLog().warn(" Error adding directory " + relativized + ": "
589                                 + addDirResult.getCommandOutput());
590                     }
591                 } catch (ScmException e) {
592                     //
593                 }
594             }
595         } else { // add all directories in one command
596             try {
597                 List<File> dirs = new ArrayList<>(dirsToAdd);
598                 ScmFileSet fileSet = new ScmFileSet(checkoutDirectory, dirs);
599                 getLog().info("scm add directories: " + dirs);
600                 AddScmResult addDirResult = scmProvider.add(scmRepository, fileSet, "Adding directories");
601                 if (!addDirResult.isSuccess()) {
602                     getLog().warn(" Error adding directories " + dirs + ": " + addDirResult.getCommandOutput());
603                 }
604             } catch (ScmException e) {
605                 //
606             }
607         }
608 
609         // remove directories already added !
610         addedList.removeAll(dirsToAdd);
611 
612         ScmFileSet addedFileSet = new ScmFileSet(checkoutDirectory, addedList);
613         getLog().info("scm add files: " + addedList);
614         try {
615             CommandParameters commandParameters = new CommandParameters();
616             commandParameters.setString(CommandParameter.MESSAGE, "Adding new site files.");
617             commandParameters.setString(CommandParameter.FORCE_ADD, Boolean.TRUE.toString());
618             checkScmResult(scmProvider.add(scmRepository, addedFileSet, commandParameters), "add new files to SCM");
619         } catch (ScmException e) {
620             throw new MojoExecutionException("Failed to add new files to SCM", e);
621         }
622     }
623 
624     private <T extends ScmResult> T checkScmResult(T result, String failure) throws MojoExecutionException {
625         if (!result.isSuccess()) {
626             String msg = "Failed to " + failure + ": " + result.getProviderMessage() + " " + result.getCommandOutput();
627             logError(msg);
628             throw new MojoExecutionException(msg);
629         }
630         return result;
631     }
632 
633     public boolean isDryRun() {
634         return dryRun;
635     }
636 
637     public abstract void scmPublishExecute() throws MojoExecutionException, MojoFailureException;
638 
639     public void setPubScmUrl(String pubScmUrl) {
640         // Fix required for Windows, which fit other OS as well
641         if (pubScmUrl.startsWith("scm:svn:")) {
642             pubScmUrl = pubScmUrl.replaceFirst("file:/[/]*", "file:///");
643         }
644 
645         this.pubScmUrl = pubScmUrl;
646     }
647 }