1 package org.apache.maven.index.cli;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.BufferedInputStream;
23 import java.io.File;
24 import java.io.FileInputStream;
25 import java.io.IOException;
26 import java.lang.reflect.Proxy;
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.List;
30
31 import org.apache.commons.cli.CommandLine;
32 import org.apache.commons.cli.HelpFormatter;
33 import org.apache.commons.cli.OptionBuilder;
34 import org.apache.commons.cli.Options;
35 import org.apache.commons.cli.ParseException;
36 import org.apache.lucene.store.FSDirectory;
37 import org.apache.maven.index.ArtifactContext;
38 import org.apache.maven.index.ArtifactInfo;
39 import org.apache.maven.index.ArtifactScanningListener;
40 import org.apache.maven.index.NexusIndexer;
41 import org.apache.maven.index.ScanningResult;
42 import org.apache.maven.index.context.IndexCreator;
43 import org.apache.maven.index.context.IndexingContext;
44 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
45 import org.apache.maven.index.packer.IndexPacker;
46 import org.apache.maven.index.packer.IndexPackingRequest;
47 import org.apache.maven.index.packer.IndexPackingRequest.IndexFormat;
48 import org.apache.maven.index.updater.DefaultIndexUpdater;
49 import org.codehaus.plexus.PlexusContainer;
50 import org.codehaus.plexus.classworlds.ClassWorld;
51 import org.codehaus.plexus.component.repository.exception.ComponentLookupException;
52 import org.codehaus.plexus.logging.Logger;
53 import org.codehaus.plexus.logging.LoggerManager;
54 import org.codehaus.plexus.tools.cli.AbstractCli;
55 import org.codehaus.plexus.util.IOUtil;
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77 public class NexusIndexerCli
78 extends AbstractCli
79 {
80
81
82 public static final char REPO = 'r';
83
84 public static final char INDEX = 'i';
85
86 public static final char NAME = 'n';
87
88 public static final char TYPE = 't';
89
90 public static final char TARGET_DIR = 'd';
91
92 public static final char CREATE_INCREMENTAL_CHUNKS = 'c';
93
94 public static final char CREATE_FILE_CHECKSUMS = 's';
95
96 public static final char INCREMENTAL_CHUNK_KEEP_COUNT = 'k';
97
98 public static final char LEGACY = 'l';
99
100 public static final char UNPACK = 'u';
101
102 private static final long MB = 1024 * 1024;
103
104 private Options options;
105
106 private int status = 0;
107
108 public static void main( String[] args )
109 throws Exception
110 {
111 NexusIndexerCli cli = new NexusIndexerCli();
112
113 cli.execute( args );
114
115 System.exit( cli.status );
116 }
117
118 @Override
119 public int execute( String[] arg0, ClassWorld arg1 )
120 {
121 int value = super.execute( arg0, arg1 );
122
123 if ( status == 0 )
124 {
125 status = value;
126 }
127
128 return status;
129 }
130
131 @Override
132 public int execute( String[] args )
133 {
134 int value = super.execute( args );
135
136 if ( status == 0 )
137 {
138 status = value;
139 }
140
141 return status;
142 }
143
144 @Override
145 protected void showError( String message, Exception e, boolean show )
146 {
147 status = 1;
148 super.showError( message, e, show );
149 }
150
151 @Override
152 protected int showFatalError( String message, Exception e, boolean show )
153 {
154 status = 1;
155 return super.showFatalError( message, e, show );
156 }
157
158 @Override
159 public CommandLine parse( String[] args )
160 throws ParseException
161 {
162 try
163 {
164 return super.parse( args );
165 }
166 catch ( ParseException e )
167 {
168 status = 1;
169 throw e;
170 }
171 }
172
173 @Override
174 public String getPomPropertiesPath()
175 {
176 return "META-INF/maven/org.sonatype.nexus/nexus-indexer/pom.properties";
177 }
178
179 @Override
180 @SuppressWarnings( "static-access" )
181 public Options buildCliOptions( Options options )
182 {
183 this.options = options;
184
185 options.addOption( OptionBuilder.withLongOpt( "index" ).hasArg()
186 .withDescription( "Path to the index folder." ).create( INDEX ) );
187
188 options.addOption( OptionBuilder.withLongOpt( "destination" ).hasArg()
189 .withDescription( "Target folder." ).create( TARGET_DIR ) );
190
191 options.addOption( OptionBuilder.withLongOpt( "repository" ).hasArg()
192 .withDescription( "Path to the Maven repository." ).create( REPO ) );
193
194 options.addOption( OptionBuilder.withLongOpt( "name" ).hasArg()
195 .withDescription( "Repository name." ).create( NAME ) );
196
197 options.addOption( OptionBuilder.withLongOpt( "chunks" )
198 .withDescription( "Create incremental chunks." ).create( CREATE_INCREMENTAL_CHUNKS ) );
199
200 options.addOption( OptionBuilder.withLongOpt( "keep" ).hasArg().withDescription(
201 "Number of incremental chunks to keep." ).create( INCREMENTAL_CHUNK_KEEP_COUNT ) );
202
203 options.addOption( OptionBuilder.withLongOpt( "checksums" )
204 .withDescription( "Create checksums for all files (sha1, md5)." ).create( CREATE_FILE_CHECKSUMS ) );
205
206 options.addOption( OptionBuilder.withLongOpt( "type" ).hasArg()
207 .withDescription( "Indexer type (default, min, full or coma separated list of custom types)." ).create( TYPE ) );
208
209 options.addOption( OptionBuilder.withLongOpt( "legacy" )
210 .withDescription( "Build legacy .zip index file" ).create( LEGACY ) );
211
212 options.addOption( OptionBuilder.withLongOpt( "unpack" )
213 .withDescription( "Unpack an index file" ).create( UNPACK ) );
214
215 return options;
216 }
217
218 @Override
219 public void displayHelp()
220 {
221 System.out.println();
222
223 HelpFormatter formatter = new HelpFormatter();
224
225 formatter.printHelp( "nexus-indexer [options]", "\nOptions:", options, "\n" );
226 }
227
228 public void displayHelp( String message )
229 {
230 System.out.println();
231
232 System.out.println( message );
233
234 System.out.println();
235
236 displayHelp();
237 }
238
239 @Override
240 public void invokePlexusComponent( final CommandLine cli, PlexusContainer plexus )
241 throws Exception
242 {
243 if ( cli.hasOption( QUIET ) )
244 {
245 setLogLevel( plexus, Logger.LEVEL_DISABLED );
246 }
247 else if ( cli.hasOption( DEBUG ) )
248 {
249 setLogLevel( plexus, Logger.LEVEL_DEBUG );
250 }
251 else if ( cli.hasOption( ERRORS ) )
252 {
253 setLogLevel( plexus, Logger.LEVEL_ERROR );
254 }
255
256 if ( cli.hasOption( UNPACK ) )
257 {
258 unpack( cli, plexus );
259 }
260 else if ( cli.hasOption( INDEX ) && cli.hasOption( REPO ) )
261 {
262 index( cli, plexus );
263 }
264 else
265 {
266 status = 1;
267
268 displayHelp( "Use either unpack (\"" + UNPACK + "\") or index (\"" + INDEX + "\" and \"" + REPO
269 + "\") options, but none has been found!" );
270 }
271 }
272
273 private void setLogLevel( PlexusContainer plexus, int logLevel )
274 throws ComponentLookupException
275 {
276 plexus.lookup( LoggerManager.class ).setThresholds( logLevel );
277 }
278
279 private void index( final CommandLine cli, PlexusContainer plexus )
280 throws ComponentLookupException, IOException, UnsupportedExistingLuceneIndexException
281 {
282 String indexDirectoryName = cli.getOptionValue( INDEX );
283
284 File indexFolder = new File( indexDirectoryName );
285
286 String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
287
288 File outputFolder = new File( outputDirectoryName );
289
290 File repositoryFolder = new File( cli.getOptionValue( REPO ) );
291
292 String repositoryName = cli.getOptionValue( NAME, indexFolder.getName() );
293
294 List<IndexCreator> indexers = getIndexers( cli, plexus );
295
296 boolean createChecksums = cli.hasOption( CREATE_FILE_CHECKSUMS );
297
298 boolean createIncrementalChunks = cli.hasOption( CREATE_INCREMENTAL_CHUNKS );
299
300 boolean createLegacyIndex = cli.hasOption( LEGACY );
301
302 boolean debug = cli.hasOption( DEBUG );
303
304 boolean quiet = cli.hasOption( QUIET );
305
306 Integer chunkCount =
307 cli.hasOption( INCREMENTAL_CHUNK_KEEP_COUNT ) ? Integer.parseInt( cli.getOptionValue( INCREMENTAL_CHUNK_KEEP_COUNT ) )
308 : null;
309
310 if ( !quiet )
311 {
312 System.err.printf( "Repository Folder: %s\n", repositoryFolder.getAbsolutePath() );
313 System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() );
314 System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() );
315 System.err.printf( "Repository name: %s\n", repositoryName );
316 System.err.printf( "Indexers: %s\n", indexers.toString() );
317
318 if ( createChecksums )
319 {
320 System.err.printf( "Will create checksum files for all published files (sha1, md5).\n" );
321 }
322 else
323 {
324 System.err.printf( "Will not create checksum files.\n" );
325 }
326
327 if ( createIncrementalChunks )
328 {
329 System.err.printf( "Will create incremental chunks for changes, along with baseline file.\n" );
330 }
331 else
332 {
333 System.err.printf( "Will create baseline file.\n" );
334 }
335
336 if ( createLegacyIndex )
337 {
338 System.err.printf( "Will also create legacy .zip index file.\n" );
339 }
340 }
341
342 NexusIndexer indexer = plexus.lookup( NexusIndexer.class );
343
344 long tstart = System.currentTimeMillis();
345
346 IndexingContext context = indexer.addIndexingContext(
347 repositoryName,
348 repositoryName,
349 repositoryFolder,
350 indexFolder,
351 null,
352 null,
353 indexers );
354
355 try
356 {
357 IndexPacker packer = plexus.lookup( IndexPacker.class );
358
359 ArtifactScanningListener listener = new IndexerListener( context, debug, quiet );
360
361 indexer.scan( context, listener, true );
362
363 IndexPackingRequest request = new IndexPackingRequest( context, outputFolder );
364
365 request.setCreateChecksumFiles( createChecksums );
366
367 request.setCreateIncrementalChunks( createIncrementalChunks );
368
369 if ( createLegacyIndex )
370 {
371 request.setFormats( Arrays.asList( IndexFormat.FORMAT_LEGACY, IndexFormat.FORMAT_V1 ) );
372 }
373 else
374 {
375 request.setFormats( Arrays.asList( IndexFormat.FORMAT_V1 ) );
376 }
377
378 if ( chunkCount != null )
379 {
380 request.setMaxIndexChunks( chunkCount.intValue() );
381 }
382
383 packIndex( packer, request, debug, quiet );
384
385 if ( !quiet )
386 {
387 printStats( tstart );
388 }
389 }
390 finally
391 {
392 indexer.removeIndexingContext( context, false );
393 }
394 }
395
396 private void unpack( CommandLine cli, PlexusContainer plexus )
397 throws ComponentLookupException, IOException
398 {
399 final String indexDirectoryName = cli.getOptionValue( INDEX, "." );
400 final File indexFolder = new File( indexDirectoryName ).getCanonicalFile();
401 final File indexArchive = new File( indexFolder, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
402
403 final String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." );
404 final File outputFolder = new File( outputDirectoryName ).getCanonicalFile();
405
406 final boolean quiet = cli.hasOption( QUIET );
407 if ( !quiet )
408 {
409 System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() );
410 System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() );
411 }
412
413 long tstart = System.currentTimeMillis();
414
415 final FSDirectory directory = FSDirectory.open( outputFolder );
416
417 final List<IndexCreator> indexers = getIndexers( cli, plexus );
418
419 BufferedInputStream is = null;
420 try
421 {
422 is = new BufferedInputStream( new FileInputStream( indexArchive ) );
423 DefaultIndexUpdater.unpackIndexData( is, directory, (IndexingContext) Proxy.newProxyInstance(
424 getClass().getClassLoader(), new Class[] { IndexingContext.class }, new PartialImplementation()
425 {
426 public List<IndexCreator> getIndexCreators()
427 {
428 return indexers;
429 }
430 } )
431
432 );
433 }
434 finally
435 {
436 IOUtil.close( is );
437 if ( directory != null )
438 {
439 directory.close();
440 }
441 }
442
443 if ( !quiet )
444 {
445 printStats( tstart );
446 }
447 }
448
449 private List<IndexCreator> getIndexers( final CommandLine cli, PlexusContainer plexus )
450 throws ComponentLookupException
451 {
452 String type = "default";
453
454 if ( cli.hasOption( TYPE ) )
455 {
456 type = cli.getOptionValue( TYPE );
457 }
458
459 List<IndexCreator> indexers = new ArrayList<IndexCreator>();
460
461 if ( "default".equals( type ) )
462 {
463 indexers.add( plexus.lookup( IndexCreator.class, "min" ) );
464 indexers.add( plexus.lookup( IndexCreator.class, "jarContent" ) );
465 }
466 else if ( "full".equals( type ) )
467 {
468 for ( Object component : plexus.lookupList( IndexCreator.class ) )
469 {
470 indexers.add( (IndexCreator) component );
471 }
472 }
473 else
474 {
475 for ( String hint : type.split( "," ) )
476 {
477 indexers.add( plexus.lookup( IndexCreator.class, hint ) );
478 }
479 }
480 return indexers;
481 }
482
483 private void packIndex( IndexPacker packer, IndexPackingRequest request, boolean debug, boolean quiet )
484 {
485 try
486 {
487 packer.packIndex( request );
488 }
489 catch ( IOException e )
490 {
491 if ( !quiet )
492 {
493 System.err.printf( "Cannot zip index; \n", e.getMessage() );
494
495 if ( debug )
496 {
497 e.printStackTrace();
498 }
499 }
500 }
501 }
502
503 private void printStats( final long startTimeInMillis )
504 {
505 long t = System.currentTimeMillis() - startTimeInMillis;
506
507 long s = t / 1000L;
508 if ( t > 60 * 1000 )
509 {
510 long m = t / 1000L / 60L;
511
512 System.err.printf( "Total time: %d min %d sec\n", m, s - ( m * 60 ) );
513 }
514 else
515 {
516 System.err.printf( "Total time: %d sec\n", s );
517 }
518
519 Runtime r = Runtime.getRuntime();
520
521 System.err.printf( "Final memory: %dM/%dM\n",
522 ( r.totalMemory() - r.freeMemory() ) / MB, r.totalMemory() / MB );
523 }
524
525
526
527
528 private static final class IndexerListener
529 implements ArtifactScanningListener
530 {
531 private final IndexingContext context;
532
533 private final boolean debug;
534
535 private boolean quiet;
536
537 private long ts = System.currentTimeMillis();
538
539 private int count;
540
541 IndexerListener( IndexingContext context, boolean debug, boolean quiet )
542 {
543 this.context = context;
544 this.debug = debug;
545 this.quiet = quiet;
546 }
547
548 public void scanningStarted( IndexingContext context )
549 {
550 if ( !quiet )
551 {
552 System.err.println( "Scanning started" );
553 }
554 }
555
556 public void artifactDiscovered( ArtifactContext ac )
557 {
558 count++;
559
560 long t = System.currentTimeMillis();
561
562 ArtifactInfo ai = ac.getArtifactInfo();
563
564 if ( !quiet && debug && "maven-plugin".equals( ai.packaging ) )
565 {
566 System.err.printf( "Plugin: %s:%s:%s - %s %s\n",
567 ai.groupId, ai.artifactId, ai.version, ai.prefix, "" + ai.goals );
568 }
569
570 if ( !quiet && ( debug || ( t - ts ) > 2000L ) )
571 {
572 System.err.printf( " %6d %s\n", count, formatFile( ac.getPom() ) );
573 ts = t;
574 }
575 }
576
577 public void artifactError( ArtifactContext ac, Exception e )
578 {
579 if ( !quiet )
580 {
581 System.err.printf( "! %6d %s - %s\n", count, formatFile( ac.getPom() ), e.getMessage() );
582
583 System.err.printf( " %s\n", formatFile( ac.getArtifact() ) );
584
585 if ( debug )
586 {
587 e.printStackTrace();
588 }
589 }
590
591 ts = System.currentTimeMillis();
592 }
593
594 private String formatFile( File file )
595 {
596 return file.getAbsolutePath().substring( context.getRepository().getAbsolutePath().length() + 1 );
597 }
598
599 public void scanningFinished( IndexingContext context, ScanningResult result )
600 {
601 if ( !quiet )
602 {
603 if ( result.hasExceptions() )
604 {
605 System.err.printf( "Scanning errors: %s\n", result.getExceptions().size() );
606 }
607
608 System.err.printf( "Artifacts added: %s\n", result.getTotalFiles() );
609 System.err.printf( "Artifacts deleted: %s\n", result.getDeletedFiles() );
610 }
611 }
612 }
613
614 }