1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.index.cli;
20
21 import java.io.BufferedInputStream;
22 import java.io.File;
23 import java.io.FileInputStream;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.lang.reflect.Proxy;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.Properties;
30 import java.util.concurrent.TimeUnit;
31
32 import com.google.inject.Guice;
33 import com.google.inject.Module;
34 import org.apache.commons.cli.CommandLine;
35 import org.apache.commons.cli.DefaultParser;
36 import org.apache.commons.cli.HelpFormatter;
37 import org.apache.commons.cli.Option;
38 import org.apache.commons.cli.Options;
39 import org.apache.commons.cli.ParseException;
40 import org.apache.lucene.search.IndexSearcher;
41 import org.apache.lucene.store.FSDirectory;
42 import org.apache.maven.index.ArtifactContext;
43 import org.apache.maven.index.ArtifactInfo;
44 import org.apache.maven.index.ArtifactScanningListener;
45 import org.apache.maven.index.ScanningResult;
46 import org.apache.maven.index.context.IndexCreator;
47 import org.apache.maven.index.context.IndexingContext;
48 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
49 import org.apache.maven.index.packer.IndexPacker;
50 import org.apache.maven.index.packer.IndexPackingRequest;
51 import org.apache.maven.index.packer.IndexPackingRequest.IndexFormat;
52 import org.apache.maven.index.updater.DefaultIndexUpdater;
53 import org.eclipse.sisu.launch.Main;
54 import org.eclipse.sisu.space.BeanScanning;
55
56 import static java.util.Objects.requireNonNull;
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79 public class NexusIndexerCli {
80
81
82
83 public static final String QUIET = "q";
84
85 public static final String DEBUG = "X";
86
87 public static final String HELP = "h";
88
89 public static final String VERSION = "v";
90
91
92
93 public static final String REPO = "r";
94
95 public static final String INDEX = "i";
96
97 public static final String NAME = "n";
98
99 public static final String TYPE = "t";
100
101 public static final String TARGET_DIR = "d";
102
103 public static final String CREATE_INCREMENTAL_CHUNKS = "c";
104
105 public static final String CREATE_FILE_CHECKSUMS = "s";
106
107 public static final String INCREMENTAL_CHUNK_KEEP_COUNT = "k";
108
109 public static final String UNPACK = "u";
110
111 private static final long MB = 1024 * 1024;
112
113 private Options options;
114
115 public static void main(String[] args) {
116 System.exit(new NexusIndexerCli().execute(args));
117 }
118
119
120
121
122 int execute(String[] args) {
123 CommandLine cli;
124
125 try {
126 cli = new DefaultParser().parse(buildCliOptions(), cleanArgs(args));
127 } catch (ParseException e) {
128 System.err.println("Unable to parse command line options: " + e.getMessage());
129
130 displayHelp();
131
132 return 1;
133 }
134
135 boolean debug = cli.hasOption(DEBUG);
136
137 if (cli.hasOption(HELP)) {
138 displayHelp();
139
140 return 0;
141 }
142
143 if (cli.hasOption(VERSION)) {
144 showVersion();
145
146 return 0;
147 } else if (debug) {
148 showVersion();
149 }
150
151 final Module app = Main.wire(BeanScanning.INDEX);
152
153 Components components = Guice.createInjector(app).getInstance(Components.class);
154
155 if (cli.hasOption(UNPACK)) {
156 try {
157 return unpack(cli, components);
158 } catch (Exception e) {
159 e.printStackTrace(System.err);
160 return 1;
161 }
162 } else if (cli.hasOption(INDEX) && cli.hasOption(REPO)) {
163 try {
164 return index(cli, components);
165 } catch (Exception e) {
166 e.printStackTrace(System.err);
167 return 1;
168 }
169 } else {
170 System.out.println();
171 System.out.println("Use either unpack (\"" + UNPACK + "\") or index (\"" + INDEX + "\" and \"" + REPO
172 + "\") options, but none has been found!");
173 System.out.println();
174 displayHelp();
175 return 1;
176 }
177 }
178
179
180
181
182 Options buildCliOptions() {
183 this.options = new Options();
184
185 options.addOption(Option.builder(QUIET)
186 .longOpt("quiet")
187 .desc("Quiet output - only show errors")
188 .build());
189
190 options.addOption(Option.builder(DEBUG)
191 .longOpt("debug")
192 .desc("Produce execution debug output")
193 .build());
194
195 options.addOption(Option.builder(VERSION)
196 .longOpt("version")
197 .desc("Display version information")
198 .build());
199
200 options.addOption(Option.builder(HELP)
201 .longOpt("help")
202 .desc("Display help information")
203 .build());
204
205 options.addOption(Option.builder(INDEX)
206 .longOpt("index")
207 .argName("path")
208 .hasArg()
209 .desc("Path to the index folder")
210 .build());
211
212 options.addOption(Option.builder(TARGET_DIR)
213 .longOpt("destination")
214 .argName("path")
215 .hasArg()
216 .desc("Target folder")
217 .build());
218
219 options.addOption(Option.builder(REPO)
220 .longOpt("repository")
221 .argName("path")
222 .hasArg()
223 .desc("Path to the Maven repository")
224 .build());
225
226 options.addOption(Option.builder(NAME)
227 .longOpt("name")
228 .argName("string")
229 .hasArg()
230 .desc("Repository name")
231 .build());
232
233 options.addOption(Option.builder(CREATE_INCREMENTAL_CHUNKS)
234 .longOpt("chunks")
235 .desc("Create incremental chunks")
236 .build());
237
238 options.addOption(Option.builder(INCREMENTAL_CHUNK_KEEP_COUNT)
239 .longOpt("keep")
240 .argName("num")
241 .hasArg()
242 .desc("Number of incremental chunks to keep")
243 .build());
244
245 options.addOption(Option.builder(CREATE_FILE_CHECKSUMS)
246 .longOpt("checksums")
247 .desc("Create checksums for all files (sha1, md5)")
248 .build());
249
250 options.addOption(Option.builder(TYPE)
251 .longOpt("type")
252 .argName("type")
253 .hasArg()
254 .desc("Indexer type (default, min, full or comma separated list of custom types)")
255 .build());
256
257 options.addOption(Option.builder(UNPACK)
258 .longOpt("unpack")
259 .desc("Unpack an index file")
260 .build());
261
262 return options;
263 }
264
265 private String[] cleanArgs(String[] args) {
266 List<String> cleaned = new ArrayList<>();
267
268 StringBuilder currentArg = null;
269
270 for (String arg : args) {
271 boolean addedToBuffer = false;
272
273 if (arg.startsWith("\"")) {
274
275
276 if (currentArg != null) {
277 cleaned.add(currentArg.toString());
278 }
279
280
281 currentArg = new StringBuilder(arg.substring(1));
282
283 addedToBuffer = true;
284 }
285
286
287 if (arg.endsWith("\"")) {
288 String cleanArgPart = arg.substring(0, arg.length() - 1);
289
290
291 if (currentArg != null) {
292
293 if (addedToBuffer) {
294 currentArg.setLength(currentArg.length() - 1);
295 }
296
297 else {
298
299 currentArg.append(' ').append(cleanArgPart);
300 }
301
302
303 cleaned.add(currentArg.toString());
304 } else {
305
306 cleaned.add(cleanArgPart);
307 }
308
309
310 currentArg = null;
311
312 continue;
313 }
314
315
316
317
318
319 if (!addedToBuffer) {
320
321 if (currentArg != null) {
322 currentArg.append(' ').append(arg);
323 }
324
325 else {
326 cleaned.add(arg);
327 }
328 }
329 }
330
331
332 if (currentArg != null) {
333 cleaned.add(currentArg.toString());
334 }
335
336 int cleanedSz = cleaned.size();
337 String[] cleanArgs;
338
339 if (cleanedSz == 0) {
340
341 cleanArgs = args;
342 } else {
343 cleanArgs = cleaned.toArray(new String[cleanedSz]);
344 }
345
346 return cleanArgs;
347 }
348
349 private void displayHelp() {
350 System.out.println();
351
352 HelpFormatter formatter = new HelpFormatter();
353
354 formatter.printHelp("nexus-indexer [options]", "\nOptions:", options, "\n");
355 }
356
357 private void showVersion() {
358 InputStream is;
359
360 try {
361 Properties properties = new Properties();
362
363 is = getClass()
364 .getClassLoader()
365 .getResourceAsStream("META-INF/maven/org.apache.maven.indexer/indexer-core/pom.properties");
366
367 if (is == null) {
368 System.err.println("Unable determine version from JAR file.");
369
370 return;
371 }
372
373 properties.load(is);
374
375 if (properties.getProperty("builtOn") != null) {
376 System.out.println("Version: " + properties.getProperty("version", "unknown") + " built on "
377 + properties.getProperty("builtOn"));
378 } else {
379 System.out.println("Version: " + properties.getProperty("version", "unknown"));
380 }
381 } catch (IOException e) {
382 System.err.println("Unable determine version from JAR file: " + e.getMessage());
383 }
384 }
385
386 private int index(final CommandLine cli, Components components)
387 throws IOException, UnsupportedExistingLuceneIndexException {
388 String indexDirectoryName = cli.getOptionValue(INDEX);
389
390 File indexFolder = new File(indexDirectoryName);
391
392 String outputDirectoryName = cli.getOptionValue(TARGET_DIR, ".");
393
394 File outputFolder = new File(outputDirectoryName);
395
396 File repositoryFolder = new File(cli.getOptionValue(REPO));
397
398 String repositoryName = cli.getOptionValue(NAME, indexFolder.getName());
399
400 List<IndexCreator> indexers = getIndexers(cli, components);
401
402 boolean createChecksums = cli.hasOption(CREATE_FILE_CHECKSUMS);
403
404 boolean createIncrementalChunks = cli.hasOption(CREATE_INCREMENTAL_CHUNKS);
405
406 boolean debug = cli.hasOption(DEBUG);
407
408 boolean quiet = cli.hasOption(QUIET);
409
410 Integer chunkCount = cli.hasOption(INCREMENTAL_CHUNK_KEEP_COUNT)
411 ? Integer.parseInt(cli.getOptionValue(INCREMENTAL_CHUNK_KEEP_COUNT))
412 : null;
413
414 if (!quiet) {
415 System.err.printf("Repository Folder: %s\n", repositoryFolder.getAbsolutePath());
416 System.err.printf("Index Folder: %s\n", indexFolder.getAbsolutePath());
417 System.err.printf("Output Folder: %s\n", outputFolder.getAbsolutePath());
418 System.err.printf("Repository name: %s\n", repositoryName);
419 System.err.printf("Indexers: %s\n", indexers);
420
421 if (createChecksums) {
422 System.err.print("Will create checksum files for all published files (sha1, md5).\n");
423 } else {
424 System.err.print("Will not create checksum files.\n");
425 }
426
427 if (createIncrementalChunks) {
428 System.err.print("Will create incremental chunks for changes, along with baseline file.\n");
429 } else {
430 System.err.print("Will create baseline file.\n");
431 }
432 }
433
434 long tstart = System.currentTimeMillis();
435
436 IndexingContext context = components.indexer.addIndexingContext(
437 repositoryName,
438 repositoryName,
439 repositoryFolder,
440 indexFolder,
441 null,
442 null,
443 indexers);
444
445 try {
446 ArtifactScanningListener listener = new IndexerListener(context, debug, quiet);
447
448 components.indexer.scan(context, listener, true);
449
450 IndexSearcher indexSearcher = context.acquireIndexSearcher();
451
452 try {
453 IndexPackingRequest request =
454 new IndexPackingRequest(context, indexSearcher.getIndexReader(), outputFolder);
455
456 request.setCreateChecksumFiles(createChecksums);
457
458 request.setCreateIncrementalChunks(createIncrementalChunks);
459
460 request.setFormats(List.of(IndexFormat.FORMAT_V1));
461
462 if (chunkCount != null) {
463 request.setMaxIndexChunks(chunkCount);
464 }
465
466 packIndex(components.packer, request, debug, quiet);
467 } finally {
468 context.releaseIndexSearcher(indexSearcher);
469 }
470
471 if (!quiet) {
472 printStats(tstart);
473 }
474 } finally {
475 components.indexer.removeIndexingContext(context, false);
476 }
477 return 0;
478 }
479
480 private int unpack(CommandLine cli, Components components) throws IOException {
481 final String indexDirectoryName = cli.getOptionValue(INDEX, ".");
482 final File indexFolder = new File(indexDirectoryName).getCanonicalFile();
483 final File indexArchive = new File(indexFolder, IndexingContext.INDEX_FILE_PREFIX + ".gz");
484
485 final String outputDirectoryName = cli.getOptionValue(TARGET_DIR, ".");
486 final File outputFolder = new File(outputDirectoryName).getCanonicalFile();
487
488 final boolean quiet = cli.hasOption(QUIET);
489 if (!quiet) {
490 System.err.printf("Index Folder: %s\n", indexFolder.getAbsolutePath());
491 System.err.printf("Output Folder: %s\n", outputFolder.getAbsolutePath());
492 }
493
494 long tstart = System.currentTimeMillis();
495
496 final List<IndexCreator> indexers = getIndexers(cli, components);
497
498 try (BufferedInputStream is = new BufferedInputStream(new FileInputStream(indexArchive));
499 FSDirectory directory = FSDirectory.open(outputFolder.toPath())) {
500 DefaultIndexUpdater.unpackIndexData(is, 4, directory, (IndexingContext) Proxy.newProxyInstance(
501 getClass().getClassLoader(), new Class[] {IndexingContext.class}, new PartialImplementation() {
502 public List<IndexCreator> getIndexCreators() {
503 return indexers;
504 }
505 }));
506 }
507
508 if (!quiet) {
509 printStats(tstart);
510 }
511 return 0;
512 }
513
514 private List<IndexCreator> getIndexers(final CommandLine cli, Components components) {
515 String type = "default";
516
517 if (cli.hasOption(TYPE)) {
518 type = cli.getOptionValue(TYPE);
519 }
520
521 List<IndexCreator> indexers = new ArrayList<>();
522
523 if ("default".equals(type)) {
524 indexers.add(requireNonNull(components.allIndexCreators.get("min")));
525 indexers.add(requireNonNull(components.allIndexCreators.get("jarContent")));
526 } else if ("full".equals(type)) {
527 indexers.addAll(components.allIndexCreators.values());
528 } else {
529 for (String name : type.split(",")) {
530 indexers.add(requireNonNull(components.allIndexCreators.get(name)));
531 }
532 }
533 return indexers;
534 }
535
536 private void packIndex(IndexPacker packer, IndexPackingRequest request, boolean debug, boolean quiet) {
537 try {
538 packer.packIndex(request);
539 } catch (IOException e) {
540 if (!quiet) {
541 System.err.printf("Cannot zip index: %s\n", e.getMessage());
542
543 if (debug) {
544 e.printStackTrace();
545 }
546 }
547 }
548 }
549
550 private void printStats(final long startTimeInMillis) {
551 long t = System.currentTimeMillis() - startTimeInMillis;
552
553 long s = TimeUnit.MILLISECONDS.toSeconds(t);
554 if (t > TimeUnit.MINUTES.toMillis(1)) {
555 long m = TimeUnit.MILLISECONDS.toMinutes(t);
556
557 System.err.printf("Total time: %d min %d sec\n", m, s - (m * 60));
558 } else {
559 System.err.printf("Total time: %d sec\n", s);
560 }
561
562 Runtime r = Runtime.getRuntime();
563
564 System.err.printf(
565 "Final memory: %dM/%dM\n",
566 (r.totalMemory() - r.freeMemory()) / MB, r.totalMemory() / MB);
567 }
568
569
570
571
572 private static final class IndexerListener implements ArtifactScanningListener {
573 private final IndexingContext context;
574
575 private final boolean debug;
576
577 private final boolean quiet;
578
579 private long ts = System.currentTimeMillis();
580
581 private int count;
582
583 IndexerListener(IndexingContext context, boolean debug, boolean quiet) {
584 this.context = context;
585 this.debug = debug;
586 this.quiet = quiet;
587 }
588
589 @Override
590 public void scanningStarted(IndexingContext context) {
591 if (!quiet) {
592 System.err.println("Scanning started");
593 }
594 }
595
596 @Override
597 public void artifactDiscovered(ArtifactContext ac) {
598 count++;
599
600 long t = System.currentTimeMillis();
601
602 ArtifactInfo ai = ac.getArtifactInfo();
603
604 if (!quiet && debug && "maven-plugin".equals(ai.getPackaging())) {
605 System.err.printf(
606 "Plugin: %s:%s:%s - %s %s\n",
607 ai.getGroupId(), ai.getArtifactId(), ai.getVersion(), ai.getPrefix(), "" + ai.getGoals());
608 }
609
610 if (!quiet && (debug || (t - ts) > 2000L)) {
611 System.err.printf(" %6d %s\n", count, formatFile(ac.getPom()));
612 ts = t;
613 }
614 }
615
616 @Override
617 public void artifactError(ArtifactContext ac, Exception e) {
618 if (!quiet) {
619 System.err.printf("! %6d %s - %s\n", count, formatFile(ac.getPom()), e.getMessage());
620
621 System.err.printf(" %s\n", formatFile(ac.getArtifact()));
622
623 if (debug) {
624 e.printStackTrace();
625 }
626 }
627
628 ts = System.currentTimeMillis();
629 }
630
631 private String formatFile(File file) {
632 return file.getAbsolutePath()
633 .substring(context.getRepository().getAbsolutePath().length() + 1);
634 }
635
636 @Override
637 public void scanningFinished(IndexingContext context, ScanningResult result) {
638 if (!quiet) {
639 if (result.hasExceptions()) {
640 System.err.printf(
641 "Scanning errors: %s\n", result.getExceptions().size());
642 }
643
644 System.err.printf("Artifacts added: %s\n", result.getTotalFiles());
645 System.err.printf("Artifacts deleted: %s\n", result.getDeletedFiles());
646 }
647 }
648 }
649 }