1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.index.context;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.nio.channels.FileChannel;
24 import java.nio.channels.FileLock;
25 import java.nio.file.Files;
26 import java.nio.file.Path;
27 import java.nio.file.StandardOpenOption;
28 import java.util.Collection;
29 import java.util.Collections;
30 import java.util.Date;
31 import java.util.HashSet;
32 import java.util.LinkedHashSet;
33 import java.util.List;
34 import java.util.Set;
35 import java.util.concurrent.atomic.AtomicReference;
36
37 import org.apache.lucene.analysis.Analyzer;
38 import org.apache.lucene.document.Document;
39 import org.apache.lucene.document.Field;
40 import org.apache.lucene.document.StoredField;
41 import org.apache.lucene.index.CorruptIndexException;
42 import org.apache.lucene.index.DirectoryReader;
43 import org.apache.lucene.index.IndexReader;
44 import org.apache.lucene.index.IndexWriter;
45 import org.apache.lucene.index.IndexWriterConfig;
46 import org.apache.lucene.index.MultiBits;
47 import org.apache.lucene.index.Term;
48 import org.apache.lucene.search.IndexSearcher;
49 import org.apache.lucene.search.SearcherManager;
50 import org.apache.lucene.search.TermQuery;
51 import org.apache.lucene.search.TopScoreDocCollector;
52 import org.apache.lucene.store.Directory;
53 import org.apache.lucene.store.FSDirectory;
54 import org.apache.lucene.store.FSLockFactory;
55 import org.apache.lucene.store.Lock;
56 import org.apache.lucene.store.LockObtainFailedException;
57 import org.apache.lucene.util.Bits;
58 import org.apache.maven.index.ArtifactInfo;
59 import org.apache.maven.index.IndexerField;
60 import org.apache.maven.index.artifact.GavCalculator;
61 import org.apache.maven.index.artifact.M2GavCalculator;
62 import org.codehaus.plexus.util.StringUtils;
63
64
65
66
67
68
69
70 public class DefaultIndexingContext extends AbstractIndexingContext {
71
72
73
74 private static final String INDEX_DIRECTORY = ".index";
75
76 public static final String FLD_DESCRIPTOR = "DESCRIPTOR";
77
78 public static final String FLD_DESCRIPTOR_CONTENTS = "NexusIndex";
79
80 public static final String FLD_IDXINFO = "IDXINFO";
81
82 public static final String VERSION = "1.0";
83
84 private static final Term DESCRIPTOR_TERM = new Term(FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS);
85
86 private Directory indexDirectory;
87
88 private TrackingLockFactory lockFactory;
89
90 private File indexDirectoryFile;
91
92 private String id;
93
94 private boolean searchable;
95
96 private String repositoryId;
97
98 private File repository;
99
100 private String repositoryUrl;
101
102 private String indexUpdateUrl;
103
104 private NexusIndexWriter indexWriter;
105
106 private SearcherManager searcherManager;
107
108 private Date timestamp;
109
110 private List<? extends IndexCreator> indexCreators;
111
112
113
114
115
116
117 private GavCalculator gavCalculator;
118
119 private DefaultIndexingContext(
120 String id,
121 String repositoryId,
122 File repository,
123 String repositoryUrl,
124 String indexUpdateUrl,
125 List<? extends IndexCreator> indexCreators,
126 Directory indexDirectory,
127 TrackingLockFactory lockFactory,
128 boolean reclaimIndex,
129 File indexDirectoryFile)
130 throws ExistingLuceneIndexMismatchException, IOException {
131
132 this.id = id;
133
134 this.searchable = true;
135
136 this.repositoryId = repositoryId;
137
138 this.repository = repository;
139
140 this.repositoryUrl = repositoryUrl;
141
142 this.indexUpdateUrl = indexUpdateUrl;
143
144 this.indexWriter = null;
145
146 this.searcherManager = null;
147
148 this.indexCreators = indexCreators;
149
150 this.indexDirectory = indexDirectory;
151
152 this.lockFactory = lockFactory;
153
154
155
156
157
158 for (IndexCreator indexCreator : indexCreators) {
159 indexCreator.getIndexerFields();
160 }
161
162 this.gavCalculator = new M2GavCalculator();
163
164 prepareIndex(reclaimIndex);
165
166 setIndexDirectoryFile(indexDirectoryFile);
167 }
168
169 private DefaultIndexingContext(
170 String id,
171 String repositoryId,
172 File repository,
173 File indexDirectoryFile,
174 TrackingLockFactory lockFactory,
175 String repositoryUrl,
176 String indexUpdateUrl,
177 List<? extends IndexCreator> indexCreators,
178 boolean reclaimIndex)
179 throws IOException, ExistingLuceneIndexMismatchException {
180 this(
181 id,
182 repositoryId,
183 repository,
184 repositoryUrl,
185 indexUpdateUrl,
186 indexCreators,
187 FSDirectory.open(indexDirectoryFile.toPath(), lockFactory),
188 lockFactory,
189 reclaimIndex,
190 indexDirectoryFile);
191 }
192
193 public DefaultIndexingContext(
194 String id,
195 String repositoryId,
196 File repository,
197 File indexDirectoryFile,
198 String repositoryUrl,
199 String indexUpdateUrl,
200 List<? extends IndexCreator> indexCreators,
201 boolean reclaimIndex)
202 throws IOException, ExistingLuceneIndexMismatchException {
203 this(
204 id,
205 repositoryId,
206 repository,
207 indexDirectoryFile,
208 new TrackingLockFactory(FSLockFactory.getDefault()),
209 repositoryUrl,
210 indexUpdateUrl,
211 indexCreators,
212 reclaimIndex);
213 }
214
215 @Deprecated
216 public DefaultIndexingContext(
217 String id,
218 String repositoryId,
219 File repository,
220 Directory indexDirectory,
221 String repositoryUrl,
222 String indexUpdateUrl,
223 List<? extends IndexCreator> indexCreators,
224 boolean reclaimIndex)
225 throws IOException, ExistingLuceneIndexMismatchException {
226 this(
227 id,
228 repositoryId,
229 repository,
230 repositoryUrl,
231 indexUpdateUrl,
232 indexCreators,
233 indexDirectory,
234 null,
235 reclaimIndex,
236 indexDirectory instanceof FSDirectory
237 ? ((FSDirectory) indexDirectory).getDirectory().toFile()
238 : null);
239 }
240
241 public Directory getIndexDirectory() {
242 return indexDirectory;
243 }
244
245
246
247
248
249 protected void setIndexDirectoryFile(File dir) throws IOException {
250 if (dir == null) {
251
252 this.indexDirectoryFile =
253 Files.createTempDirectory("mindexer-ctx" + id).toFile();
254 this.indexDirectoryFile.deleteOnExit();
255 } else {
256 this.indexDirectoryFile = dir;
257 }
258 }
259
260 public File getIndexDirectoryFile() {
261 return indexDirectoryFile;
262 }
263
264 private void prepareIndex(boolean reclaimIndex) throws IOException, ExistingLuceneIndexMismatchException {
265 if (DirectoryReader.indexExists(indexDirectory)) {
266 try {
267
268 try {
269 indexDirectory.obtainLock(IndexWriter.WRITE_LOCK_NAME).close();
270 } catch (LockObtainFailedException failed) {
271 unlockForcibly(lockFactory, indexDirectory);
272 }
273
274 openAndWarmup();
275
276 checkAndUpdateIndexDescriptor(reclaimIndex);
277 } catch (IOException e) {
278 if (reclaimIndex) {
279 prepareCleanIndex(true);
280 } else {
281 throw e;
282 }
283 }
284 } else {
285 prepareCleanIndex(false);
286 }
287
288 timestamp = IndexUtils.getTimestamp(indexDirectory);
289 }
290
291 private void prepareCleanIndex(boolean deleteExisting) throws IOException {
292 if (deleteExisting) {
293 closeReaders();
294
295
296 try {
297 indexDirectory.obtainLock(IndexWriter.WRITE_LOCK_NAME).close();
298 } catch (LockObtainFailedException failed) {
299 unlockForcibly(lockFactory, indexDirectory);
300 }
301
302 deleteIndexFiles(true);
303 }
304
305 openAndWarmup();
306
307 if (StringUtils.isEmpty(getRepositoryId())) {
308 throw new IllegalArgumentException("The repositoryId cannot be null when creating new repository!");
309 }
310
311 storeDescriptor();
312 }
313
314 private void checkAndUpdateIndexDescriptor(boolean reclaimIndex)
315 throws IOException, ExistingLuceneIndexMismatchException {
316 if (reclaimIndex) {
317
318 storeDescriptor();
319 return;
320 }
321
322
323 if (getSize() > 0) {
324 final TopScoreDocCollector collector = TopScoreDocCollector.create(1, Integer.MAX_VALUE);
325 final IndexSearcher indexSearcher = acquireIndexSearcher();
326 try {
327 indexSearcher.search(new TermQuery(DESCRIPTOR_TERM), collector);
328
329 if (collector.getTotalHits() == 0) {
330 throw new ExistingLuceneIndexMismatchException("The existing index has no NexusIndexer descriptor");
331 }
332
333 if (collector.getTotalHits() > 1) {
334
335 storeDescriptor();
336 } else {
337
338 Document descriptor = indexSearcher.doc(collector.topDocs().scoreDocs[0].doc);
339 String[] h = StringUtils.split(descriptor.get(FLD_IDXINFO), ArtifactInfo.FS);
340
341 String repoId = h[1];
342
343
344
345
346
347
348
349
350 if (getRepositoryId() == null) {
351 repositoryId = repoId;
352 } else if (!getRepositoryId().equals(repoId)) {
353 throw new ExistingLuceneIndexMismatchException(
354 "The existing index is for repository "
355 + "[" + repoId + "] and not for repository [" + getRepositoryId() + "]");
356 }
357 }
358 } finally {
359 releaseIndexSearcher(indexSearcher);
360 }
361 }
362 }
363
364 private void storeDescriptor() throws IOException {
365 Document hdr = new Document();
366
367 hdr.add(new Field(FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS, IndexerField.KEYWORD_STORED));
368
369 hdr.add(new StoredField(
370 FLD_IDXINFO, VERSION + ArtifactInfo.FS + getRepositoryId(), IndexerField.KEYWORD_STORED));
371
372 IndexWriter w = getIndexWriter();
373
374 w.updateDocument(DESCRIPTOR_TERM, hdr);
375
376 w.commit();
377 }
378
379 private void deleteIndexFiles(boolean full) throws IOException {
380 if (indexDirectory != null) {
381 String[] names = indexDirectory.listAll();
382
383 if (names != null) {
384
385 for (String name : names) {
386 if (!(name.equals(INDEX_PACKER_PROPERTIES_FILE) || name.equals(INDEX_UPDATER_PROPERTIES_FILE))) {
387 indexDirectory.deleteFile(name);
388 }
389 }
390 }
391
392 if (full) {
393 try {
394 indexDirectory.deleteFile(INDEX_PACKER_PROPERTIES_FILE);
395 } catch (IOException ioe) {
396
397 }
398
399 try {
400 indexDirectory.deleteFile(INDEX_UPDATER_PROPERTIES_FILE);
401 } catch (IOException ioe) {
402
403 }
404 }
405
406 IndexUtils.deleteTimestamp(indexDirectory);
407 }
408 }
409
410
411
412 public boolean isSearchable() {
413 return searchable;
414 }
415
416 public void setSearchable(boolean searchable) {
417 this.searchable = searchable;
418 }
419
420 public String getId() {
421 return id;
422 }
423
424 public void updateTimestamp() throws IOException {
425 updateTimestamp(false);
426 }
427
428 public void updateTimestamp(boolean save) throws IOException {
429 updateTimestamp(save, new Date());
430 }
431
432 public void updateTimestamp(boolean save, Date timestamp) throws IOException {
433 this.timestamp = timestamp;
434
435 if (save) {
436 IndexUtils.updateTimestamp(indexDirectory, getTimestamp());
437 }
438 }
439
440 public Date getTimestamp() {
441 return timestamp;
442 }
443
444 public int getSize() throws IOException {
445 final IndexSearcher is = acquireIndexSearcher();
446 try {
447 return is.getIndexReader().numDocs();
448 } finally {
449 releaseIndexSearcher(is);
450 }
451 }
452
453 public String getRepositoryId() {
454 return repositoryId;
455 }
456
457 public File getRepository() {
458 return repository;
459 }
460
461 public String getRepositoryUrl() {
462 return repositoryUrl;
463 }
464
465 public String getIndexUpdateUrl() {
466 if (repositoryUrl != null) {
467 if (indexUpdateUrl == null || indexUpdateUrl.trim().length() == 0) {
468 return repositoryUrl + (repositoryUrl.endsWith("/") ? "" : "/") + INDEX_DIRECTORY;
469 }
470 }
471 return indexUpdateUrl;
472 }
473
474 public Analyzer getAnalyzer() {
475 return new NexusAnalyzer();
476 }
477
478 protected void openAndWarmup() throws IOException {
479
480 if (indexWriter != null) {
481 indexWriter.close();
482
483 indexWriter = null;
484 }
485 if (searcherManager != null) {
486 searcherManager.close();
487
488 searcherManager = null;
489 }
490
491 this.indexWriter = new NexusIndexWriter(getIndexDirectory(), getWriterConfig());
492 this.indexWriter.commit();
493 this.searcherManager = new SearcherManager(indexWriter, false, false, new NexusIndexSearcherFactory(this));
494 }
495
496
497
498
499
500
501 protected IndexWriterConfig getWriterConfig() {
502 return NexusIndexWriter.defaultConfig();
503 }
504
505 public IndexWriter getIndexWriter() throws IOException {
506 return indexWriter;
507 }
508
509 public IndexSearcher acquireIndexSearcher() throws IOException {
510
511 searcherManager.maybeRefresh();
512 return searcherManager.acquire();
513 }
514
515 public void releaseIndexSearcher(final IndexSearcher is) throws IOException {
516 if (is == null) {
517 return;
518 }
519 searcherManager.release(is);
520 }
521
522 public void commit() throws IOException {
523 getIndexWriter().commit();
524 }
525
526 public void rollback() throws IOException {
527 getIndexWriter().rollback();
528 }
529
530 public synchronized void optimize() throws CorruptIndexException, IOException {
531 commit();
532 }
533
534 public synchronized void close(boolean deleteFiles) throws IOException {
535 if (indexDirectory != null) {
536 IndexUtils.updateTimestamp(indexDirectory, getTimestamp());
537 closeReaders();
538 if (deleteFiles) {
539 deleteIndexFiles(true);
540 }
541 indexDirectory.close();
542 }
543 indexDirectory = null;
544 }
545
546 public synchronized void purge() throws IOException {
547 closeReaders();
548 deleteIndexFiles(true);
549 try {
550 prepareIndex(true);
551 } catch (ExistingLuceneIndexMismatchException e) {
552
553 }
554 rebuildGroups();
555 updateTimestamp(true, null);
556 }
557
558 public synchronized void replace(Directory directory) throws IOException {
559 replace(directory, null, null);
560 }
561
562 public synchronized void replace(Directory directory, Set<String> allGroups, Set<String> rootGroups)
563 throws IOException {
564 final Date ts = IndexUtils.getTimestamp(directory);
565 closeReaders();
566 deleteIndexFiles(false);
567 IndexUtils.copyDirectory(directory, indexDirectory);
568 openAndWarmup();
569
570 storeDescriptor();
571 if (allGroups == null && rootGroups == null) {
572 rebuildGroups();
573 } else {
574 if (allGroups != null) {
575 setAllGroups(allGroups);
576 }
577 if (rootGroups != null) {
578 setRootGroups(rootGroups);
579 }
580 }
581 updateTimestamp(true, ts);
582 optimize();
583 }
584
585 public synchronized void merge(Directory directory) throws IOException {
586 merge(directory, null);
587 }
588
589 public synchronized void merge(Directory directory, DocumentFilter filter) throws IOException {
590 final IndexSearcher s = acquireIndexSearcher();
591 try {
592 final IndexWriter w = getIndexWriter();
593 try (IndexReader directoryReader = DirectoryReader.open(directory)) {
594 TopScoreDocCollector collector;
595 int numDocs = directoryReader.maxDoc();
596
597 Bits liveDocs = MultiBits.getLiveDocs(directoryReader);
598 for (int i = 0; i < numDocs; i++) {
599 if (liveDocs != null && !liveDocs.get(i)) {
600 continue;
601 }
602
603 Document d = directoryReader.document(i);
604 if (filter != null && !filter.accept(d)) {
605 continue;
606 }
607
608 String uinfo = d.get(ArtifactInfo.UINFO);
609 if (uinfo != null) {
610 collector = TopScoreDocCollector.create(1, Integer.MAX_VALUE);
611 s.search(new TermQuery(new Term(ArtifactInfo.UINFO, uinfo)), collector);
612 if (collector.getTotalHits() == 0) {
613 w.addDocument(IndexUtils.updateDocument(d, this, false));
614 }
615 } else {
616 String deleted = d.get(ArtifactInfo.DELETED);
617
618 if (deleted != null) {
619
620
621
622 w.deleteDocuments(new Term(ArtifactInfo.UINFO, deleted));
623 w.addDocument(d);
624 }
625 }
626 }
627
628 } finally {
629 commit();
630 }
631
632 rebuildGroups();
633 Date mergedTimestamp = IndexUtils.getTimestamp(directory);
634
635 if (getTimestamp() != null && mergedTimestamp != null && mergedTimestamp.after(getTimestamp())) {
636
637 updateTimestamp(true, mergedTimestamp);
638 } else {
639 updateTimestamp(true);
640 }
641 optimize();
642 } finally {
643 releaseIndexSearcher(s);
644 }
645 }
646
647 private void closeReaders() throws CorruptIndexException, IOException {
648 if (searcherManager != null) {
649 searcherManager.close();
650 searcherManager = null;
651 }
652 if (indexWriter != null) {
653 indexWriter.close();
654 indexWriter = null;
655 }
656 }
657
658 public GavCalculator getGavCalculator() {
659 return gavCalculator;
660 }
661
662 public List<IndexCreator> getIndexCreators() {
663 return Collections.unmodifiableList(indexCreators);
664 }
665
666
667
668 public synchronized void rebuildGroups() throws IOException {
669 final IndexSearcher is = acquireIndexSearcher();
670 try {
671 final IndexReader r = is.getIndexReader();
672
673 Set<String> rootGroups = new LinkedHashSet<>();
674 Set<String> allGroups = new LinkedHashSet<>();
675
676 int numDocs = r.maxDoc();
677 Bits liveDocs = MultiBits.getLiveDocs(r);
678
679 for (int i = 0; i < numDocs; i++) {
680 if (liveDocs != null && !liveDocs.get(i)) {
681 continue;
682 }
683
684 Document d = r.document(i);
685
686 String uinfo = d.get(ArtifactInfo.UINFO);
687
688 if (uinfo != null) {
689 ArtifactInfo info = IndexUtils.constructArtifactInfo(d, this);
690 rootGroups.add(info.getRootGroup());
691 allGroups.add(info.getGroupId());
692 }
693 }
694
695 setRootGroups(rootGroups);
696 setAllGroups(allGroups);
697
698 optimize();
699 } finally {
700 releaseIndexSearcher(is);
701 }
702 }
703
704 public Set<String> getAllGroups() {
705 return allGroups.get();
706 }
707
708 public synchronized void setAllGroups(Collection<String> groups) {
709 allGroups.set(new HashSet<>(groups));
710 }
711
712 public Set<String> getRootGroups() throws IOException {
713 return rootGroups.get();
714 }
715
716 public synchronized void setRootGroups(Collection<String> groups) {
717 rootGroups.set(new HashSet<>(groups));
718 }
719
720 private final AtomicReference<HashSet<String>> rootGroups = new AtomicReference<>(new HashSet<>());
721
722 private final AtomicReference<HashSet<String>> allGroups = new AtomicReference<>(new HashSet<>());
723
724 @Override
725 public String toString() {
726 return id + " : " + timestamp;
727 }
728
729 private static void unlockForcibly(final TrackingLockFactory lockFactory, final Directory dir) throws IOException {
730
731
732
733
734
735 if (lockFactory != null) {
736 final Set<? extends Lock> emittedLocks = lockFactory.getEmittedLocks(IndexWriter.WRITE_LOCK_NAME);
737 for (Lock emittedLock : emittedLocks) {
738 emittedLock.close();
739 }
740 }
741 if (dir instanceof FSDirectory) {
742 final FSDirectory fsdir = (FSDirectory) dir;
743 final Path dirPath = fsdir.getDirectory();
744 if (Files.isDirectory(dirPath)) {
745 Path lockPath = dirPath.resolve(IndexWriter.WRITE_LOCK_NAME);
746 try {
747 lockPath = lockPath.toRealPath();
748 } catch (IOException ioe) {
749
750 return;
751 }
752 try (FileChannel fc = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
753 final FileLock lck = fc.tryLock();
754 if (lck == null) {
755
756 throw new LockObtainFailedException("Lock held by another process: " + lockPath);
757 } else {
758
759 lck.close();
760 }
761 }
762 Files.delete(lockPath);
763 }
764 }
765 }
766 }