1 package org.apache.maven.index.context;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.nio.channels.FileChannel;
25 import java.nio.channels.FileLock;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.StandardOpenOption;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.Date;
32 import java.util.HashSet;
33 import java.util.LinkedHashSet;
34 import java.util.List;
35 import java.util.Set;
36 import java.util.concurrent.atomic.AtomicReference;
37
38 import org.apache.lucene.analysis.Analyzer;
39 import org.apache.lucene.document.Document;
40 import org.apache.lucene.document.Field;
41 import org.apache.lucene.document.StoredField;
42 import org.apache.lucene.index.CorruptIndexException;
43 import org.apache.lucene.index.DirectoryReader;
44 import org.apache.lucene.index.IndexReader;
45 import org.apache.lucene.index.IndexWriter;
46 import org.apache.lucene.index.IndexWriterConfig;
47 import org.apache.lucene.index.MultiBits;
48 import org.apache.lucene.index.Term;
49 import org.apache.lucene.search.IndexSearcher;
50 import org.apache.lucene.search.SearcherManager;
51 import org.apache.lucene.search.TermQuery;
52 import org.apache.lucene.search.TopScoreDocCollector;
53 import org.apache.lucene.store.Directory;
54 import org.apache.lucene.store.FSDirectory;
55 import org.apache.lucene.store.FSLockFactory;
56 import org.apache.lucene.store.Lock;
57 import org.apache.lucene.store.LockObtainFailedException;
58 import org.apache.lucene.util.Bits;
59 import org.apache.maven.index.ArtifactInfo;
60 import org.apache.maven.index.IndexerField;
61 import org.apache.maven.index.artifact.GavCalculator;
62 import org.apache.maven.index.artifact.M2GavCalculator;
63 import org.codehaus.plexus.util.StringUtils;
64
65
66
67
68
69
70
71 public class DefaultIndexingContext
72 extends AbstractIndexingContext
73 {
74
75
76
77 private static final String INDEX_DIRECTORY = ".index";
78
79 public static final String FLD_DESCRIPTOR = "DESCRIPTOR";
80
81 public static final String FLD_DESCRIPTOR_CONTENTS = "NexusIndex";
82
83 public static final String FLD_IDXINFO = "IDXINFO";
84
85 public static final String VERSION = "1.0";
86
87 private static final Term DESCRIPTOR_TERM = new Term( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS );
88
89 private Directory indexDirectory;
90
91 private TrackingLockFactory lockFactory;
92
93 private File indexDirectoryFile;
94
95 private String id;
96
97 private boolean searchable;
98
99 private String repositoryId;
100
101 private File repository;
102
103 private String repositoryUrl;
104
105 private String indexUpdateUrl;
106
107 private NexusIndexWriter indexWriter;
108
109 private SearcherManager searcherManager;
110
111 private Date timestamp;
112
113 private List<? extends IndexCreator> indexCreators;
114
115
116
117
118
119
120 private GavCalculator gavCalculator;
121
122 private DefaultIndexingContext( String id,
123 String repositoryId,
124 File repository,
125 String repositoryUrl, String indexUpdateUrl,
126 List<? extends IndexCreator> indexCreators, Directory indexDirectory,
127 TrackingLockFactory lockFactory,
128 boolean reclaimIndex,
129 File indexDirectoryFile )
130 throws ExistingLuceneIndexMismatchException, IOException
131 {
132
133 this.id = id;
134
135 this.searchable = true;
136
137 this.repositoryId = repositoryId;
138
139 this.repository = repository;
140
141 this.repositoryUrl = repositoryUrl;
142
143 this.indexUpdateUrl = indexUpdateUrl;
144
145 this.indexWriter = null;
146
147 this.searcherManager = null;
148
149 this.indexCreators = indexCreators;
150
151 this.indexDirectory = indexDirectory;
152
153 this.lockFactory = lockFactory;
154
155
156
157
158
159 for ( IndexCreator indexCreator : indexCreators )
160 {
161 indexCreator.getIndexerFields();
162 }
163
164 this.gavCalculator = new M2GavCalculator();
165
166 prepareIndex( reclaimIndex );
167
168 setIndexDirectoryFile( indexDirectoryFile );
169 }
170
171 private DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
172 TrackingLockFactory lockFactory, String repositoryUrl, String indexUpdateUrl,
173 List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
174 throws IOException, ExistingLuceneIndexMismatchException
175 {
176 this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators,
177 FSDirectory.open( indexDirectoryFile.toPath(), lockFactory ), lockFactory, reclaimIndex,
178 indexDirectoryFile );
179 }
180
181 public DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
182 String repositoryUrl, String indexUpdateUrl,
183 List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
184 throws IOException, ExistingLuceneIndexMismatchException
185 {
186 this( id, repositoryId, repository, indexDirectoryFile, new TrackingLockFactory( FSLockFactory.getDefault() ),
187 repositoryUrl, indexUpdateUrl, indexCreators, reclaimIndex );
188 }
189
190 @Deprecated
191 public DefaultIndexingContext( String id, String repositoryId, File repository, Directory indexDirectory,
192 String repositoryUrl, String indexUpdateUrl,
193 List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
194 throws IOException, ExistingLuceneIndexMismatchException
195 {
196 this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators, indexDirectory, null,
197 reclaimIndex, indexDirectory instanceof FSDirectory
198 ? ( (FSDirectory) indexDirectory ).getDirectory().toFile() : null
199 );
200 }
201
202 public Directory getIndexDirectory()
203 {
204 return indexDirectory;
205 }
206
207
208
209
210
211 protected void setIndexDirectoryFile( File dir )
212 throws IOException
213 {
214 if ( dir == null )
215 {
216
217 this.indexDirectoryFile = Files.createTempDirectory( "mindexer-ctx" + id ).toFile();
218 this.indexDirectoryFile.deleteOnExit();
219 }
220 else
221 {
222 this.indexDirectoryFile = dir;
223 }
224 }
225
226 public File getIndexDirectoryFile()
227 {
228 return indexDirectoryFile;
229 }
230
231 private void prepareIndex( boolean reclaimIndex )
232 throws IOException, ExistingLuceneIndexMismatchException
233 {
234 if ( DirectoryReader.indexExists( indexDirectory ) )
235 {
236 try
237 {
238
239 try
240 {
241 indexDirectory.obtainLock( IndexWriter.WRITE_LOCK_NAME ).close();
242 }
243 catch ( LockObtainFailedException failed )
244 {
245 unlockForcibly( lockFactory, indexDirectory );
246 }
247
248 openAndWarmup();
249
250 checkAndUpdateIndexDescriptor( reclaimIndex );
251 }
252 catch ( IOException e )
253 {
254 if ( reclaimIndex )
255 {
256 prepareCleanIndex( true );
257 }
258 else
259 {
260 throw e;
261 }
262 }
263 }
264 else
265 {
266 prepareCleanIndex( false );
267 }
268
269 timestamp = IndexUtils.getTimestamp( indexDirectory );
270 }
271
272 private void prepareCleanIndex( boolean deleteExisting )
273 throws IOException
274 {
275 if ( deleteExisting )
276 {
277 closeReaders();
278
279
280 try
281 {
282 indexDirectory.obtainLock( IndexWriter.WRITE_LOCK_NAME ).close();
283 }
284 catch ( LockObtainFailedException failed )
285 {
286 unlockForcibly( lockFactory, indexDirectory );
287 }
288
289 deleteIndexFiles( true );
290 }
291
292 openAndWarmup();
293
294 if ( StringUtils.isEmpty( getRepositoryId() ) )
295 {
296 throw new IllegalArgumentException( "The repositoryId cannot be null when creating new repository!" );
297 }
298
299 storeDescriptor();
300 }
301
302 private void checkAndUpdateIndexDescriptor( boolean reclaimIndex )
303 throws IOException, ExistingLuceneIndexMismatchException
304 {
305 if ( reclaimIndex )
306 {
307
308 storeDescriptor();
309 return;
310 }
311
312
313 if ( getSize() > 0 )
314 {
315 final TopScoreDocCollector collector = TopScoreDocCollector.create( 1, Integer.MAX_VALUE );
316 final IndexSearcher indexSearcher = acquireIndexSearcher();
317 try
318 {
319 indexSearcher.search( new TermQuery( DESCRIPTOR_TERM ), collector );
320
321 if ( collector.getTotalHits() == 0 )
322 {
323 throw new ExistingLuceneIndexMismatchException(
324 "The existing index has no NexusIndexer descriptor" );
325 }
326
327 if ( collector.getTotalHits() > 1 )
328 {
329
330 storeDescriptor();
331 }
332 else
333 {
334
335 Document descriptor = indexSearcher.doc( collector.topDocs().scoreDocs[0].doc );
336 String[] h = StringUtils.split( descriptor.get( FLD_IDXINFO ), ArtifactInfo.FS );
337
338 String repoId = h[1];
339
340
341
342
343
344
345
346
347 if ( getRepositoryId() == null )
348 {
349 repositoryId = repoId;
350 }
351 else if ( !getRepositoryId().equals( repoId ) )
352 {
353 throw new ExistingLuceneIndexMismatchException( "The existing index is for repository "
354 + "[" + repoId + "] and not for repository [" + getRepositoryId() + "]" );
355 }
356 }
357 }
358 finally
359 {
360 releaseIndexSearcher( indexSearcher );
361 }
362 }
363 }
364
365 private void storeDescriptor()
366 throws IOException
367 {
368 Document hdr = new Document();
369
370 hdr.add( new Field( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS, IndexerField.KEYWORD_STORED ) );
371
372 hdr.add( new StoredField( FLD_IDXINFO, VERSION + ArtifactInfo.FS + getRepositoryId(),
373 IndexerField.KEYWORD_STORED ) );
374
375 IndexWriter w = getIndexWriter();
376
377 w.updateDocument( DESCRIPTOR_TERM, hdr );
378
379 w.commit();
380 }
381
382 private void deleteIndexFiles( boolean full )
383 throws IOException
384 {
385 if ( indexDirectory != null )
386 {
387 String[] names = indexDirectory.listAll();
388
389 if ( names != null )
390 {
391
392 for ( String name : names )
393 {
394 if ( !( name.equals( INDEX_PACKER_PROPERTIES_FILE )
395 || name.equals( INDEX_UPDATER_PROPERTIES_FILE ) ) )
396 {
397 indexDirectory.deleteFile( name );
398 }
399 }
400 }
401
402 if ( full )
403 {
404 try
405 {
406 indexDirectory.deleteFile( INDEX_PACKER_PROPERTIES_FILE );
407 }
408 catch ( IOException ioe )
409 {
410
411 }
412
413 try
414 {
415 indexDirectory.deleteFile( INDEX_UPDATER_PROPERTIES_FILE );
416 }
417 catch ( IOException ioe )
418 {
419
420 }
421 }
422
423 IndexUtils.deleteTimestamp( indexDirectory );
424 }
425 }
426
427
428
429 public boolean isSearchable()
430 {
431 return searchable;
432 }
433
434 public void setSearchable( boolean searchable )
435 {
436 this.searchable = searchable;
437 }
438
439 public String getId()
440 {
441 return id;
442 }
443
444 public void updateTimestamp()
445 throws IOException
446 {
447 updateTimestamp( false );
448 }
449
450 public void updateTimestamp( boolean save )
451 throws IOException
452 {
453 updateTimestamp( save, new Date() );
454 }
455
456 public void updateTimestamp( boolean save, Date timestamp )
457 throws IOException
458 {
459 this.timestamp = timestamp;
460
461 if ( save )
462 {
463 IndexUtils.updateTimestamp( indexDirectory, getTimestamp() );
464 }
465 }
466
467 public Date getTimestamp()
468 {
469 return timestamp;
470 }
471
472 public int getSize()
473 throws IOException
474 {
475 final IndexSearcher is = acquireIndexSearcher();
476 try
477 {
478 return is.getIndexReader().numDocs();
479 }
480 finally
481 {
482 releaseIndexSearcher( is );
483 }
484 }
485
486 public String getRepositoryId()
487 {
488 return repositoryId;
489 }
490
491 public File getRepository()
492 {
493 return repository;
494 }
495
496 public String getRepositoryUrl()
497 {
498 return repositoryUrl;
499 }
500
501 public String getIndexUpdateUrl()
502 {
503 if ( repositoryUrl != null )
504 {
505 if ( indexUpdateUrl == null || indexUpdateUrl.trim().length() == 0 )
506 {
507 return repositoryUrl + ( repositoryUrl.endsWith( "/" ) ? "" : "/" ) + INDEX_DIRECTORY;
508 }
509 }
510 return indexUpdateUrl;
511 }
512
513 public Analyzer getAnalyzer()
514 {
515 return new NexusAnalyzer();
516 }
517
518 protected void openAndWarmup()
519 throws IOException
520 {
521
522 if ( indexWriter != null )
523 {
524 indexWriter.close();
525
526 indexWriter = null;
527 }
528 if ( searcherManager != null )
529 {
530 searcherManager.close();
531
532 searcherManager = null;
533 }
534
535 this.indexWriter = new NexusIndexWriter( getIndexDirectory(), getWriterConfig() );
536 this.indexWriter.commit();
537 this.searcherManager = new SearcherManager( indexWriter, false, false, new NexusIndexSearcherFactory( this ) );
538 }
539
540
541
542
543
544
545 protected IndexWriterConfig getWriterConfig()
546 {
547 return NexusIndexWriter.defaultConfig();
548 }
549
550 public IndexWriter getIndexWriter()
551 throws IOException
552 {
553 return indexWriter;
554 }
555
556 public IndexSearcher acquireIndexSearcher()
557 throws IOException
558 {
559
560 searcherManager.maybeRefresh();
561 return searcherManager.acquire();
562 }
563
564 public void releaseIndexSearcher( final IndexSearcher is )
565 throws IOException
566 {
567 if ( is == null )
568 {
569 return;
570 }
571 searcherManager.release( is );
572 }
573
574 public void commit()
575 throws IOException
576 {
577 getIndexWriter().commit();
578 }
579
580 public void rollback()
581 throws IOException
582 {
583 getIndexWriter().rollback();
584 }
585
586 public synchronized void optimize()
587 throws CorruptIndexException, IOException
588 {
589 commit();
590 }
591
592 public synchronized void close( boolean deleteFiles )
593 throws IOException
594 {
595 if ( indexDirectory != null )
596 {
597 IndexUtils.updateTimestamp( indexDirectory, getTimestamp() );
598 closeReaders();
599 if ( deleteFiles )
600 {
601 deleteIndexFiles( true );
602 }
603 indexDirectory.close();
604 }
605 indexDirectory = null;
606 }
607
608 public synchronized void purge()
609 throws IOException
610 {
611 closeReaders();
612 deleteIndexFiles( true );
613 try
614 {
615 prepareIndex( true );
616 }
617 catch ( ExistingLuceneIndexMismatchException e )
618 {
619
620 }
621 rebuildGroups();
622 updateTimestamp( true, null );
623 }
624
625 public synchronized void replace( Directory directory )
626 throws IOException
627 {
628 replace( directory, null, null );
629 }
630
631 public synchronized void replace( Directory directory, Set<String> allGroups, Set<String> rootGroups )
632 throws IOException
633 {
634 final Date ts = IndexUtils.getTimestamp( directory );
635 closeReaders();
636 deleteIndexFiles( false );
637 IndexUtils.copyDirectory( directory, indexDirectory );
638 openAndWarmup();
639
640 storeDescriptor();
641 if ( allGroups == null && rootGroups == null )
642 {
643 rebuildGroups();
644 }
645 else
646 {
647 if ( allGroups != null )
648 {
649 setAllGroups( allGroups );
650 }
651 if ( rootGroups != null )
652 {
653 setRootGroups( rootGroups );
654 }
655 }
656 updateTimestamp( true, ts );
657 optimize();
658 }
659
660 public synchronized void merge( Directory directory )
661 throws IOException
662 {
663 merge( directory, null );
664 }
665
666 public synchronized void merge( Directory directory, DocumentFilter filter )
667 throws IOException
668 {
669 final IndexSearcher s = acquireIndexSearcher();
670 try
671 {
672 final IndexWriter w = getIndexWriter();
673 try ( IndexReader directoryReader = DirectoryReader.open( directory ) )
674 {
675 TopScoreDocCollector collector;
676 int numDocs = directoryReader.maxDoc();
677
678 Bits liveDocs = MultiBits.getLiveDocs( directoryReader );
679 for ( int i = 0; i < numDocs; i++ )
680 {
681 if ( liveDocs != null && !liveDocs.get( i ) )
682 {
683 continue;
684 }
685
686 Document d = directoryReader.document( i );
687 if ( filter != null && !filter.accept( d ) )
688 {
689 continue;
690 }
691
692 String uinfo = d.get( ArtifactInfo.UINFO );
693 if ( uinfo != null )
694 {
695 collector = TopScoreDocCollector.create( 1, Integer.MAX_VALUE );
696 s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), collector );
697 if ( collector.getTotalHits() == 0 )
698 {
699 w.addDocument( IndexUtils.updateDocument( d, this, false ) );
700 }
701 }
702 else
703 {
704 String deleted = d.get( ArtifactInfo.DELETED );
705
706 if ( deleted != null )
707 {
708
709
710
711 w.deleteDocuments( new Term( ArtifactInfo.UINFO, deleted ) );
712 w.addDocument( d );
713 }
714 }
715 }
716
717 }
718 finally
719 {
720 commit();
721 }
722
723 rebuildGroups();
724 Date mergedTimestamp = IndexUtils.getTimestamp( directory );
725
726 if ( getTimestamp() != null && mergedTimestamp != null && mergedTimestamp.after( getTimestamp() ) )
727 {
728
729 updateTimestamp( true, mergedTimestamp );
730 }
731 else
732 {
733 updateTimestamp( true );
734 }
735 optimize();
736 }
737 finally
738 {
739 releaseIndexSearcher( s );
740 }
741 }
742
743 private void closeReaders()
744 throws CorruptIndexException, IOException
745 {
746 if ( searcherManager != null )
747 {
748 searcherManager.close();
749 searcherManager = null;
750 }
751 if ( indexWriter != null )
752 {
753 indexWriter.close();
754 indexWriter = null;
755 }
756 }
757
758 public GavCalculator getGavCalculator()
759 {
760 return gavCalculator;
761 }
762
763 public List<IndexCreator> getIndexCreators()
764 {
765 return Collections.unmodifiableList( indexCreators );
766 }
767
768
769
770 public synchronized void rebuildGroups()
771 throws IOException
772 {
773 final IndexSearcher is = acquireIndexSearcher();
774 try
775 {
776 final IndexReader r = is.getIndexReader();
777
778 Set<String> rootGroups = new LinkedHashSet<>();
779 Set<String> allGroups = new LinkedHashSet<>();
780
781 int numDocs = r.maxDoc();
782 Bits liveDocs = MultiBits.getLiveDocs( r );
783
784 for ( int i = 0; i < numDocs; i++ )
785 {
786 if ( liveDocs != null && !liveDocs.get( i ) )
787 {
788 continue;
789 }
790
791 Document d = r.document( i );
792
793 String uinfo = d.get( ArtifactInfo.UINFO );
794
795 if ( uinfo != null )
796 {
797 ArtifactInfo info = IndexUtils.constructArtifactInfo( d, this );
798 rootGroups.add( info.getRootGroup() );
799 allGroups.add( info.getGroupId() );
800 }
801 }
802
803 setRootGroups( rootGroups );
804 setAllGroups( allGroups );
805
806 optimize();
807 }
808 finally
809 {
810 releaseIndexSearcher( is );
811 }
812 }
813
814 public Set<String> getAllGroups()
815 {
816 return allGroups.get();
817 }
818
819 public synchronized void setAllGroups( Collection<String> groups )
820 {
821 allGroups.set( new HashSet<>( groups ) );
822 }
823
824 public Set<String> getRootGroups()
825 throws IOException
826 {
827 return rootGroups.get();
828 }
829
830 public synchronized void setRootGroups( Collection<String> groups )
831 {
832 rootGroups.set( new HashSet<>( groups ) );
833 }
834
835 private final AtomicReference<HashSet<String>> rootGroups = new AtomicReference<>( new HashSet<>() );
836
837 private final AtomicReference<HashSet<String>> allGroups = new AtomicReference<>( new HashSet<>() );
838
839 @Override
840 public String toString()
841 {
842 return id + " : " + timestamp;
843 }
844
845 private static void unlockForcibly( final TrackingLockFactory lockFactory, final Directory dir )
846 throws IOException
847 {
848
849
850
851
852
853 if ( lockFactory != null )
854 {
855 final Set<? extends Lock> emittedLocks = lockFactory.getEmittedLocks( IndexWriter.WRITE_LOCK_NAME );
856 for ( Lock emittedLock : emittedLocks )
857 {
858 emittedLock.close();
859 }
860 }
861 if ( dir instanceof FSDirectory )
862 {
863 final FSDirectory fsdir = (FSDirectory) dir;
864 final Path dirPath = fsdir.getDirectory();
865 if ( Files.isDirectory( dirPath ) )
866 {
867 Path lockPath = dirPath.resolve( IndexWriter.WRITE_LOCK_NAME );
868 try
869 {
870 lockPath = lockPath.toRealPath();
871 }
872 catch ( IOException ioe )
873 {
874
875 return;
876 }
877 try ( FileChannel fc = FileChannel.open( lockPath, StandardOpenOption.CREATE,
878 StandardOpenOption.WRITE ) )
879 {
880 final FileLock lck = fc.tryLock();
881 if ( lck == null )
882 {
883
884 throw new LockObtainFailedException( "Lock held by another process: " + lockPath );
885 }
886 else
887 {
888
889 lck.close();
890 }
891 }
892 Files.delete( lockPath );
893 }
894 }
895 }
896 }