1 package org.apache.maven.index.context;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.File;
23 import java.io.IOException;
24 import java.nio.channels.FileChannel;
25 import java.nio.channels.FileLock;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.StandardOpenOption;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.Date;
33 import java.util.LinkedHashSet;
34 import java.util.List;
35 import java.util.Set;
36 import org.apache.lucene.analysis.Analyzer;
37 import org.apache.lucene.document.Document;
38 import org.apache.lucene.document.Field;
39 import org.apache.lucene.index.CorruptIndexException;
40 import org.apache.lucene.index.DirectoryReader;
41 import org.apache.lucene.index.IndexReader;
42 import org.apache.lucene.index.IndexWriter;
43 import org.apache.lucene.index.IndexWriterConfig;
44 import org.apache.lucene.index.MultiFields;
45 import org.apache.lucene.index.Term;
46 import org.apache.lucene.search.IndexSearcher;
47 import org.apache.lucene.search.SearcherManager;
48 import org.apache.lucene.search.TermQuery;
49 import org.apache.lucene.search.TopDocs;
50 import org.apache.lucene.search.TopScoreDocCollector;
51 import org.apache.lucene.store.Directory;
52 import org.apache.lucene.store.FSDirectory;
53 import org.apache.lucene.store.FSLockFactory;
54 import org.apache.lucene.store.Lock;
55 import org.apache.lucene.store.LockObtainFailedException;
56 import org.apache.lucene.util.Bits;
57 import org.apache.maven.index.ArtifactInfo;
58 import org.apache.maven.index.artifact.GavCalculator;
59 import org.apache.maven.index.artifact.M2GavCalculator;
60 import org.codehaus.plexus.util.StringUtils;
61
62
63
64
65
66
67
68 public class DefaultIndexingContext
69 extends AbstractIndexingContext
70 {
71
72
73
74 private static final String INDEX_DIRECTORY = ".index";
75
76 public static final String FLD_DESCRIPTOR = "DESCRIPTOR";
77
78 public static final String FLD_DESCRIPTOR_CONTENTS = "NexusIndex";
79
80 public static final String FLD_IDXINFO = "IDXINFO";
81
82 public static final String VERSION = "1.0";
83
84 private static final Term DESCRIPTOR_TERM = new Term( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS );
85
86 private Directory indexDirectory;
87
88 private TrackingLockFactory lockFactory;
89
90 private File indexDirectoryFile;
91
92 private String id;
93
94 private boolean searchable;
95
96 private String repositoryId;
97
98 private File repository;
99
100 private String repositoryUrl;
101
102 private String indexUpdateUrl;
103
104 private NexusIndexWriter indexWriter;
105
106 private SearcherManager searcherManager;
107
108 private Date timestamp;
109
110 private List<? extends IndexCreator> indexCreators;
111
112
113
114
115
116
117 private GavCalculator gavCalculator;
118
119 private DefaultIndexingContext( String id,
120 String repositoryId,
121 File repository,
122 String repositoryUrl, String indexUpdateUrl,
123 List<? extends IndexCreator> indexCreators, Directory indexDirectory,
124 TrackingLockFactory lockFactory,
125 boolean reclaimIndex )
126 throws ExistingLuceneIndexMismatchException, IOException
127 {
128
129 this.id = id;
130
131 this.searchable = true;
132
133 this.repositoryId = repositoryId;
134
135 this.repository = repository;
136
137 this.repositoryUrl = repositoryUrl;
138
139 this.indexUpdateUrl = indexUpdateUrl;
140
141 this.indexWriter = null;
142
143 this.searcherManager = null;
144
145 this.indexCreators = indexCreators;
146
147 this.indexDirectory = indexDirectory;
148
149 this.lockFactory = lockFactory;
150
151
152
153
154
155 for ( IndexCreator indexCreator : indexCreators )
156 {
157 indexCreator.getIndexerFields();
158 }
159
160 this.gavCalculator = new M2GavCalculator();
161
162 prepareIndex( reclaimIndex );
163
164 setIndexDirectoryFile( null );
165 }
166
167 private DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
168 TrackingLockFactory lockFactory, String repositoryUrl, String indexUpdateUrl,
169 List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
170 throws IOException, ExistingLuceneIndexMismatchException
171 {
172 this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators,
173 FSDirectory.open( indexDirectoryFile.toPath(), lockFactory ), lockFactory, reclaimIndex );
174
175 setIndexDirectoryFile( indexDirectoryFile );
176 }
177
178 public DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
179 String repositoryUrl, String indexUpdateUrl,
180 List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
181 throws IOException, ExistingLuceneIndexMismatchException
182 {
183 this( id, repositoryId, repository, indexDirectoryFile, new TrackingLockFactory( FSLockFactory.getDefault() ),
184 repositoryUrl, indexUpdateUrl, indexCreators, reclaimIndex );
185 }
186
187 @Deprecated
188 public DefaultIndexingContext( String id, String repositoryId, File repository, Directory indexDirectory,
189 String repositoryUrl, String indexUpdateUrl,
190 List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
191 throws IOException, ExistingLuceneIndexMismatchException
192 {
193 this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators, indexDirectory, null,
194 reclaimIndex );
195
196 if ( indexDirectory instanceof FSDirectory )
197 {
198 setIndexDirectoryFile( ( (FSDirectory) indexDirectory ).getDirectory().toFile() );
199 }
200 }
201
202 public Directory getIndexDirectory()
203 {
204 return indexDirectory;
205 }
206
207
208
209
210
211 protected void setIndexDirectoryFile( File dir )
212 throws IOException
213 {
214 if ( dir == null )
215 {
216
217 File tmpFile = File.createTempFile( "mindexer-ctx" + id, "tmp" );
218 tmpFile.deleteOnExit();
219 tmpFile.delete();
220 tmpFile.mkdirs();
221 this.indexDirectoryFile = tmpFile;
222 }
223 else
224 {
225 this.indexDirectoryFile = dir;
226 }
227 }
228
229 public File getIndexDirectoryFile()
230 {
231 return indexDirectoryFile;
232 }
233
234 private void prepareIndex( boolean reclaimIndex )
235 throws IOException, ExistingLuceneIndexMismatchException
236 {
237 if ( DirectoryReader.indexExists( indexDirectory ) )
238 {
239 try
240 {
241
242 if ( IndexWriter.isLocked( indexDirectory ) )
243 {
244 unlockForcibly( lockFactory, indexDirectory );
245 }
246
247 openAndWarmup();
248
249 checkAndUpdateIndexDescriptor( reclaimIndex );
250 }
251 catch ( IOException e )
252 {
253 if ( reclaimIndex )
254 {
255 prepareCleanIndex( true );
256 }
257 else
258 {
259 throw e;
260 }
261 }
262 }
263 else
264 {
265 prepareCleanIndex( false );
266 }
267
268 timestamp = IndexUtils.getTimestamp( indexDirectory );
269 }
270
271 private void prepareCleanIndex( boolean deleteExisting )
272 throws IOException
273 {
274 if ( deleteExisting )
275 {
276 closeReaders();
277
278
279 if ( IndexWriter.isLocked( indexDirectory ) )
280 {
281 unlockForcibly( lockFactory, indexDirectory );
282 }
283
284 deleteIndexFiles( true );
285 }
286
287 openAndWarmup();
288
289 if ( StringUtils.isEmpty( getRepositoryId() ) )
290 {
291 throw new IllegalArgumentException( "The repositoryId cannot be null when creating new repository!" );
292 }
293
294 storeDescriptor();
295 }
296
297 private void checkAndUpdateIndexDescriptor( boolean reclaimIndex )
298 throws IOException, ExistingLuceneIndexMismatchException
299 {
300 if ( reclaimIndex )
301 {
302
303 storeDescriptor();
304 return;
305 }
306
307
308 if ( getSize() > 0 )
309 {
310 final TopScoreDocCollector collector = TopScoreDocCollector.create( 1 );
311 final IndexSearcher indexSearcher = acquireIndexSearcher();
312 try
313 {
314 indexSearcher.search( new TermQuery( DESCRIPTOR_TERM ), collector );
315
316 if ( collector.getTotalHits() == 0 )
317 {
318 throw new ExistingLuceneIndexMismatchException(
319 "The existing index has no NexusIndexer descriptor" );
320 }
321
322 if ( collector.getTotalHits() > 1 )
323 {
324
325 storeDescriptor();
326 return;
327 }
328 else
329 {
330
331 Document descriptor = indexSearcher.doc( collector.topDocs().scoreDocs[0].doc );
332 String[] h = StringUtils.split( descriptor.get( FLD_IDXINFO ), ArtifactInfo.FS );
333
334 String repoId = h[1];
335
336
337
338
339
340
341
342
343 if ( getRepositoryId() == null )
344 {
345 repositoryId = repoId;
346 }
347 else if ( !getRepositoryId().equals( repoId ) )
348 {
349 throw new ExistingLuceneIndexMismatchException( "The existing index is for repository "
350 + "[" + repoId + "] and not for repository [" + getRepositoryId() + "]" );
351 }
352 }
353 }
354 finally
355 {
356 releaseIndexSearcher( indexSearcher );
357 }
358 }
359 }
360
361 private void storeDescriptor()
362 throws IOException
363 {
364 Document hdr = new Document();
365
366 hdr.add( new Field( FLD_DESCRIPTOR, FLD_DESCRIPTOR_CONTENTS, Field.Store.YES, Field.Index.NOT_ANALYZED ) );
367
368 hdr.add( new Field( FLD_IDXINFO, VERSION + ArtifactInfo.FS + getRepositoryId(), Field.Store.YES,
369 Field.Index.NO ) );
370
371 IndexWriter w = getIndexWriter();
372
373 w.updateDocument( DESCRIPTOR_TERM, hdr );
374
375 w.commit();
376 }
377
378 private void deleteIndexFiles( boolean full )
379 throws IOException
380 {
381 if ( indexDirectory != null )
382 {
383 String[] names = indexDirectory.listAll();
384
385 if ( names != null )
386 {
387
388 for ( String name : names )
389 {
390 if ( !( name.equals( INDEX_PACKER_PROPERTIES_FILE )
391 || name.equals( INDEX_UPDATER_PROPERTIES_FILE ) ) )
392 {
393 indexDirectory.deleteFile( name );
394 }
395 }
396 }
397
398 if ( full )
399 {
400 try
401 {
402 indexDirectory.deleteFile( INDEX_PACKER_PROPERTIES_FILE );
403 }
404 catch ( IOException ioe )
405 {
406
407 }
408
409 try
410 {
411 indexDirectory.deleteFile( INDEX_UPDATER_PROPERTIES_FILE );
412 }
413 catch ( IOException ioe )
414 {
415
416 }
417 }
418
419 IndexUtils.deleteTimestamp( indexDirectory );
420 }
421 }
422
423
424
425 public boolean isSearchable()
426 {
427 return searchable;
428 }
429
430 public void setSearchable( boolean searchable )
431 {
432 this.searchable = searchable;
433 }
434
435 public String getId()
436 {
437 return id;
438 }
439
440 public void updateTimestamp()
441 throws IOException
442 {
443 updateTimestamp( false );
444 }
445
446 public void updateTimestamp( boolean save )
447 throws IOException
448 {
449 updateTimestamp( save, new Date() );
450 }
451
452 public void updateTimestamp( boolean save, Date timestamp )
453 throws IOException
454 {
455 this.timestamp = timestamp;
456
457 if ( save )
458 {
459 IndexUtils.updateTimestamp( indexDirectory, getTimestamp() );
460 }
461 }
462
463 public Date getTimestamp()
464 {
465 return timestamp;
466 }
467
468 public int getSize()
469 throws IOException
470 {
471 final IndexSearcher is = acquireIndexSearcher();
472 try
473 {
474 return is.getIndexReader().numDocs();
475 }
476 finally
477 {
478 releaseIndexSearcher( is );
479 }
480 }
481
482 public String getRepositoryId()
483 {
484 return repositoryId;
485 }
486
487 public File getRepository()
488 {
489 return repository;
490 }
491
492 public String getRepositoryUrl()
493 {
494 return repositoryUrl;
495 }
496
497 public String getIndexUpdateUrl()
498 {
499 if ( repositoryUrl != null )
500 {
501 if ( indexUpdateUrl == null || indexUpdateUrl.trim().length() == 0 )
502 {
503 return repositoryUrl + ( repositoryUrl.endsWith( "/" ) ? "" : "/" ) + INDEX_DIRECTORY;
504 }
505 }
506 return indexUpdateUrl;
507 }
508
509 public Analyzer getAnalyzer()
510 {
511 return new NexusAnalyzer();
512 }
513
514 protected void openAndWarmup()
515 throws IOException
516 {
517
518 if ( indexWriter != null )
519 {
520 indexWriter.close();
521
522 indexWriter = null;
523 }
524 if ( searcherManager != null )
525 {
526 searcherManager.close();
527
528 searcherManager = null;
529 }
530
531 this.indexWriter = new NexusIndexWriter( getIndexDirectory(), getWriterConfig() );
532 this.indexWriter.commit();
533 this.searcherManager = new SearcherManager( indexWriter, false, new NexusIndexSearcherFactory( this ) );
534 }
535
536
537
538
539
540
541 protected IndexWriterConfig getWriterConfig()
542 {
543 return NexusIndexWriter.defaultConfig();
544 }
545
546 public IndexWriter getIndexWriter()
547 throws IOException
548 {
549 return indexWriter;
550 }
551
552 public IndexSearcher acquireIndexSearcher()
553 throws IOException
554 {
555
556 searcherManager.maybeRefresh();
557 return searcherManager.acquire();
558 }
559
560 public void releaseIndexSearcher( final IndexSearcher is )
561 throws IOException
562 {
563 if ( is == null )
564 {
565 return;
566 }
567 searcherManager.release( is );
568 }
569
570 public void commit()
571 throws IOException
572 {
573 getIndexWriter().commit();
574 }
575
576 public void rollback()
577 throws IOException
578 {
579 getIndexWriter().rollback();
580 }
581
582 public synchronized void optimize()
583 throws CorruptIndexException, IOException
584 {
585 commit();
586 }
587
588 public synchronized void close( boolean deleteFiles )
589 throws IOException
590 {
591 if ( indexDirectory != null )
592 {
593 IndexUtils.updateTimestamp( indexDirectory, getTimestamp() );
594 closeReaders();
595 if ( deleteFiles )
596 {
597 deleteIndexFiles( true );
598 }
599 indexDirectory.close();
600 }
601 indexDirectory = null;
602 }
603
604 public synchronized void purge()
605 throws IOException
606 {
607 closeReaders();
608 deleteIndexFiles( true );
609 openAndWarmup();
610 try
611 {
612 prepareIndex( true );
613 }
614 catch ( ExistingLuceneIndexMismatchException e )
615 {
616
617 }
618 rebuildGroups();
619 updateTimestamp( true, null );
620 }
621
622 public synchronized void replace( Directory directory )
623 throws IOException
624 {
625 replace( directory, null, null );
626 }
627
628 public synchronized void replace( Directory directory, Set<String> allGroups, Set<String> rootGroups )
629 throws IOException
630 {
631 final Date ts = IndexUtils.getTimestamp( directory );
632 closeReaders();
633 deleteIndexFiles( false );
634 IndexUtils.copyDirectory( directory, indexDirectory );
635 openAndWarmup();
636
637 storeDescriptor();
638 if ( allGroups == null && rootGroups == null )
639 {
640 rebuildGroups();
641 }
642 else
643 {
644 if ( allGroups != null )
645 {
646 setAllGroups( allGroups );
647 }
648 if ( rootGroups != null )
649 {
650 setRootGroups( rootGroups );
651 }
652 }
653 updateTimestamp( true, ts );
654 optimize();
655 }
656
657 public synchronized void merge( Directory directory )
658 throws IOException
659 {
660 merge( directory, null );
661 }
662
663 public synchronized void merge( Directory directory, DocumentFilter filter )
664 throws IOException
665 {
666 final IndexSearcher s = acquireIndexSearcher();
667 try
668 {
669 final IndexWriter w = getIndexWriter();
670 final IndexReader directoryReader = DirectoryReader.open( directory );
671 TopScoreDocCollector collector = null;
672 try
673 {
674 int numDocs = directoryReader.maxDoc();
675
676 Bits liveDocs = MultiFields.getLiveDocs( directoryReader );
677 for ( int i = 0; i < numDocs; i++ )
678 {
679 if ( liveDocs != null && !liveDocs.get( i ) )
680 {
681 continue;
682 }
683
684 Document d = directoryReader.document( i );
685 if ( filter != null && !filter.accept( d ) )
686 {
687 continue;
688 }
689
690 String uinfo = d.get( ArtifactInfo.UINFO );
691 if ( uinfo != null )
692 {
693 collector = TopScoreDocCollector.create( 1 );
694 s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), collector );
695 if ( collector.getTotalHits() == 0 )
696 {
697 w.addDocument( IndexUtils.updateDocument( d, this, false ) );
698 }
699 }
700 else
701 {
702 String deleted = d.get( ArtifactInfo.DELETED );
703
704 if ( deleted != null )
705 {
706
707
708
709 w.deleteDocuments( new Term( ArtifactInfo.UINFO, deleted ) );
710 w.addDocument( d );
711 }
712 }
713 }
714
715 }
716 finally
717 {
718 directoryReader.close();
719 commit();
720 }
721
722 rebuildGroups();
723 Date mergedTimestamp = IndexUtils.getTimestamp( directory );
724
725 if ( getTimestamp() != null && mergedTimestamp != null && mergedTimestamp.after( getTimestamp() ) )
726 {
727
728 updateTimestamp( true, mergedTimestamp );
729 }
730 else
731 {
732 updateTimestamp( true );
733 }
734 optimize();
735 }
736 finally
737 {
738 releaseIndexSearcher( s );
739 }
740 }
741
742 private void closeReaders()
743 throws CorruptIndexException, IOException
744 {
745 if ( searcherManager != null )
746 {
747 searcherManager.close();
748 searcherManager = null;
749 }
750 if ( indexWriter != null )
751 {
752 indexWriter.close();
753 indexWriter = null;
754 }
755 }
756
757 public GavCalculator getGavCalculator()
758 {
759 return gavCalculator;
760 }
761
762 public List<IndexCreator> getIndexCreators()
763 {
764 return Collections.<IndexCreator>unmodifiableList( indexCreators );
765 }
766
767
768
769 public synchronized void rebuildGroups()
770 throws IOException
771 {
772 final IndexSearcher is = acquireIndexSearcher();
773 try
774 {
775 final IndexReader r = is.getIndexReader();
776
777 Set<String> rootGroups = new LinkedHashSet<String>();
778 Set<String> allGroups = new LinkedHashSet<String>();
779
780 int numDocs = r.maxDoc();
781 Bits liveDocs = MultiFields.getLiveDocs( r );
782
783 for ( int i = 0; i < numDocs; i++ )
784 {
785 if ( liveDocs != null && !liveDocs.get( i ) )
786 {
787 continue;
788 }
789
790 Document d = r.document( i );
791
792 String uinfo = d.get( ArtifactInfo.UINFO );
793
794 if ( uinfo != null )
795 {
796 ArtifactInfo info = IndexUtils.constructArtifactInfo( d, this );
797 rootGroups.add( info.getRootGroup() );
798 allGroups.add( info.getGroupId() );
799 }
800 }
801
802 setRootGroups( rootGroups );
803 setAllGroups( allGroups );
804
805 optimize();
806 }
807 finally
808 {
809 releaseIndexSearcher( is );
810 }
811 }
812
813 public Set<String> getAllGroups()
814 throws IOException
815 {
816 return getGroups( ArtifactInfo.ALL_GROUPS, ArtifactInfo.ALL_GROUPS_VALUE, ArtifactInfo.ALL_GROUPS_LIST );
817 }
818
819 public synchronized void setAllGroups( Collection<String> groups )
820 throws IOException
821 {
822 setGroups( groups, ArtifactInfo.ALL_GROUPS, ArtifactInfo.ALL_GROUPS_VALUE, ArtifactInfo.ALL_GROUPS_LIST );
823 commit();
824 }
825
826 public Set<String> getRootGroups()
827 throws IOException
828 {
829 return getGroups( ArtifactInfo.ROOT_GROUPS, ArtifactInfo.ROOT_GROUPS_VALUE, ArtifactInfo.ROOT_GROUPS_LIST );
830 }
831
832 public synchronized void setRootGroups( Collection<String> groups )
833 throws IOException
834 {
835 setGroups( groups, ArtifactInfo.ROOT_GROUPS, ArtifactInfo.ROOT_GROUPS_VALUE, ArtifactInfo.ROOT_GROUPS_LIST );
836 commit();
837 }
838
839 protected Set<String> getGroups( String field, String filedValue, String listField )
840 throws IOException, CorruptIndexException
841 {
842 final TopScoreDocCollector collector = TopScoreDocCollector.create( 1 );
843 final IndexSearcher indexSearcher = acquireIndexSearcher();
844 try
845 {
846 indexSearcher.search( new TermQuery( new Term( field, filedValue ) ), collector );
847 TopDocs topDocs = collector.topDocs();
848 Set<String> groups = new LinkedHashSet<String>( Math.max( 10, topDocs.totalHits ) );
849 if ( topDocs.totalHits > 0 )
850 {
851 Document doc = indexSearcher.doc( topDocs.scoreDocs[0].doc );
852 String groupList = doc.get( listField );
853 if ( groupList != null )
854 {
855 groups.addAll( Arrays.asList( groupList.split( "\\|" ) ) );
856 }
857 }
858 return groups;
859 }
860 finally
861 {
862 releaseIndexSearcher( indexSearcher );
863 }
864 }
865
866 protected void setGroups( Collection<String> groups, String groupField, String groupFieldValue,
867 String groupListField )
868 throws IOException, CorruptIndexException
869 {
870 final IndexWriter w = getIndexWriter();
871 w.updateDocument( new Term( groupField, groupFieldValue ),
872 createGroupsDocument( groups, groupField, groupFieldValue, groupListField ) );
873 }
874
875 protected Document createGroupsDocument( Collection<String> groups, String field, String fieldValue,
876 String listField )
877 {
878 final Document groupDoc = new Document();
879 groupDoc.add( new Field( field,
880 fieldValue, Field.Store.YES, Field.Index.NOT_ANALYZED ) );
881 groupDoc.add( new Field( listField,
882 ArtifactInfo.lst2str( groups ), Field.Store.YES, Field.Index.NO ) );
883 return groupDoc;
884 }
885
886 @Override
887 public String toString()
888 {
889 return id + " : " + timestamp;
890 }
891
892 private static void unlockForcibly( final TrackingLockFactory lockFactory, final Directory dir )
893 throws IOException
894 {
895
896
897
898
899
900 if ( lockFactory != null )
901 {
902 final Set<? extends Lock> emittedLocks = lockFactory.getEmittedLocks( IndexWriter.WRITE_LOCK_NAME );
903 for ( Lock emittedLock : emittedLocks )
904 {
905 emittedLock.close();
906 }
907 }
908 if ( dir instanceof FSDirectory )
909 {
910 final FSDirectory fsdir = (FSDirectory) dir;
911 final Path dirPath = fsdir.getDirectory();
912 if ( Files.isDirectory( dirPath ) )
913 {
914 Path lockPath = dirPath.resolve( IndexWriter.WRITE_LOCK_NAME );
915 try
916 {
917 lockPath = lockPath.toRealPath();
918 }
919 catch ( IOException ioe )
920 {
921
922 return;
923 }
924 try ( final FileChannel fc =
925 FileChannel.open( lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE ) )
926 {
927 final FileLock lck = fc.tryLock();
928 if ( lck == null )
929 {
930
931 throw new LockObtainFailedException( "Lock held by another process: " + lockPath );
932 }
933 else
934 {
935
936 lck.close();
937 }
938 }
939 Files.delete( lockPath );
940 }
941 }
942 }
943 }