View Javadoc
1   package org.apache.maven.indexer.examples.indexing;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
23  import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
24  import org.apache.lucene.queryparser.classic.ParseException;
25  import org.apache.lucene.search.BooleanQuery;
26  import org.apache.lucene.search.Query;
27  import org.apache.maven.index.ArtifactContext;
28  import org.apache.maven.index.ArtifactInfo;
29  import org.apache.maven.index.ArtifactScanningListener;
30  import org.apache.maven.index.FlatSearchRequest;
31  import org.apache.maven.index.FlatSearchResponse;
32  import org.apache.maven.index.Indexer;
33  import org.apache.maven.index.MAVEN;
34  import org.apache.maven.index.Scanner;
35  import org.apache.maven.index.ScanningRequest;
36  import org.apache.maven.index.ScanningResult;
37  import org.apache.maven.index.context.IndexCreator;
38  import org.apache.maven.index.context.IndexingContext;
39  import org.apache.maven.index.expr.SourcedSearchExpression;
40  import org.slf4j.Logger;
41  import org.slf4j.LoggerFactory;
42  
43  import java.io.File;
44  import java.io.IOException;
45  import java.util.ArrayList;
46  import java.util.Collection;
47  import java.util.List;
48  import java.util.Set;
49  
50  import static java.util.Arrays.asList;
51  import static org.apache.lucene.search.BooleanClause.Occur.MUST;
52  
53  /**
54   * This class provides means to index and search for artifacts in a repository on the file system.
55   *
56   * @author mtodorov
57   */
58  public class RepositoryIndexer
59  {
60  
61      private static final Logger LOGGER = LoggerFactory.getLogger( RepositoryIndexer.class );
62  
63      private static final String[] LUCENE_FIELDS = new String[]{"g", "a", "v", "p", "c" };
64  
65      private static final WhitespaceAnalyzer LUCENE_ANALYZER = new WhitespaceAnalyzer( );
66  
67      private Indexer indexer;
68  
69      private Scanner scanner;
70  
71      private List<IndexCreator> indexers;
72  
73      private IndexingContext indexingContext;
74  
75      private String repositoryId;
76  
77      private File repositoryBasedir;
78  
79      private File indexDir;
80  
81  
82      public RepositoryIndexer()
83      {
84          // no op
85      }
86  
87      public void close()
88          throws IOException
89      {
90          indexer.closeIndexingContext( indexingContext, false );
91      }
92  
93      public void close( boolean deleteFiles )
94          throws IOException
95      {
96          indexingContext.close( deleteFiles );
97      }
98  
99      public void delete( final Collection<ArtifactInfo> artifacts )
100         throws IOException
101     {
102         final List<ArtifactContext> delete = new ArrayList<>();
103         for ( final ArtifactInfo artifact : artifacts )
104         {
105             LOGGER.debug( "Deleting artifact: {}; ctx id: {}; idx dir: {}",
106                           new String[]{ artifact.toString(), indexingContext.getId(),
107                               indexingContext.getIndexDirectory().toString() } );
108 
109             delete.add( new ArtifactContext( null, null, null, artifact, null ) );
110         }
111 
112         getIndexer().deleteArtifactsFromIndex( delete, indexingContext );
113     }
114 
115     public Set<ArtifactInfo> search( final String groupId, final String artifactId, final String version,
116                                      final String packaging, final String classifier )
117         throws IOException
118     {
119         final BooleanQuery query = new BooleanQuery();
120 
121         if ( groupId != null )
122         {
123             query.add( getIndexer().constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression( groupId ) ), MUST );
124         }
125 
126         if ( artifactId != null )
127         {
128             query.add( getIndexer().constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression( artifactId ) ),
129                        MUST );
130         }
131 
132         if ( version != null )
133         {
134             query.add( getIndexer().constructQuery( MAVEN.VERSION, new SourcedSearchExpression( version ) ), MUST );
135         }
136 
137         if ( packaging != null )
138         {
139             query.add( getIndexer().constructQuery( MAVEN.PACKAGING, new SourcedSearchExpression( packaging ) ), MUST );
140         }
141         else
142         {
143             // Fallback to jar
144             query.add( getIndexer().constructQuery( MAVEN.PACKAGING, new SourcedSearchExpression( "jar" ) ), MUST );
145         }
146 
147         if ( classifier != null )
148         {
149             query.add( getIndexer().constructQuery( MAVEN.CLASSIFIER, new SourcedSearchExpression( classifier ) ),
150                        MUST );
151         }
152 
153         LOGGER.debug( "Executing search query: {}; ctx id: {}; idx dir: {}",
154                       new String[]{ query.toString(), indexingContext.getId(),
155                           indexingContext.getIndexDirectory().toString() } );
156 
157         final FlatSearchResponse response = getIndexer().searchFlat( new FlatSearchRequest( query, indexingContext ) );
158 
159         LOGGER.info( "Hit count: {}", response.getReturnedHitsCount() );
160 
161         final Set<ArtifactInfo> results = response.getResults();
162         if ( LOGGER.isDebugEnabled() )
163         {
164             for ( final ArtifactInfo result : results )
165             {
166                 LOGGER.debug( "Found artifact: {}", result.toString() );
167             }
168         }
169 
170         return results;
171     }
172 
173     public Set<ArtifactInfo> search( final String queryText )
174         throws ParseException, IOException
175     {
176         final Query query = new MultiFieldQueryParser( LUCENE_FIELDS, LUCENE_ANALYZER ).parse( queryText );
177 
178         LOGGER.debug( "Executing search query: {}; ctx id: {}; idx dir: {}",
179                       new String[]{ query.toString(), indexingContext.getId(),
180                           indexingContext.getIndexDirectory().toString() } );
181 
182         final FlatSearchResponse response = getIndexer().searchFlat( new FlatSearchRequest( query, indexingContext ) );
183 
184         final Set<ArtifactInfo> results = response.getResults();
185         if ( LOGGER.isDebugEnabled() )
186         {
187             LOGGER.debug( "Hit count: {}", response.getReturnedHitsCount() );
188 
189             for ( final ArtifactInfo result : results )
190             {
191                 LOGGER.debug( "Found artifact: {}; uinfo: {}", result.toString(), result.getUinfo() );
192             }
193         }
194 
195         return results;
196     }
197 
198     public Set<ArtifactInfo> searchBySHA1( final String checksum )
199         throws IOException
200     {
201         final BooleanQuery query = new BooleanQuery();
202         query.add( getIndexer().constructQuery( MAVEN.SHA1, new SourcedSearchExpression( checksum ) ), MUST );
203 
204         LOGGER.debug( "Executing search query: {}; ctx id: {}; idx dir: {}",
205                       new String[]{ query.toString(), indexingContext.getId(),
206                           indexingContext.getIndexDirectory().toString() } );
207 
208         final FlatSearchResponse response = getIndexer().searchFlat( new FlatSearchRequest( query, indexingContext ) );
209 
210         LOGGER.info( "Hit count: {}", response.getReturnedHitsCount() );
211 
212         final Set<ArtifactInfo> results = response.getResults();
213         if ( LOGGER.isDebugEnabled() )
214         {
215             for ( final ArtifactInfo result : results )
216             {
217                 LOGGER.debug( "Found artifact: {}", result.toString() );
218             }
219         }
220 
221         return results;
222     }
223 
224     public int index( final File startingPath )
225     {
226         final ScanningResult scan = getScanner().scan(
227             new ScanningRequest( indexingContext, new ReindexArtifactScanningListener(),
228                                  startingPath == null ? "." : startingPath.getPath() ) );
229         return scan.getTotalFiles();
230     }
231 
232     public void addArtifactToIndex( final File artifactFile, final ArtifactInfo artifactInfo )
233         throws IOException
234     {
235         getIndexer().addArtifactsToIndex( asList( new ArtifactContext( null, artifactFile, null, artifactInfo, null ) ),
236                                           indexingContext );
237     }
238 
239     public void addArtifactToIndex( String repository, File artifactFile, String groupId, String artifactId,
240                                     String version, String extension, String classifier )
241         throws IOException
242     {
243         ArtifactInfo artifactInfo = new ArtifactInfo( repository, groupId, artifactId, version, classifier, extension );
244         if ( extension != null )
245         {
246             artifactInfo.setFieldValue( MAVEN.EXTENSION, extension );
247         }
248 
249         LOGGER.debug( "Adding artifact: {}; repo: {}; type: {}",
250                       new String[]{ artifactInfo.getUinfo(), repository, extension } );
251 
252         getIndexer().addArtifactsToIndex(
253             asList( new ArtifactContext( null, artifactFile, null, artifactInfo, artifactInfo.calculateGav() ) ),
254             indexingContext );
255     }
256 
257     private class ReindexArtifactScanningListener
258         implements ArtifactScanningListener
259     {
260 
261         int totalFiles = 0;
262 
263         private IndexingContext context;
264 
265         @Override
266         public void scanningStarted( final IndexingContext context )
267         {
268             this.context = context;
269         }
270 
271         @Override
272         public void scanningFinished( final IndexingContext context, final ScanningResult result )
273         {
274             result.setTotalFiles( totalFiles );
275             LOGGER.debug( "Scanning finished; total files: {}; has exception: {}", result.getTotalFiles(),
276                           result.hasExceptions() );
277         }
278 
279         @Override
280         public void artifactError( final ArtifactContext ac, final Exception ex )
281         {
282             LOGGER.error( "Artifact error!", ex );
283         }
284 
285         @Override
286         public void artifactDiscovered( final ArtifactContext ac )
287         {
288             try
289             {
290                 LOGGER.debug( "Adding artifact gav: {}; ctx id: {}; idx dir: {}",
291                               new String[]{ ac.getGav().toString(), context.getId(),
292                                   context.getIndexDirectory().toString() } );
293 
294                 getIndexer().addArtifactsToIndex( asList( ac ), context );
295                 totalFiles++;
296             }
297             catch ( IOException ex )
298             {
299                 LOGGER.error( "Artifact index error", ex );
300             }
301         }
302     }
303 
304     public Indexer getIndexer()
305     {
306         return indexer;
307     }
308 
309     public void setIndexer( Indexer indexer )
310     {
311         this.indexer = indexer;
312     }
313 
314     public Scanner getScanner()
315     {
316         return scanner;
317     }
318 
319     public void setScanner( Scanner scanner )
320     {
321         this.scanner = scanner;
322     }
323 
324     public List<IndexCreator> getIndexers()
325     {
326         return indexers;
327     }
328 
329     public void setIndexers( List<IndexCreator> indexers )
330     {
331         this.indexers = indexers;
332     }
333 
334     public IndexingContext getIndexingContext()
335     {
336         return indexingContext;
337     }
338 
339     public void setIndexingContext( IndexingContext indexingContext )
340     {
341         this.indexingContext = indexingContext;
342     }
343 
344     public String getRepositoryId()
345     {
346         return repositoryId;
347     }
348 
349     public void setRepositoryId( String repositoryId )
350     {
351         this.repositoryId = repositoryId;
352     }
353 
354     public File getRepositoryBasedir()
355     {
356         return repositoryBasedir;
357     }
358 
359     public void setRepositoryBasedir( File repositoryBasedir )
360     {
361         this.repositoryBasedir = repositoryBasedir;
362     }
363 
364     public File getIndexDir()
365     {
366         return indexDir;
367     }
368 
369     public void setIndexDir( File indexDir )
370     {
371         this.indexDir = indexDir;
372     }
373 
374 }