View Javadoc
1   package org.apache.maven.index;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0    
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.io.FileInputStream;
24  import java.util.HashSet;
25  import java.util.Properties;
26  import java.util.Set;
27  
28  import org.apache.lucene.search.IndexSearcher;
29  import org.apache.maven.index.context.IndexingContext;
30  import org.apache.maven.index.packer.IndexPacker;
31  import org.apache.maven.index.packer.IndexPackingRequest;
32  import org.codehaus.plexus.util.FileUtils;
33  import org.junit.Test;
34  
35  import static org.junit.Assert.assertEquals;
36  import static org.junit.Assert.assertFalse;
37  import static org.junit.Assert.assertNotNull;
38  import static org.junit.Assert.assertNull;
39  import static org.junit.Assert.assertTrue;
40  
41  //FIXME - hardcoded assumptions in test that break with lucene 4, or bugs?
42  //@Ignore("Segment merge may work differently in Lucene 4")
43  public class Nexus1911IncrementalTest
44      extends AbstractIndexCreatorHelper
45  {
46      NexusIndexer indexer;
47  
48      IndexingContext context;
49  
50      IndexingContext reindexedContext;
51  
52      IndexPacker packer;
53  
54      File indexDir;
55  
56      File indexPackDir;
57  
58      File reposTargetDir;
59  
60      @Override
61      public void setUp()
62          throws Exception
63      {
64          super.setUp();
65  
66          indexer = lookup( NexusIndexer.class );
67          packer = lookup( IndexPacker.class );
68  
69          indexDir = super.getDirectory( "index/nexus-1911" );
70          indexPackDir = indexDir; //super.getDirectory( "index/nexus-1911-pack" );
71  
72          File reposSrcDir = new File( getBasedir(), "src/test/nexus-1911" );
73          this.reposTargetDir = super.getDirectory( "repos/nexus-1911" );
74  
75          FileUtils.copyDirectoryStructure( reposSrcDir, reposTargetDir );
76  
77          File repo = new File( reposTargetDir, "repo" );
78          repo.mkdirs();
79          reindexedContext =
80              context = indexer.addIndexingContext( "test", "test", repo, indexDir, null, null, DEFAULT_CREATORS );
81          indexer.scan( context );
82      }
83  
84      @Override
85      public void tearDown()
86          throws Exception
87      {
88          indexer.removeIndexingContext( context, true );
89          super.deleteDirectory( this.reposTargetDir );
90          super.deleteDirectory( this.indexDir );
91          super.deleteDirectory( this.indexPackDir );
92          super.tearDown();
93      }
94  
95      @Test
96      public void testNoIncremental()
97          throws Exception
98      {
99          final IndexSearcher indexSearcher = context.acquireIndexSearcher();
100         try
101         {
102             IndexPackingRequest request =
103                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
104             request.setCreateIncrementalChunks( true );
105             packer.packIndex( request );
106         }
107         finally
108         {
109             context.releaseIndexSearcher( indexSearcher );
110         }
111 
112         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
113         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
114 
115         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
116         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
117         assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
118         assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
119 
120         assertNotNull( props );
121 
122         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ) );
123         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ) );
124         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ) );
125         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
126         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
127         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "0" );
128         assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
129     }
130 
131     @Test
132     public void test1Incremental()
133         throws Exception
134     {
135         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
136         try
137         {
138             IndexPackingRequest request =
139                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
140             request.setCreateIncrementalChunks( true );
141             packer.packIndex( request );
142         }
143         finally
144         {
145             context.releaseIndexSearcher( indexSearcher );
146         }
147 
148         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-1" ), IndexPackingRequest.MAX_CHUNKS );
149 
150         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
151         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
152 
153         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
154         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
155         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
156         assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
157 
158         assertNotNull( props );
159 
160         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ), "1" );
161         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ) );
162         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ) );
163         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
164         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
165         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "1" );
166         assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
167     }
168 
169     @Test
170     public void test2Incremental()
171         throws Exception
172     {
173         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
174         try
175         {
176             IndexPackingRequest request =
177                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
178             request.setCreateIncrementalChunks( true );
179             packer.packIndex( request );
180         }
181         finally
182         {
183             context.releaseIndexSearcher( indexSearcher );
184         }
185 
186         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-1" ), IndexPackingRequest.MAX_CHUNKS );
187         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-2" ), IndexPackingRequest.MAX_CHUNKS );
188 
189         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
190         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
191 
192         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
193         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
194         //1 is missing with updated Lucene 4 implementation
195 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
196         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
197         assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".3.gz" ) );
198 
199         assertNotNull( props );
200 
201         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ), "2" );
202         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ), "1" );
203         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ) );
204         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
205         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
206         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "2" );
207         assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
208     }
209 
210     @Test
211     public void test3Incremental()
212         throws Exception
213     {
214         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
215         try
216         {
217             IndexPackingRequest request =
218                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
219             request.setCreateIncrementalChunks( true );
220             packer.packIndex( request );
221         }
222         finally
223         {
224             context.releaseIndexSearcher( indexSearcher );
225         }
226 
227         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-1" ), IndexPackingRequest.MAX_CHUNKS );
228         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-2" ), IndexPackingRequest.MAX_CHUNKS );
229         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-3" ), IndexPackingRequest.MAX_CHUNKS );
230 
231         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
232         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
233 
234         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
235         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
236         //1,2 are missing with updated Lucene 4 implementation
237 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
238 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
239         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".3.gz" ) );
240 
241         assertNotNull( props );
242 
243         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ), "3" );
244         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ), "2" );
245         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ), "1" );
246         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
247         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
248         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "3" );
249         assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
250     }
251 
252     @Test
253     public void testMaxChunks()
254         throws Exception
255     {
256         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
257         try
258         {
259             IndexPackingRequest request =
260                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
261             request.setCreateIncrementalChunks( true );
262             request.setMaxIndexChunks( 3 );
263             packer.packIndex( request );
264         }
265         finally
266         {
267             context.releaseIndexSearcher( indexSearcher );
268         }
269 
270         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-1" ), 3 );
271         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-2" ), 3 );
272         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-3" ), 3 );
273         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-4" ), 3 );
274 
275         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
276         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
277         
278         System.out.println( filenames );
279         
280         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
281         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
282         assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
283         
284     //2,3 are missing with updated Lucene 4 implementation
285 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
286 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".3.gz" ) );
287         assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".4.gz" ) );
288 
289         assertNotNull( props );
290 
291         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ), "4" );
292         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ), "3" );
293         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ), "2" );
294         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
295         assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
296         assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "4" );
297         assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
298     }
299 
300     private void copyRepoContentsAndReindex( File src, int maxIndexChunks )
301         throws Exception
302     {
303         File reposTargetDir = new File( getBasedir(), "target/repos/nexus-1911/repo" );
304 
305         FileUtils.copyDirectoryStructure( src, reposTargetDir );
306 
307         // this was ALWAYS broken, if incremental reindex wanted, this has to be TRUE!!!
308         // TODO: fix this!
309         indexer.scan( reindexedContext, false );
310 
311         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
312         try
313         {
314             IndexPackingRequest request =
315                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
316             request.setCreateIncrementalChunks( true );
317             request.setMaxIndexChunks( maxIndexChunks );
318             packer.packIndex( request );
319         }
320         finally
321         {
322             context.releaseIndexSearcher( indexSearcher );
323         }
324     }
325 
326     private Set<String> getFilenamesFromFiles( File[] files )
327     {
328         Set<String> filenames = new HashSet<>();
329 
330         for ( File file : files )
331         {
332             filenames.add( file.getName() );
333         }
334 
335         return filenames;
336     }
337 
338     private Properties getPropertiesFromFiles( File[] files )
339         throws Exception
340     {
341         Properties props = new Properties();
342         File propertyFile = null;
343 
344         for ( File file : files )
345         {
346             if ( ( IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ).equalsIgnoreCase( file.getName() ) )
347             {
348                 propertyFile = file;
349                 break;
350             }
351         }
352 
353         try ( FileInputStream fis = new FileInputStream( propertyFile ) )
354         {
355             props.load( fis );
356         }
357 
358         return props;
359     }
360 }