View Javadoc
1   package org.apache.maven.index;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0    
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.io.FileInputStream;
24  import java.util.HashSet;
25  import java.util.Properties;
26  import java.util.Set;
27  import junit.framework.Assert;
28  
29  import org.apache.lucene.search.IndexSearcher;
30  import org.apache.maven.index.context.IndexingContext;
31  import org.apache.maven.index.packer.IndexPacker;
32  import org.apache.maven.index.packer.IndexPackingRequest;
33  import org.codehaus.plexus.util.FileUtils;
34  
35  //FIXME - hardcoded assumptions in test that break with lucene 4, or bugs?
36  //@Ignore("Segment merge may work differently in Lucene 4")
37  public class Nexus1911IncrementalTest
38      extends AbstractIndexCreatorHelper
39  {
40      NexusIndexer indexer;
41  
42      IndexingContext context;
43  
44      IndexingContext reindexedContext;
45  
46      IndexPacker packer;
47  
48      File indexDir;
49  
50      File indexPackDir;
51  
52      File reposTargetDir;
53  
54      @Override
55      protected void setUp()
56          throws Exception
57      {
58          super.setUp();
59  
60          indexer = lookup( NexusIndexer.class );
61          packer = lookup( IndexPacker.class );
62  
63          indexDir = super.getDirectory( "index/nexus-1911" );
64          indexPackDir = indexDir; //super.getDirectory( "index/nexus-1911-pack" );
65  
66          File reposSrcDir = new File( getBasedir(), "src/test/nexus-1911" );
67          this.reposTargetDir = super.getDirectory( "repos/nexus-1911" );
68  
69          FileUtils.copyDirectoryStructure( reposSrcDir, reposTargetDir );
70  
71          File repo = new File( reposTargetDir, "repo" );
72          repo.mkdirs();
73          reindexedContext =
74              context = indexer.addIndexingContext( "test", "test", repo, indexDir, null, null, DEFAULT_CREATORS );
75          indexer.scan( context );
76      }
77  
78      @Override
79      protected void tearDown()
80          throws Exception
81      {
82          indexer.removeIndexingContext( context, true );
83          super.deleteDirectory( this.reposTargetDir );
84          super.deleteDirectory( this.indexDir );
85          super.deleteDirectory( this.indexPackDir );
86          super.tearDown();
87      }
88  
89      public void testNoIncremental()
90          throws Exception
91      {
92          final IndexSearcher indexSearcher = context.acquireIndexSearcher();
93          try
94          {
95              IndexPackingRequest request =
96                  new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
97              request.setCreateIncrementalChunks( true );
98              packer.packIndex( request );
99          }
100         finally
101         {
102             context.releaseIndexSearcher( indexSearcher );
103         }
104 
105         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
106         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
107 
108         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
109         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
110         Assert.assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
111         Assert.assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
112 
113         Assert.assertNotNull( props );
114 
115         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ) );
116         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ) );
117         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ) );
118         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
119         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
120         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "0" );
121         Assert.assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
122     }
123 
124     public void test1Incremental()
125         throws Exception
126     {
127         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
128         try
129         {
130             IndexPackingRequest request =
131                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
132             request.setCreateIncrementalChunks( true );
133             packer.packIndex( request );
134         }
135         finally
136         {
137             context.releaseIndexSearcher( indexSearcher );
138         }
139 
140         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-1" ), IndexPackingRequest.MAX_CHUNKS );
141 
142         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
143         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
144 
145         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
146         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
147         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
148         Assert.assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
149 
150         Assert.assertNotNull( props );
151 
152         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ), "1" );
153         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ) );
154         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ) );
155         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
156         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
157         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "1" );
158         Assert.assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
159     }
160 
161     public void test2Incremental()
162         throws Exception
163     {
164         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
165         try
166         {
167             IndexPackingRequest request =
168                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
169             request.setCreateIncrementalChunks( true );
170             packer.packIndex( request );
171         }
172         finally
173         {
174             context.releaseIndexSearcher( indexSearcher );
175         }
176 
177         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-1" ), IndexPackingRequest.MAX_CHUNKS );
178         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-2" ), IndexPackingRequest.MAX_CHUNKS );
179 
180         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
181         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
182 
183         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
184         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
185         //1 is missing with updated Lucene 4 implementation
186 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
187         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
188         Assert.assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".3.gz" ) );
189 
190         Assert.assertNotNull( props );
191 
192         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ), "2" );
193         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ), "1" );
194         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ) );
195         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
196         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
197         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "2" );
198         Assert.assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
199     }
200 
201     public void test3Incremental()
202         throws Exception
203     {
204         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
205         try
206         {
207             IndexPackingRequest request =
208                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
209             request.setCreateIncrementalChunks( true );
210             packer.packIndex( request );
211         }
212         finally
213         {
214             context.releaseIndexSearcher( indexSearcher );
215         }
216 
217         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-1" ), IndexPackingRequest.MAX_CHUNKS );
218         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-2" ), IndexPackingRequest.MAX_CHUNKS );
219         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-3" ), IndexPackingRequest.MAX_CHUNKS );
220 
221         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
222         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
223 
224         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
225         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
226         //1,2 are missing with updated Lucene 4 implementation
227 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
228 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
229         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".3.gz" ) );
230 
231         Assert.assertNotNull( props );
232 
233         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ), "3" );
234         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ), "2" );
235         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ), "1" );
236         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
237         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
238         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "3" );
239         Assert.assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
240     }
241 
242     public void testMaxChunks()
243         throws Exception
244     {
245         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
246         try
247         {
248             IndexPackingRequest request =
249                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
250             request.setCreateIncrementalChunks( true );
251             request.setMaxIndexChunks( 3 );
252             packer.packIndex( request );
253         }
254         finally
255         {
256             context.releaseIndexSearcher( indexSearcher );
257         }
258 
259         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-1" ), 3 );
260         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-2" ), 3 );
261         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-3" ), 3 );
262         copyRepoContentsAndReindex( new File( getBasedir(), "src/test/nexus-1911/repo-inc-4" ), 3 );
263 
264         Set<String> filenames = getFilenamesFromFiles( indexPackDir.listFiles() );
265         Properties props = getPropertiesFromFiles( indexPackDir.listFiles() );
266         
267         System.out.println( filenames );
268         
269         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
270         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".properties" ) );
271         Assert.assertFalse( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
272         
273     //2,3 are missing with updated Lucene 4 implementation
274 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
275 //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".3.gz" ) );
276         Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".4.gz" ) );
277 
278         Assert.assertNotNull( props );
279 
280         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "0" ), "4" );
281         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "1" ), "3" );
282         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "2" ), "2" );
283         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "3" ) );
284         Assert.assertNull( props.getProperty( IndexingContext.INDEX_CHUNK_PREFIX + "4" ) );
285         Assert.assertEquals( props.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ), "4" );
286         Assert.assertNotNull( props.getProperty( IndexingContext.INDEX_CHAIN_ID ) );
287     }
288 
289     private void copyRepoContentsAndReindex( File src, int maxIndexChunks )
290         throws Exception
291     {
292         File reposTargetDir = new File( getBasedir(), "target/repos/nexus-1911/repo" );
293 
294         FileUtils.copyDirectoryStructure( src, reposTargetDir );
295 
296         // this was ALWAYS broken, if incremental reindex wanted, this has to be TRUE!!!
297         // TODO: fix this!
298         indexer.scan( reindexedContext, false );
299 
300         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
301         try
302         {
303             IndexPackingRequest request =
304                 new IndexPackingRequest( context, indexSearcher.getIndexReader(), indexPackDir );
305             request.setCreateIncrementalChunks( true );
306             request.setMaxIndexChunks( maxIndexChunks );
307             packer.packIndex( request );
308         }
309         finally
310         {
311             context.releaseIndexSearcher( indexSearcher );
312         }
313     }
314 
315     private Set<String> getFilenamesFromFiles( File[] files )
316     {
317         Set<String> filenames = new HashSet<>();
318 
319         for ( File file : files )
320         {
321             filenames.add( file.getName() );
322         }
323 
324         return filenames;
325     }
326 
327     private Properties getPropertiesFromFiles( File[] files )
328         throws Exception
329     {
330         Properties props = new Properties();
331         File propertyFile = null;
332 
333         for ( File file : files )
334         {
335             if ( ( IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ).equalsIgnoreCase( file.getName() ) )
336             {
337                 propertyFile = file;
338                 break;
339             }
340         }
341 
342         try ( FileInputStream fis = new FileInputStream( propertyFile ) )
343         {
344             props.load( fis );
345         }
346 
347         return props;
348     }
349 }