View Javadoc
1   package org.apache.maven.index.updater;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0    
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.File;
23  import java.io.FileOutputStream;
24  import java.io.IOException;
25  import java.io.InputStream;
26  
27  import org.apache.lucene.index.Term;
28  import org.apache.lucene.search.TermQuery;
29  import org.apache.maven.index.ArtifactInfo;
30  import org.apache.maven.index.FlatSearchRequest;
31  import org.apache.maven.index.FlatSearchResponse;
32  import org.apache.maven.index.context.IndexingContext;
33  import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
34  import org.apache.maven.index.fs.Locker;
35  import org.codehaus.plexus.util.FileUtils;
36  import org.codehaus.plexus.util.IOUtil;
37  
38  public class LocalIndexCacheTest
39      extends AbstractIndexUpdaterTest
40  {
41      private File remoteRepo;
42  
43      private File localCacheDir;
44  
45      private File indexDir;
46  
47      private IndexingContext tempContext;
48  
49      @Override
50      protected void setUp()
51          throws Exception
52      {
53          super.setUp();
54  
55          remoteRepo = new File( "target/localcache/remoterepo" ).getCanonicalFile();
56          FileUtils.deleteDirectory( remoteRepo );
57          remoteRepo.mkdirs();
58  
59          localCacheDir = new File( "target/localcache/cache" ).getCanonicalFile();
60          FileUtils.deleteDirectory( localCacheDir );
61          localCacheDir.mkdirs();
62  
63          indexDir = new File( "target/localcache/index" ).getCanonicalFile();
64          FileUtils.deleteDirectory( indexDir );
65          indexDir.mkdirs();
66      }
67  
68      @Override
69      protected void tearDown()
70          throws Exception
71      {
72          removeTempContext();
73  
74          super.tearDown();
75      }
76  
77      private IndexingContext getNewTempContext()
78          throws IOException, UnsupportedExistingLuceneIndexException
79      {
80          removeTempContext();
81  
82          tempContext =
83              indexer.addIndexingContext( repositoryId + "temp", repositoryId, repoDir, indexDir, repositoryUrl, null,
84                  MIN_CREATORS );
85  
86          return tempContext;
87      }
88  
89      private void removeTempContext()
90          throws IOException
91      {
92          if ( tempContext != null )
93          {
94              indexer.removeIndexingContext( tempContext, true );
95              tempContext = null;
96              FileUtils.cleanDirectory( indexDir );
97          }
98      }
99  
100     public void testBasic()
101         throws Exception
102     {
103         // create initial remote repo index
104         indexer.addArtifactToIndex( createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.2", null ),
105             context );
106         packIndex( remoteRepo, context );
107 
108         //
109         TrackingFetcher fetcher;
110         IndexUpdateRequest updateRequest;
111         IndexingContext testContext;
112 
113         // initial index download (expected: full index download)
114         testContext = getNewTempContext();
115         fetcher = new TrackingFetcher( remoteRepo );
116         updateRequest = new IndexUpdateRequest( testContext, fetcher );
117         updateRequest.setLocalIndexCacheDir( localCacheDir );
118         updater.fetchAndUpdateIndex( updateRequest );
119         assertEquals( 2, fetcher.getRetrievedResources().size() );
120         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.gz" ).exists() );
121         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.properties" ).exists() );
122         assertGroupCount( 1, "commons-lang", testContext );
123 
124         // update the same index (expected: no index download)
125         fetcher = new TrackingFetcher( remoteRepo );
126         updateRequest = new IndexUpdateRequest( testContext, fetcher );
127         updateRequest.setLocalIndexCacheDir( localCacheDir );
128         updater.fetchAndUpdateIndex( updateRequest );
129         assertEquals( 1, fetcher.getRetrievedResources().size() );
130         assertEquals( "nexus-maven-repository-index.properties", fetcher.getRetrievedResources().get( 0 ) );
131         assertGroupCount( 1, "commons-lang", testContext );
132 
133         // nuke index but keep the cache (expected: no index download)
134         testContext = getNewTempContext();
135         fetcher = new TrackingFetcher( remoteRepo );
136         updateRequest = new IndexUpdateRequest( testContext, fetcher );
137         updateRequest.setLocalIndexCacheDir( localCacheDir );
138         updater.fetchAndUpdateIndex( updateRequest );
139         assertEquals( 1, fetcher.getRetrievedResources().size() );
140         assertEquals( "nexus-maven-repository-index.properties", fetcher.getRetrievedResources().get( 0 ) );
141         assertGroupCount( 1, "commons-lang", testContext );
142 
143         // incremental remote update
144         indexer.addArtifactToIndex( createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.3", null ),
145             context );
146         packIndex( remoteRepo, context );
147 
148         // update via cache (expected: incremental chunk download)
149         fetcher = new TrackingFetcher( remoteRepo );
150         updateRequest = new IndexUpdateRequest( testContext, fetcher );
151         updateRequest.setLocalIndexCacheDir( localCacheDir );
152         updater.fetchAndUpdateIndex( updateRequest );
153         assertEquals( 2, fetcher.getRetrievedResources().size() );
154         assertEquals( "nexus-maven-repository-index.properties", fetcher.getRetrievedResources().get( 0 ) );
155         assertEquals( "nexus-maven-repository-index.1.gz", fetcher.getRetrievedResources().get( 1 ) );
156         assertGroupCount( 2, "commons-lang", testContext );
157 
158         // nuke index but keep the cache (expected: no index download, index contains both initial and delta chunks)
159         testContext = getNewTempContext();
160         fetcher = new TrackingFetcher( remoteRepo );
161         updateRequest = new IndexUpdateRequest( testContext, fetcher );
162         updateRequest.setLocalIndexCacheDir( localCacheDir );
163         updater.fetchAndUpdateIndex( updateRequest );
164         assertEquals( 1, fetcher.getRetrievedResources().size() );
165         assertEquals( "nexus-maven-repository-index.properties", fetcher.getRetrievedResources().get( 0 ) );
166         assertGroupCount( 2, "commons-lang", testContext );
167 
168         // kill the cache, but keep the index (expected: full index download)
169         // TODO how to assert if merge==false internally?
170         FileUtils.deleteDirectory( localCacheDir );
171         fetcher = new TrackingFetcher( remoteRepo );
172         updateRequest = new IndexUpdateRequest( testContext, fetcher );
173         updateRequest.setLocalIndexCacheDir( localCacheDir );
174         updater.fetchAndUpdateIndex( updateRequest );
175         assertEquals( 2, fetcher.getRetrievedResources().size() );
176         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.gz" ).exists() );
177         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.properties" ).exists() );
178         assertGroupCount( 2, "commons-lang", testContext );
179     }
180 
181     private void assertGroupCount( int expectedCount, String groupId, IndexingContext context )
182         throws IOException
183     {
184         TermQuery query = new TermQuery( new Term( ArtifactInfo.GROUP_ID, groupId ) );
185         FlatSearchResponse response = indexer.searchFlat( new FlatSearchRequest( query, context ) );
186         assertEquals( expectedCount, response.getTotalHits() );
187     }
188 
189     public void testForceIndexDownload()
190         throws Exception
191     {
192         indexer.addArtifactToIndex( createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.2", null ),
193             context );
194         packIndex( remoteRepo, context );
195 
196         //
197         TrackingFetcher fetcher;
198         IndexUpdateRequest updateRequest;
199 
200         // initial index download (expected: no index download)
201         fetcher = new TrackingFetcher( remoteRepo );
202         updateRequest = new IndexUpdateRequest( getNewTempContext(), fetcher );
203         updateRequest.setLocalIndexCacheDir( localCacheDir );
204         updater.fetchAndUpdateIndex( updateRequest );
205 
206         // corrupt local cache
207         try ( FileOutputStream fos = new FileOutputStream( new File( localCacheDir, "nexus-maven-repository-index.gz" ) ) )
208         {
209             IOUtil.copy( "corrupted", fos );
210         }
211 
212         // try download again (it would have failed if force did not update local cache)
213         removeTempContext();
214         fetcher = new TrackingFetcher( remoteRepo );
215         updateRequest = new IndexUpdateRequest( getNewTempContext(), fetcher );
216         updateRequest.setLocalIndexCacheDir( localCacheDir );
217         updateRequest.setForceFullUpdate( true );
218         updater.fetchAndUpdateIndex( updateRequest );
219     }
220 
221     public void testInitialForcedFullDownload()
222         throws Exception
223     {
224         indexer.addArtifactToIndex( createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.2", null ),
225             context );
226         packIndex( remoteRepo, context );
227 
228         //
229         TrackingFetcher fetcher;
230         IndexUpdateRequest updateRequest;
231 
232         // initial forced full index download (expected: successfull download)
233         fetcher = new TrackingFetcher( remoteRepo );
234         updateRequest = new IndexUpdateRequest( getNewTempContext(), fetcher );
235         updateRequest.setLocalIndexCacheDir( localCacheDir );
236         updateRequest.setForceFullUpdate( true );
237         updater.fetchAndUpdateIndex( updateRequest );
238         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.gz" ).exists() );
239         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.properties" ).exists() );
240     }
241 
242     public void testFailedIndexDownload()
243         throws Exception
244     {
245         indexer.addArtifactToIndex( createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.2", null ),
246             context );
247         packIndex( remoteRepo, context );
248 
249         //
250         TrackingFetcher fetcher;
251         IndexUpdateRequest updateRequest;
252 
253         // failed download
254         fetcher = new TrackingFetcher( remoteRepo )
255         {
256             public InputStream retrieve( String name )
257                 throws IOException, java.io.FileNotFoundException
258             {
259                 if ( name.equals( IndexingContext.INDEX_FILE_PREFIX + ".gz" )
260                     || name.equals( IndexingContext.INDEX_FILE_PREFIX + ".zip" ) )
261                 {
262                     throw new IOException();
263                 }
264                 return super.retrieve( name );
265             }
266         };
267         updateRequest = new IndexUpdateRequest( getNewTempContext(), fetcher );
268         updateRequest.setLocalIndexCacheDir( localCacheDir );
269         try
270         {
271             updater.fetchAndUpdateIndex( updateRequest );
272             fail();
273         }
274         catch ( IOException e )
275         {
276             // expected
277         }
278 
279         // try successful download
280         fetcher = new TrackingFetcher( remoteRepo );
281         updateRequest = new IndexUpdateRequest( getNewTempContext(), fetcher );
282         updateRequest.setLocalIndexCacheDir( localCacheDir );
283         updater.fetchAndUpdateIndex( updateRequest );
284         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.gz" ).exists() );
285         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.properties" ).exists() );
286     }
287 
288     public void testCleanCacheDirectory()
289         throws Exception
290     {
291         indexer.addArtifactToIndex( createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.2", null ),
292             context );
293         packIndex( remoteRepo, context );
294 
295         //
296         TrackingFetcher fetcher;
297         IndexUpdateRequest updateRequest;
298 
299         // initial index download (expected: successfull download)
300         fetcher = new TrackingFetcher( remoteRepo );
301         updateRequest = new IndexUpdateRequest( getNewTempContext(), fetcher );
302         updateRequest.setLocalIndexCacheDir( localCacheDir );
303         updater.fetchAndUpdateIndex( updateRequest );
304 
305         // new remote index delta
306         indexer.addArtifactToIndex( createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.3", null ),
307             context );
308         packIndex( remoteRepo, context );
309 
310         // delta index download (expected: successfull download)
311         fetcher = new TrackingFetcher( remoteRepo );
312         updateRequest = new IndexUpdateRequest( getNewTempContext(), fetcher );
313         updateRequest.setLocalIndexCacheDir( localCacheDir );
314         updater.fetchAndUpdateIndex( updateRequest );
315 
316         // sanity check
317         assertTrue( new File( localCacheDir, "nexus-maven-repository-index.1.gz" ).canRead() );
318 
319         // .lock files are expected to be preserved
320         File lockFile = new File( localCacheDir, Locker.LOCK_FILE );
321         try ( FileOutputStream lockFileOutput = new FileOutputStream( lockFile ) )
322         {
323             IOUtil.copy( "", lockFileOutput );
324         }
325         assertTrue( lockFile.canRead() );
326 
327         // all unknown files and directories are expected to be removed
328         File unknownFile = new File( localCacheDir, "unknownFile" );
329         try ( FileOutputStream fileOutputStream = new FileOutputStream( unknownFile ) )
330         {
331             IOUtil.copy( "", fileOutputStream );
332         }
333 
334         File unknownDirectory = new File( localCacheDir, "unknownDirectory" );
335         unknownDirectory.mkdirs();
336         assertTrue( unknownFile.canRead() );
337         assertTrue( unknownDirectory.isDirectory() );
338 
339         // forced full update
340         fetcher = new TrackingFetcher( remoteRepo );
341         updateRequest = new IndexUpdateRequest( getNewTempContext(), fetcher );
342         updateRequest.setLocalIndexCacheDir( localCacheDir );
343         updateRequest.setForceFullUpdate( true );
344         updater.fetchAndUpdateIndex( updateRequest );
345 
346         assertTrue( lockFile.canRead() );
347         assertFalse( new File( localCacheDir, "nexus-maven-repository-index.1.gz" ).canRead() );
348         assertFalse( unknownFile.canRead() );
349         assertFalse( unknownDirectory.isDirectory() );
350     }
351 
352     public void testOffline()
353         throws Exception
354     {
355         indexer.addArtifactToIndex( createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.2", null ),
356             context );
357         packIndex( remoteRepo, context );
358 
359         //
360         TrackingFetcher fetcher;
361         IndexUpdateRequest updateRequest;
362 
363         // initial index download (expected: successfull download)
364         fetcher = new TrackingFetcher( remoteRepo );
365         IndexingContext testContext = getNewTempContext();
366         updateRequest = new IndexUpdateRequest( testContext, fetcher );
367         updateRequest.setLocalIndexCacheDir( localCacheDir );
368         updater.fetchAndUpdateIndex( updateRequest );
369 
370         // recreate local index from the cache without remote access (and NULL fetcher)
371         // fetcher is null, so we no way to assert that
372         updateRequest = new IndexUpdateRequest( testContext, fetcher );
373         updateRequest.setLocalIndexCacheDir( localCacheDir );
374         updateRequest.setOffline( true );
375         updater.fetchAndUpdateIndex( updateRequest );
376         assertGroupCount( 1, "commons-lang", testContext );
377 
378         // recreate local index from the cache without remote access (and NOT NULL fetcher)
379         fetcher = new TrackingFetcher( remoteRepo );
380         updateRequest = new IndexUpdateRequest( testContext, fetcher );
381         updateRequest.setLocalIndexCacheDir( localCacheDir );
382         updateRequest.setOffline( true );
383         updater.fetchAndUpdateIndex( updateRequest );
384         assertEquals( 0, fetcher.getRetrievedResources().size() );
385         assertGroupCount( 1, "commons-lang", testContext );
386     }
387 
388 }