1 package org.apache.maven.index.packer;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.BufferedOutputStream;
23 import java.io.File;
24 import java.io.FileInputStream;
25 import java.io.FileOutputStream;
26 import java.io.IOException;
27 import java.io.OutputStream;
28 import java.text.SimpleDateFormat;
29 import java.util.Date;
30 import java.util.List;
31 import java.util.Properties;
32 import java.util.TimeZone;
33 import java.util.zip.ZipEntry;
34 import java.util.zip.ZipOutputStream;
35
36 import org.apache.lucene.document.Document;
37 import org.apache.lucene.document.Field;
38 import org.apache.lucene.index.CorruptIndexException;
39 import org.apache.lucene.index.IndexReader;
40 import org.apache.lucene.index.IndexWriter;
41 import org.apache.lucene.search.IndexSearcher;
42 import org.apache.lucene.store.Directory;
43 import org.apache.lucene.store.FSDirectory;
44 import org.apache.lucene.store.IndexInput;
45 import org.apache.lucene.store.LockObtainFailedException;
46 import org.apache.maven.index.ArtifactInfo;
47 import org.apache.maven.index.context.IndexCreator;
48 import org.apache.maven.index.context.IndexUtils;
49 import org.apache.maven.index.context.IndexingContext;
50 import org.apache.maven.index.context.NexusIndexWriter;
51 import org.apache.maven.index.context.NexusLegacyAnalyzer;
52 import org.apache.maven.index.creator.LegacyDocumentUpdater;
53 import org.apache.maven.index.incremental.IncrementalHandler;
54 import org.apache.maven.index.updater.IndexDataWriter;
55 import org.codehaus.plexus.component.annotations.Component;
56 import org.codehaus.plexus.component.annotations.Requirement;
57 import org.codehaus.plexus.logging.AbstractLogEnabled;
58 import org.codehaus.plexus.util.FileUtils;
59 import org.codehaus.plexus.util.IOUtil;
60
61
62
63
64
65
66
67 @Component( role = IndexPacker.class )
68 public class DefaultIndexPacker
69 extends AbstractLogEnabled
70 implements IndexPacker
71 {
72 @Requirement( role = IncrementalHandler.class )
73 IncrementalHandler incrementalHandler;
74
75 public void packIndex( IndexPackingRequest request )
76 throws IOException, IllegalArgumentException
77 {
78 if ( request.getTargetDir() == null )
79 {
80 throw new IllegalArgumentException( "The target dir is null" );
81 }
82
83 if ( request.getTargetDir().exists() )
84 {
85 if ( !request.getTargetDir().isDirectory() )
86 {
87 throw new IllegalArgumentException(
88 String.format( "Specified target path %s is not a directory",
89 request.getTargetDir().getAbsolutePath() ) );
90 }
91 if ( !request.getTargetDir().canWrite() )
92 {
93 throw new IllegalArgumentException( String.format( "Specified target path %s is not writtable",
94 request.getTargetDir().getAbsolutePath() ) );
95 }
96 }
97 else
98 {
99 if ( !request.getTargetDir().mkdirs() )
100 {
101 throw new IllegalArgumentException( "Can't create " + request.getTargetDir().getAbsolutePath() );
102 }
103 }
104
105
106 File legacyFile = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".zip" );
107 File v1File = new File( request.getTargetDir(), IndexingContext.INDEX_FILE_PREFIX + ".gz" );
108
109 Properties info = null;
110
111 try
112 {
113
114
115 info = readIndexProperties( request );
116
117 if ( request.isCreateIncrementalChunks() )
118 {
119 List<Integer> chunk = incrementalHandler.getIncrementalUpdates( request, info );
120
121 if ( chunk == null )
122 {
123 getLogger().debug( "Problem with Chunks, forcing regeneration of whole index" );
124 incrementalHandler.initializeProperties( info );
125 }
126 else if ( chunk.isEmpty() )
127 {
128 getLogger().debug( "No incremental changes, not writing new incremental chunk" );
129 }
130 else
131 {
132 File file =
133 new File( request.getTargetDir(),
134 IndexingContext.INDEX_FILE_PREFIX + "."
135 + info.getProperty( IndexingContext.INDEX_CHUNK_COUNTER ) + ".gz" );
136
137 writeIndexData( request.getContext(),
138 chunk, file );
139
140 if ( request.isCreateChecksumFiles() )
141 {
142 FileUtils.fileWrite(
143 new File( file.getParentFile(), file.getName() + ".sha1" ).getAbsolutePath(),
144 DigesterUtils.getSha1Digest( file ) );
145
146 FileUtils.fileWrite(
147 new File( file.getParentFile(), file.getName() + ".md5" ).getAbsolutePath(),
148 DigesterUtils.getMd5Digest( file ) );
149 }
150 }
151 }
152 }
153 catch ( IOException e )
154 {
155 getLogger().info( "Unable to read properties file, will force index regeneration" );
156 info = new Properties();
157 incrementalHandler.initializeProperties( info );
158 }
159
160 Date timestamp = request.getContext().getTimestamp();
161
162 if ( timestamp == null )
163 {
164 timestamp = new Date( 0 );
165 }
166
167 if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_LEGACY ) )
168 {
169 info.setProperty( IndexingContext.INDEX_LEGACY_TIMESTAMP, format( timestamp ) );
170
171 writeIndexArchive( request.getContext(), legacyFile );
172
173 if ( request.isCreateChecksumFiles() )
174 {
175 FileUtils.fileWrite(
176 new File( legacyFile.getParentFile(), legacyFile.getName() + ".sha1" ).getAbsolutePath(),
177 DigesterUtils.getSha1Digest( legacyFile ) );
178
179 FileUtils.fileWrite(
180 new File( legacyFile.getParentFile(), legacyFile.getName() + ".md5" ).getAbsolutePath(),
181 DigesterUtils.getMd5Digest( legacyFile ) );
182 }
183 }
184
185 if ( request.getFormats().contains( IndexPackingRequest.IndexFormat.FORMAT_V1 ) )
186 {
187 info.setProperty( IndexingContext.INDEX_TIMESTAMP, format( timestamp ) );
188
189 writeIndexData( request.getContext(), null, v1File );
190
191 if ( request.isCreateChecksumFiles() )
192 {
193 FileUtils.fileWrite( new File( v1File.getParentFile(), v1File.getName() + ".sha1" ).getAbsolutePath(),
194 DigesterUtils.getSha1Digest( v1File ) );
195
196 FileUtils.fileWrite( new File( v1File.getParentFile(), v1File.getName() + ".md5" ).getAbsolutePath(),
197 DigesterUtils.getMd5Digest( v1File ) );
198 }
199 }
200
201 writeIndexProperties( request, info );
202 }
203
204 private Properties readIndexProperties( IndexPackingRequest request )
205 throws IOException
206 {
207 File file = null;
208
209 if ( request.isUseTargetProperties() )
210 {
211 file = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
212 }
213 else
214 {
215 file =
216 new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE );
217 }
218
219 Properties properties = new Properties();
220
221 FileInputStream fos = null;
222
223 try
224 {
225 fos = new FileInputStream( file );
226 properties.load( fos );
227 }
228 finally
229 {
230 if ( fos != null )
231 {
232 fos.close();
233 }
234 }
235
236 return properties;
237 }
238
239 void writeIndexArchive( IndexingContext context, File targetArchive )
240 throws IOException
241 {
242 if ( targetArchive.exists() )
243 {
244 targetArchive.delete();
245 }
246
247 OutputStream os = null;
248
249 try
250 {
251 os = new BufferedOutputStream( new FileOutputStream( targetArchive ), 4096 );
252
253 packIndexArchive( context, os );
254 }
255 finally
256 {
257 IOUtil.close( os );
258 }
259 }
260
261
262
263
264 public static void packIndexArchive( IndexingContext context, OutputStream os )
265 throws IOException
266 {
267 File indexArchive = File.createTempFile( "nexus-index", "" );
268
269 File indexDir = new File( indexArchive.getAbsoluteFile().getParentFile(), indexArchive.getName() + ".dir" );
270
271 indexDir.mkdirs();
272
273 FSDirectory fdir = FSDirectory.open( indexDir );
274
275 try
276 {
277
278 IndexUtils.updateTimestamp( context.getIndexDirectory(), context.getTimestamp() );
279 IndexUtils.updateTimestamp( fdir, context.getTimestamp() );
280
281 final IndexSearcher indexSearcher = context.acquireIndexSearcher();
282 try
283 {
284 copyLegacyDocuments( indexSearcher.getIndexReader(), fdir, context );
285 }
286 finally
287 {
288 context.releaseIndexSearcher( indexSearcher );
289 }
290 packDirectory( fdir, os );
291 }
292 finally
293 {
294 IndexUtils.close( fdir );
295 indexArchive.delete();
296 IndexUtils.delete( indexDir );
297 }
298 }
299
300 static void copyLegacyDocuments( IndexReader r, Directory targetdir, IndexingContext context )
301 throws CorruptIndexException, LockObtainFailedException, IOException
302 {
303 IndexWriter w = null;
304 try
305 {
306 w = new NexusIndexWriter( targetdir, new NexusLegacyAnalyzer(), true );
307
308 for ( int i = 0; i < r.maxDoc(); i++ )
309 {
310 if ( !r.isDeleted( i ) )
311 {
312 w.addDocument( updateLegacyDocument( r.document( i ), context ) );
313 }
314 }
315
316 w.optimize();
317 w.commit();
318 }
319 finally
320 {
321 IndexUtils.close( w );
322 }
323 }
324
325 static Document updateLegacyDocument( Document doc, IndexingContext context )
326 {
327 ArtifactInfo ai = IndexUtils.constructArtifactInfo( doc, context );
328 if ( ai == null )
329 {
330 return doc;
331 }
332
333 Document document = new Document();
334 document.add( new Field( ArtifactInfo.UINFO, ai.getUinfo(), Field.Store.YES, Field.Index.NOT_ANALYZED ) );
335
336 for ( IndexCreator ic : context.getIndexCreators() )
337 {
338 if ( ic instanceof LegacyDocumentUpdater )
339 {
340 ( (LegacyDocumentUpdater) ic ).updateLegacyDocument( ai, document );
341 }
342 }
343
344 return document;
345 }
346
347 static void packDirectory( Directory directory, OutputStream os )
348 throws IOException
349 {
350 ZipOutputStream zos = null;
351 try
352 {
353 zos = new ZipOutputStream( os );
354 zos.setLevel( 9 );
355
356 String[] names = directory.listAll();
357
358 boolean savedTimestamp = false;
359
360 byte[] buf = new byte[8192];
361
362 for ( int i = 0; i < names.length; i++ )
363 {
364 String name = names[i];
365
366 writeFile( name, zos, directory, buf );
367
368 if ( name.equals( IndexUtils.TIMESTAMP_FILE ) )
369 {
370 savedTimestamp = true;
371 }
372 }
373
374
375 if ( !savedTimestamp && directory.fileExists( IndexUtils.TIMESTAMP_FILE ) )
376 {
377 writeFile( IndexUtils.TIMESTAMP_FILE, zos, directory, buf );
378 }
379 }
380 finally
381 {
382 IndexUtils.close( zos );
383 }
384 }
385
386 static void writeFile( String name, ZipOutputStream zos, Directory directory, byte[] buf )
387 throws IOException
388 {
389 ZipEntry e = new ZipEntry( name );
390
391 zos.putNextEntry( e );
392
393 IndexInput in = directory.openInput( name );
394
395 try
396 {
397 int toRead = 0;
398
399 int bytesLeft = (int) in.length();
400
401 while ( bytesLeft > 0 )
402 {
403 toRead = ( bytesLeft >= buf.length ) ? buf.length : bytesLeft;
404 bytesLeft -= toRead;
405
406 in.readBytes( buf, 0, toRead, false );
407
408 zos.write( buf, 0, toRead );
409 }
410 }
411 finally
412 {
413 IndexUtils.close( in );
414 }
415
416 zos.flush();
417
418 zos.closeEntry();
419 }
420
421 void writeIndexData( IndexingContext context, List<Integer> docIndexes, File targetArchive )
422 throws IOException
423 {
424 if ( targetArchive.exists() )
425 {
426 targetArchive.delete();
427 }
428
429 OutputStream os = null;
430
431 try
432 {
433 os = new FileOutputStream( targetArchive );
434
435 IndexDataWriter dw = new IndexDataWriter( os );
436 dw.write( context, docIndexes );
437
438 os.flush();
439 }
440 finally
441 {
442 IOUtil.close( os );
443 }
444 }
445
446 void writeIndexProperties( IndexPackingRequest request, Properties info )
447 throws IOException
448 {
449 File propertyFile =
450 new File( request.getContext().getIndexDirectoryFile(), IndexingContext.INDEX_PACKER_PROPERTIES_FILE );
451 File targetPropertyFile = new File( request.getTargetDir(), IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
452
453 info.setProperty( IndexingContext.INDEX_ID, request.getContext().getId() );
454
455 OutputStream os = null;
456
457 try
458 {
459 os = new FileOutputStream( propertyFile );
460
461 info.store( os, null );
462 }
463 finally
464 {
465 IOUtil.close( os );
466 }
467
468 try
469 {
470 os = new FileOutputStream( targetPropertyFile );
471
472 info.store( os, null );
473 }
474 finally
475 {
476 IOUtil.close( os );
477 }
478
479 if ( request.isCreateChecksumFiles() )
480 {
481 FileUtils.fileWrite(
482 new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".sha1" ).getAbsolutePath(),
483 DigesterUtils.getSha1Digest( targetPropertyFile ) );
484
485 FileUtils.fileWrite(
486 new File( targetPropertyFile.getParentFile(), targetPropertyFile.getName() + ".md5" ).getAbsolutePath(),
487 DigesterUtils.getMd5Digest( targetPropertyFile ) );
488 }
489 }
490
491 private String format( Date d )
492 {
493 SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT );
494 df.setTimeZone( TimeZone.getTimeZone( "GMT" ) );
495 return df.format( d );
496 }
497 }