package org.apache.maven.index.updater;
   
   /*
    * Licensed to the Apache Software Foundation (ASF) under one
    * or more contributor license agreements.  See the NOTICE file
    * distributed with this work for additional information
    * regarding copyright ownership.  The ASF licenses this file
    * to you under the Apache License, Version 2.0 (the
    * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
   *   http://www.apache.org/licenses/LICENSE-2.0    
   *
   * Unless required by applicable law or agreed to in writing,
   * software distributed under the License is distributed on an
   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   * KIND, either express or implied.  See the License for the
   * specific language governing permissions and limitations
   * under the License.
   */
  
  import java.io.BufferedInputStream;
  import java.io.BufferedOutputStream;
  import java.io.BufferedReader;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileNotFoundException;
  import java.io.FileOutputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.InputStreamReader;
  import java.io.OutputStream;
  import java.io.OutputStreamWriter;
  import java.io.Writer;
  import java.text.ParseException;
  import java.text.SimpleDateFormat;
  import java.util.ArrayList;
  import java.util.Date;
  import java.util.List;
  import java.util.Properties;
  import java.util.TimeZone;
  import java.util.zip.ZipEntry;
  import java.util.zip.ZipInputStream;
  
  import org.apache.lucene.document.Document;
  import org.apache.lucene.index.CorruptIndexException;
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.index.IndexWriter;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.store.FSDirectory;
  import org.apache.lucene.store.IndexOutput;
  import org.apache.lucene.store.LockObtainFailedException;
  import org.apache.maven.index.context.DocumentFilter;
  import org.apache.maven.index.context.IndexUtils;
  import org.apache.maven.index.context.IndexingContext;
  import org.apache.maven.index.context.NexusAnalyzer;
  import org.apache.maven.index.context.NexusIndexWriter;
  import org.apache.maven.index.fs.Lock;
  import org.apache.maven.index.fs.Locker;
  import org.apache.maven.index.incremental.IncrementalHandler;
  import org.apache.maven.index.updater.IndexDataReader.IndexDataReadResult;
  import org.codehaus.plexus.component.annotations.Component;
  import org.codehaus.plexus.component.annotations.Requirement;
  import org.codehaus.plexus.logging.AbstractLogEnabled;
  import org.codehaus.plexus.util.FileUtils;
  import org.codehaus.plexus.util.IOUtil;
  import org.codehaus.plexus.util.io.RawInputStreamFacade;
  
  /**
   * A default index updater implementation
   * 
   * @author Jason van Zyl
   * @author Eugene Kuleshov
   */
  @Component( role = IndexUpdater.class )
  public class DefaultIndexUpdater
      extends AbstractLogEnabled
      implements IndexUpdater
  {
  
      @Requirement( role = IncrementalHandler.class )
      IncrementalHandler incrementalHandler;
  
      @Requirement( role = IndexUpdateSideEffect.class )
      private List<IndexUpdateSideEffect> sideEffects;
  
      public DefaultIndexUpdater( final IncrementalHandler handler, final List<IndexUpdateSideEffect> mySideeffects )
      {
          incrementalHandler = handler;
          sideEffects = mySideeffects;
      }
  
      public DefaultIndexUpdater()
      {
  
      }
  
      public IndexUpdateResult fetchAndUpdateIndex( final IndexUpdateRequest updateRequest )
          throws IOException
     {
         IndexUpdateResult result = new IndexUpdateResult();
 
         IndexingContext context = updateRequest.getIndexingContext();
 
         ResourceFetcher fetcher = null;
 
         if ( !updateRequest.isOffline() )
         {
             fetcher = updateRequest.getResourceFetcher();
 
             // If no resource fetcher passed in, use the wagon fetcher by default
             // and put back in request for future use
             if ( fetcher == null )
             {
                 throw new IOException( "Update of the index without provided ResourceFetcher is impossible." );
             }
 
             fetcher.connect( context.getId(), context.getIndexUpdateUrl() );
         }
 
         File cacheDir = updateRequest.getLocalIndexCacheDir();
         Locker locker = updateRequest.getLocker();
         Lock lock = locker != null && cacheDir != null ? locker.lock( cacheDir ) : null;
         try
         {
             if ( cacheDir != null )
             {
                 LocalCacheIndexAdaptor cache = new LocalCacheIndexAdaptor( cacheDir, result );
 
                 if ( !updateRequest.isOffline() )
                 {
                     cacheDir.mkdirs();
 
                     try
                     {
                         fetchAndUpdateIndex( updateRequest, fetcher, cache );
                         cache.commit();
                     }
                     finally
                     {
                         fetcher.disconnect();
                     }
                 }
 
                 fetcher = cache.getFetcher();
             }
             else if ( updateRequest.isOffline() )
             {
                 throw new IllegalArgumentException( "LocalIndexCacheDir can not be null in offline mode" );
             }
 
             try
             {
                 if ( !updateRequest.isCacheOnly() )
                 {
                     LuceneIndexAdaptor target = new LuceneIndexAdaptor( updateRequest );
                     result.setTimestamp( fetchAndUpdateIndex( updateRequest, fetcher, target ) );
                     target.commit();
                 }
             }
             finally
             {
                 fetcher.disconnect();
             }
         }
         finally
         {
             if ( lock != null )
             {
                 lock.release();
             }
         }
 
         return result;
     }
 
     private Date loadIndexDirectory( final IndexUpdateRequest updateRequest, final ResourceFetcher fetcher,
                                      final boolean merge, final String remoteIndexFile )
         throws IOException
     {
         File indexDir = File.createTempFile( remoteIndexFile, ".dir" );
         indexDir.delete();
         indexDir.mkdirs();
 
         FSDirectory directory = FSDirectory.open( indexDir );
 
         BufferedInputStream is = null;
 
         try
         {
             is = new BufferedInputStream( fetcher.retrieve( remoteIndexFile ) );
 
             Date timestamp = null;
 
             if ( remoteIndexFile.endsWith( ".gz" ) )
             {
                 timestamp = unpackIndexData( is, directory, //
                     updateRequest.getIndexingContext() );
             }
             else
             {
                 // legacy transfer format
                 timestamp = unpackIndexArchive( is, directory, //
                     updateRequest.getIndexingContext() );
             }
 
             if ( updateRequest.getDocumentFilter() != null )
             {
                 filterDirectory( directory, updateRequest.getDocumentFilter() );
             }
 
             if ( merge )
             {
                 updateRequest.getIndexingContext().merge( directory );
             }
             else
             {
                 updateRequest.getIndexingContext().replace( directory );
             }
             if ( sideEffects != null && sideEffects.size() > 0 )
             {
                 getLogger().info( IndexUpdateSideEffect.class.getName() + " extensions found: " + sideEffects.size() );
                 for ( IndexUpdateSideEffect sideeffect : sideEffects )
                 {
                     sideeffect.updateIndex( directory, updateRequest.getIndexingContext(), merge );
                 }
             }
 
             return timestamp;
         }
         finally
         {
             IOUtil.close( is );
 
             if ( directory != null )
             {
                 directory.close();
             }
 
             try
             {
                 FileUtils.deleteDirectory( indexDir );
             }
             catch ( IOException ex )
             {
                 // ignore
             }
         }
     }
 
     /**
      * Unpack legacy index archive into a specified Lucene <code>Directory</code>
      * 
      * @param is a <code>ZipInputStream</code> with index data
      * @param directory Lucene <code>Directory</code> to unpack index data to
      * @return {@link Date} of the index update or null if it can't be read
      */
     public static Date unpackIndexArchive( final InputStream is, final Directory directory,
                                            final IndexingContext context )
         throws IOException
     {
         File indexArchive = File.createTempFile( "nexus-index", "" );
 
         File indexDir = new File( indexArchive.getAbsoluteFile().getParentFile(), indexArchive.getName() + ".dir" );
 
         indexDir.mkdirs();
 
         FSDirectory fdir = FSDirectory.open( indexDir );
 
         try
         {
             unpackDirectory( fdir, is );
             copyUpdatedDocuments( fdir, directory, context );
 
             Date timestamp = IndexUtils.getTimestamp( fdir );
             IndexUtils.updateTimestamp( directory, timestamp );
             return timestamp;
         }
         finally
         {
             IndexUtils.close( fdir );
             indexArchive.delete();
             IndexUtils.delete( indexDir );
         }
     }
 
     private static void unpackDirectory( final Directory directory, final InputStream is )
         throws IOException
     {
         byte[] buf = new byte[4096];
 
         ZipEntry entry;
 
         ZipInputStream zis = null;
 
         try
         {
             zis = new ZipInputStream( is );
 
             while ( ( entry = zis.getNextEntry() ) != null )
             {
                 if ( entry.isDirectory() || entry.getName().indexOf( '/' ) > -1 )
                 {
                     continue;
                 }
 
                 IndexOutput io = directory.createOutput( entry.getName() );
                 try
                 {
                     int n = 0;
 
                     while ( ( n = zis.read( buf ) ) != -1 )
                     {
                         io.writeBytes( buf, n );
                     }
                 }
                 finally
                 {
                     IndexUtils.close( io );
                 }
             }
         }
         finally
         {
             IndexUtils.close( zis );
         }
     }
 
     private static void copyUpdatedDocuments( final Directory sourcedir, final Directory targetdir,
                                               final IndexingContext context )
         throws CorruptIndexException, LockObtainFailedException, IOException
     {
         IndexWriter w = null;
         IndexReader r = null;
         try
         {
             r = IndexReader.open( sourcedir );
             w = new NexusIndexWriter( targetdir, new NexusAnalyzer(), true );
 
             for ( int i = 0; i < r.maxDoc(); i++ )
             {
                 if ( !r.isDeleted( i ) )
                 {
                     w.addDocument( IndexUtils.updateDocument( r.document( i ), context ) );
                 }
             }
 
             w.optimize();
             w.commit();
         }
         finally
         {
             IndexUtils.close( w );
             IndexUtils.close( r );
         }
     }
 
     private static void filterDirectory( final Directory directory, final DocumentFilter filter )
         throws IOException
     {
         IndexReader r = null;
         try
         {
             // explicitly RW reader needed
             r = IndexReader.open( directory, false );
 
             int numDocs = r.maxDoc();
 
             for ( int i = 0; i < numDocs; i++ )
             {
                 if ( r.isDeleted( i ) )
                 {
                     continue;
                 }
 
                 Document d = r.document( i );
 
                 if ( !filter.accept( d ) )
                 {
                     r.deleteDocument( i );
                 }
             }
         }
         finally
         {
             IndexUtils.close( r );
         }
 
         IndexWriter w = null;
         try
         {
             // analyzer is unimportant, since we are not adding/searching to/on index, only reading/deleting
             w = new NexusIndexWriter( directory, new NexusAnalyzer(), false );
 
             w.optimize();
 
             w.commit();
         }
         finally
         {
             IndexUtils.close( w );
         }
     }
 
     private Properties loadIndexProperties( final File indexDirectoryFile, final String remoteIndexPropertiesName )
     {
         File indexProperties = new File( indexDirectoryFile, remoteIndexPropertiesName );
 
         FileInputStream fis = null;
 
         try
         {
             Properties properties = new Properties();
 
             fis = new FileInputStream( indexProperties );
 
             properties.load( fis );
 
             return properties;
         }
         catch ( IOException e )
         {
             getLogger().debug( "Unable to read remote properties stored locally", e );
         }
         finally
         {
             IOUtil.close( fis );
         }
 
         return null;
     }
 
     private void storeIndexProperties( final File dir, final String indexPropertiesName, final Properties properties )
         throws IOException
     {
         File file = new File( dir, indexPropertiesName );
 
         if ( properties != null )
         {
             OutputStream os = new BufferedOutputStream( new FileOutputStream( file ) );
             try
             {
                 properties.store( os, null );
             }
             finally
             {
                 IOUtil.close( os );
             }
         }
         else
         {
             file.delete();
         }
     }
 
     private Properties downloadIndexProperties( final ResourceFetcher fetcher )
         throws IOException
     {
         InputStream fis = fetcher.retrieve( IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
 
         try
         {
             Properties properties = new Properties();
 
             properties.load( fis );
 
             return properties;
         }
         finally
         {
             IOUtil.close( fis );
         }
     }
 
     public Date getTimestamp( final Properties properties, final String key )
     {
         String indexTimestamp = properties.getProperty( key );
 
         if ( indexTimestamp != null )
         {
             try
             {
                 SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT );
                 df.setTimeZone( TimeZone.getTimeZone( "GMT" ) );
                 return df.parse( indexTimestamp );
             }
             catch ( ParseException ex )
             {
             }
         }
         return null;
     }
 
     /**
      * Unpack index data using specified Lucene Index writer
      * 
      * @param is an input stream to unpack index data from
      * @param w a writer to save index data
      * @param ics a collection of index creators for updating unpacked documents.
      */
     public static Date unpackIndexData( final InputStream is, final Directory d, final IndexingContext context )
         throws IOException
     {
         NexusIndexWriter w = new NexusIndexWriter( d, new NexusAnalyzer(), true );
         try
         {
             IndexDataReader dr = new IndexDataReader( is );
 
             IndexDataReadResult result = dr.readIndex( w, context );
 
             return result.getTimestamp();
         }
         finally
         {
             IndexUtils.close( w );
         }
     }
 
     /**
      * Filesystem-based ResourceFetcher implementation
      */
     public static class FileFetcher
         implements ResourceFetcher
     {
         private final File basedir;
 
         public FileFetcher( File basedir )
         {
             this.basedir = basedir;
         }
 
         public void connect( String id, String url )
             throws IOException
         {
             // don't need to do anything
         }
 
         public void disconnect()
             throws IOException
         {
             // don't need to do anything
         }
 
         public void retrieve( String name, File targetFile )
             throws IOException, FileNotFoundException
         {
             FileUtils.copyFile( getFile( name ), targetFile );
 
         }
 
         public InputStream retrieve( String name )
             throws IOException, FileNotFoundException
         {
             return new FileInputStream( getFile( name ) );
         }
 
         private File getFile( String name )
         {
             return new File( basedir, name );
         }
 
     }
 
     private abstract class IndexAdaptor
     {
         protected final File dir;
 
         protected Properties properties;
 
         protected IndexAdaptor( File dir )
         {
             this.dir = dir;
         }
 
         public abstract Properties getProperties();
 
         public abstract void storeProperties()
             throws IOException;
 
         public abstract void addIndexChunk( ResourceFetcher source, String filename )
             throws IOException;
 
         public abstract Date setIndexFile( ResourceFetcher source, String string )
             throws IOException;
 
         public Properties setProperties( ResourceFetcher source )
             throws IOException
         {
             this.properties = downloadIndexProperties( source );
             return properties;
         }
 
         public abstract Date getTimestamp();
 
         public void commit()
             throws IOException
         {
             storeProperties();
         }
     }
 
     private class LuceneIndexAdaptor
         extends IndexAdaptor
     {
         private final IndexUpdateRequest updateRequest;
 
         public LuceneIndexAdaptor( IndexUpdateRequest updateRequest )
         {
             super( updateRequest.getIndexingContext().getIndexDirectoryFile() );
             this.updateRequest = updateRequest;
         }
 
         public Properties getProperties()
         {
             if ( properties == null )
             {
                 properties = loadIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE );
             }
             return properties;
         }
 
         public void storeProperties()
             throws IOException
         {
             storeIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE, properties );
         }
 
         public Date getTimestamp()
         {
             return updateRequest.getIndexingContext().getTimestamp();
         }
 
         public void addIndexChunk( ResourceFetcher source, String filename )
             throws IOException
         {
             loadIndexDirectory( updateRequest, source, true, filename );
         }
 
         public Date setIndexFile( ResourceFetcher source, String filename )
             throws IOException
         {
             return loadIndexDirectory( updateRequest, source, false, filename );
         }
 
         public void commit()
             throws IOException
         {
             super.commit();
 
             updateRequest.getIndexingContext().commit();
         }
 
     }
 
     private class LocalCacheIndexAdaptor
         extends IndexAdaptor
     {
         private static final String CHUNKS_FILENAME = "chunks.lst";
 
         private static final String CHUNKS_FILE_ENCODING = "UTF-8";
 
         private final IndexUpdateResult result;
 
         private final ArrayList<String> newChunks = new ArrayList<String>();
 
         public LocalCacheIndexAdaptor( File dir, IndexUpdateResult result )
         {
             super( dir );
             this.result = result;
         }
 
         public Properties getProperties()
         {
             if ( properties == null )
             {
                 properties = loadIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
             }
             return properties;
         }
 
         public void storeProperties()
             throws IOException
         {
             storeIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE, properties );
         }
 
         public Date getTimestamp()
         {
             Properties properties = getProperties();
             if ( properties == null )
             {
                 return null;
             }
 
             Date timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_TIMESTAMP );
 
             if ( timestamp == null )
             {
                 timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
             }
 
             return timestamp;
         }
 
         public void addIndexChunk( ResourceFetcher source, String filename )
             throws IOException
         {
             File chunk = new File( dir, filename );
             FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), chunk );
             newChunks.add( filename );
         }
 
         public Date setIndexFile( ResourceFetcher source, String filename )
             throws IOException
         {
             cleanCacheDirectory( dir );
 
             result.setFullUpdate( true );
 
             File target = new File( dir, filename );
             FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), target );
 
             return null;
         }
 
         @Override
         public void commit()
             throws IOException
         {
             File chunksFile = new File( dir, CHUNKS_FILENAME );
             BufferedOutputStream os = new BufferedOutputStream( new FileOutputStream( chunksFile, true ) );
             Writer w = new OutputStreamWriter( os, CHUNKS_FILE_ENCODING );
             try
             {
                 for ( String filename : newChunks )
                 {
                     w.write( filename + "\n" );
                 }
                 w.flush();
             }
             finally
             {
                 IOUtil.close( w );
                 IOUtil.close( os );
             }
             super.commit();
         }
 
         public List<String> getChunks()
             throws IOException
         {
             ArrayList<String> chunks = new ArrayList<String>();
 
             File chunksFile = new File( dir, CHUNKS_FILENAME );
             BufferedReader r =
                 new BufferedReader( new InputStreamReader( new FileInputStream( chunksFile ), CHUNKS_FILE_ENCODING ) );
             try
             {
                 String str;
                 while ( ( str = r.readLine() ) != null )
                 {
                     chunks.add( str );
                 }
             }
             finally
             {
                 IOUtil.close( r );
             }
             return chunks;
         }
 
         public ResourceFetcher getFetcher()
         {
             return new LocalIndexCacheFetcher( dir )
             {
                 @Override
                 public List<String> getChunks()
                     throws IOException
                 {
                     return LocalCacheIndexAdaptor.this.getChunks();
                 }
             };
         }
     }
 
     abstract static class LocalIndexCacheFetcher
         extends FileFetcher
     {
         public LocalIndexCacheFetcher( File basedir )
         {
             super( basedir );
         }
 
         public abstract List<String> getChunks()
             throws IOException;
     }
 
     private Date fetchAndUpdateIndex( final IndexUpdateRequest updateRequest, ResourceFetcher source,
                                       IndexAdaptor target )
         throws IOException
     {
         if ( !updateRequest.isForceFullUpdate() )
         {
             Properties localProperties = target.getProperties();
             Date localTimestamp = null;
 
             if ( localProperties != null )
             {
                 localTimestamp = getTimestamp( localProperties, IndexingContext.INDEX_TIMESTAMP );
             }
 
             // this will download and store properties in the target, so next run
             // target.getProperties() will retrieve it
             Properties remoteProperties = target.setProperties( source );
 
             Date updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_TIMESTAMP );
 
             // If new timestamp is missing, dont bother checking incremental, we have an old file
             if ( updateTimestamp != null )
             {
                 List<String> filenames =
                     incrementalHandler.loadRemoteIncrementalUpdates( updateRequest, localProperties, remoteProperties );
 
                 // if we have some incremental files, merge them in
                 if ( filenames != null )
                 {
                     for ( String filename : filenames )
                     {
                         target.addIndexChunk( source, filename );
                     }
 
                     return updateTimestamp;
                 }
             }
             else
             {
                 updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
             }
 
             // fallback to timestamp comparison, but try with one coming from local properties, and if not possible (is
             // null)
             // fallback to context timestamp
             if ( localTimestamp != null )
             {
                 // if we have localTimestamp
                 // if incremental can't be done for whatever reason, simply use old logic of
                 // checking the timestamp, if the same, nothing to do
                 if ( updateTimestamp != null && localTimestamp != null && !updateTimestamp.after( localTimestamp ) )
                 {
                     return null; // index is up to date
                 }
             }
         }
         else
         {
             // create index properties during forced full index download
             target.setProperties( source );
         }
 
         try
         {
             Date timestamp = target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
             if ( source instanceof LocalIndexCacheFetcher )
             {
                 // local cache has inverse organization compared to remote indexes,
                 // i.e. initial index file and delta chunks to apply on top of it
                 for ( String filename : ( (LocalIndexCacheFetcher) source ).getChunks() )
                 {
                     target.addIndexChunk( source, filename );
                 }
             }
             return timestamp;
         }
         catch ( IOException ex )
         {
             // try to look for legacy index transfer format
             try
             {
                 return target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".zip" );
             }
             catch ( IOException ex2 )
             {
                 getLogger().error( "Fallback to *.zip also failed: " + ex2 ); // do not bother with stack trace
                 
                 throw ex; // original exception more likely to be interesting
             }
         }
     }
 
     /**
      * Cleans specified cache directory. If present, Locker.LOCK_FILE will not be deleted.
      */
     protected void cleanCacheDirectory( File dir )
         throws IOException
     {
         File[] members = dir.listFiles();
         if ( members == null )
         {
             return;
         }
 
         for ( File member : members )
         {
             if ( !Locker.LOCK_FILE.equals( member.getName() ) )
             {
                 FileUtils.forceDelete( member );
             }
         }
     }
 
 }