package org.apache.maven.index.updater;
   
   /*
    * Licensed to the Apache Software Foundation (ASF) under one
    * or more contributor license agreements.  See the NOTICE file
    * distributed with this work for additional information
    * regarding copyright ownership.  The ASF licenses this file
    * to you under the Apache License, Version 2.0 (the
    * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
   *   http://www.apache.org/licenses/LICENSE-2.0    
   *
   * Unless required by applicable law or agreed to in writing,
   * software distributed under the License is distributed on an
   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   * KIND, either express or implied.  See the License for the
   * specific language governing permissions and limitations
   * under the License.
   */
  
  import javax.inject.Inject;
  import javax.inject.Named;
  import javax.inject.Singleton;
  import java.io.BufferedInputStream;
  import java.io.BufferedOutputStream;
  import java.io.BufferedReader;
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileNotFoundException;
  import java.io.FileOutputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.InputStreamReader;
  import java.io.OutputStream;
  import java.io.OutputStreamWriter;
  import java.io.Writer;
  import java.nio.charset.StandardCharsets;
  import java.nio.file.Files;
  import java.text.ParseException;
  import java.text.SimpleDateFormat;
  import java.util.ArrayList;
  import java.util.Date;
  import java.util.List;
  import java.util.Properties;
  import java.util.Set;
  import java.util.TimeZone;
  
  import org.apache.lucene.document.Document;
  import org.apache.lucene.index.DirectoryReader;
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.index.IndexWriter;
  import org.apache.lucene.index.IndexWriterConfig;
  import org.apache.lucene.index.MultiBits;
  import org.apache.lucene.store.Directory;
  import org.apache.lucene.util.Bits;
  import org.apache.maven.index.context.DocumentFilter;
  import org.apache.maven.index.context.IndexUtils;
  import org.apache.maven.index.context.IndexingContext;
  import org.apache.maven.index.context.NexusAnalyzer;
  import org.apache.maven.index.context.NexusIndexWriter;
  import org.apache.maven.index.fs.Lock;
  import org.apache.maven.index.fs.Locker;
  import org.apache.maven.index.incremental.IncrementalHandler;
  import org.apache.maven.index.updater.IndexDataReader.IndexDataReadResult;
  import org.codehaus.plexus.util.FileUtils;
  import org.codehaus.plexus.util.io.RawInputStreamFacade;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  /**
   * A default index updater implementation
   * 
   * @author Jason van Zyl
   * @author Eugene Kuleshov
   */
  @Singleton
  @Named
  public class DefaultIndexUpdater
      implements IndexUpdater
  {
  
      private final Logger logger = LoggerFactory.getLogger( getClass() );
  
      protected Logger getLogger()
      {
          return logger;
      }
  
      private final IncrementalHandler incrementalHandler;
  
      private final List<IndexUpdateSideEffect> sideEffects;
  
  
      @Inject
      public DefaultIndexUpdater( final IncrementalHandler incrementalHandler,
                                  final List<IndexUpdateSideEffect> sideEffects )
      {
          this.incrementalHandler = incrementalHandler;
         this.sideEffects = sideEffects;
     }
 
     public IndexUpdateResult fetchAndUpdateIndex( final IndexUpdateRequest updateRequest )
         throws IOException
     {
         IndexUpdateResult result = new IndexUpdateResult();
 
         IndexingContext context = updateRequest.getIndexingContext();
 
         ResourceFetcher fetcher = null;
 
         if ( !updateRequest.isOffline() )
         {
             fetcher = updateRequest.getResourceFetcher();
 
             // If no resource fetcher passed in, use the wagon fetcher by default
             // and put back in request for future use
             if ( fetcher == null )
             {
                 throw new IOException( "Update of the index without provided ResourceFetcher is impossible." );
             }
 
             fetcher.connect( context.getId(), context.getIndexUpdateUrl() );
         }
 
         File cacheDir = updateRequest.getLocalIndexCacheDir();
         Locker locker = updateRequest.getLocker();
         Lock lock = locker != null && cacheDir != null ? locker.lock( cacheDir ) : null;
         try
         {
             if ( cacheDir != null )
             {
                 LocalCacheIndexAdaptor cache = new LocalCacheIndexAdaptor( cacheDir, result );
 
                 if ( !updateRequest.isOffline() )
                 {
                     cacheDir.mkdirs();
 
                     try
                     {
                         if ( fetchAndUpdateIndex( updateRequest, fetcher, cache ).isSuccessful() )
                         {
                             cache.commit();
                         }
                     }
                     finally
                     {
                         fetcher.disconnect();
                     }
                 }
 
                 fetcher = cache.getFetcher();
             }
             else if ( updateRequest.isOffline() )
             {
                 throw new IllegalArgumentException( "LocalIndexCacheDir can not be null in offline mode" );
             }
 
             try
             {
                 if ( !updateRequest.isCacheOnly() )
                 {
                     LuceneIndexAdaptor target = new LuceneIndexAdaptor( updateRequest );
                     result = fetchAndUpdateIndex( updateRequest, fetcher, target );
                     
                     if ( result.isSuccessful() )
                     {
                         target.commit();
                     }
                 }
             }
             finally
             {
                 fetcher.disconnect();
             }
         }
         finally
         {
             if ( lock != null )
             {
                 lock.release();
             }
         }
 
         return result;
     }
 
     private Date loadIndexDirectory( final IndexUpdateRequest updateRequest, final ResourceFetcher fetcher,
                                      final boolean merge, final String remoteIndexFile )
         throws IOException
     {
         File indexDir;
         if ( updateRequest.getIndexTempDir() != null )
         {
             updateRequest.getIndexTempDir().mkdirs();
             indexDir = Files.createTempDirectory( updateRequest.getIndexTempDir().toPath(),
                 remoteIndexFile + ".dir" ).toFile();
         }
         else
         {
             indexDir = Files.createTempDirectory( remoteIndexFile + ".dir" ).toFile();
         }
         try ( BufferedInputStream is = new BufferedInputStream( fetcher.retrieve( remoteIndexFile ) ); //
                         Directory directory = updateRequest.getFSDirectoryFactory().open( indexDir ) )
         {
             Date timestamp;
 
             Set<String> rootGroups;
             Set<String> allGroups;
             if ( remoteIndexFile.endsWith( ".gz" ) )
             {
                 IndexDataReadResult result = unpackIndexData( is, updateRequest.getThreads(), directory,
                         updateRequest.getIndexingContext() );
                 timestamp = result.getTimestamp();
                 rootGroups = result.getRootGroups();
                 allGroups = result.getAllGroups();
             }
             else
             {
                 // legacy transfer format
                 throw new IllegalArgumentException( "The legacy format is no longer supported "
                     + "by this version of maven-indexer." );
             }
 
             if ( updateRequest.getDocumentFilter() != null )
             {
                 filterDirectory( directory, updateRequest.getDocumentFilter() );
             }
 
             if ( merge )
             {
                 updateRequest.getIndexingContext().merge( directory );
             }
             else
             {
                 updateRequest.getIndexingContext().replace( directory, allGroups, rootGroups );
             }
             if ( sideEffects != null && sideEffects.size() > 0 )
             {
                 getLogger().info( IndexUpdateSideEffect.class.getName() + " extensions found: " + sideEffects.size() );
                 for ( IndexUpdateSideEffect sideeffect : sideEffects )
                 {
                     sideeffect.updateIndex( directory, updateRequest.getIndexingContext(), merge );
                 }
             }
 
             return timestamp;
         }
         finally
         {
             try
             {
                 FileUtils.deleteDirectory( indexDir );
             }
             catch ( IOException ex )
             {
                 // ignore
             }
         }
     }
 
     private static void filterDirectory( final Directory directory, final DocumentFilter filter )
         throws IOException
     {
         IndexReader r = null;
         IndexWriter w = null;
         try
         {
             r = DirectoryReader.open( directory );
             w = new NexusIndexWriter( directory, new IndexWriterConfig( new NexusAnalyzer() ) );
 
             Bits liveDocs = MultiBits.getLiveDocs( r );
 
             int numDocs = r.maxDoc();
 
             for ( int i = 0; i < numDocs; i++ )
             {
                 if ( liveDocs != null && !liveDocs.get( i ) )
                 {
                     continue;
                 }
 
                 Document d = r.document( i );
 
                 if ( !filter.accept( d ) )
                 {
                     boolean success = w.tryDeleteDocument( r, i ) != -1;
                     // FIXME handle deletion failure
                 }
             }
             w.commit();
         }
         finally
         {
             IndexUtils.close( r );
             IndexUtils.close( w );
         }
 
         w = null;
         try
         {
             // analyzer is unimportant, since we are not adding/searching to/on index, only reading/deleting
             w = new NexusIndexWriter( directory, new IndexWriterConfig( new NexusAnalyzer() ) );
 
             w.commit();
         }
         finally
         {
             IndexUtils.close( w );
         }
     }
 
     private Properties loadIndexProperties( final File indexDirectoryFile, final String remoteIndexPropertiesName )
     {
         File indexProperties = new File( indexDirectoryFile, remoteIndexPropertiesName );
 
         try ( FileInputStream fis = new FileInputStream( indexProperties ) )
         {
             Properties properties = new Properties();
 
             properties.load( fis );
 
             return properties;
         }
         catch ( IOException e )
         {
             getLogger().debug( "Unable to read remote properties stored locally", e );
         }
         return null;
     }
 
     private void storeIndexProperties( final File dir, final String indexPropertiesName, final Properties properties )
         throws IOException
     {
         File file = new File( dir, indexPropertiesName );
 
         if ( properties != null )
         {
             try ( OutputStream os = new BufferedOutputStream( new FileOutputStream( file ) ) )
             {
                 properties.store( os, null );
             }
         }
         else
         {
             file.delete();
         }
     }
 
     private Properties downloadIndexProperties( final ResourceFetcher fetcher )
         throws IOException
     {
         try ( InputStream fis = fetcher.retrieve( IndexingContext.INDEX_REMOTE_PROPERTIES_FILE ) )
         {
             Properties properties = new Properties();
 
             properties.load( fis );
 
             return properties;
         }
     }
 
     public Date getTimestamp( final Properties properties, final String key )
     {
         String indexTimestamp = properties.getProperty( key );
 
         if ( indexTimestamp != null )
         {
             try
             {
                 SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT );
                 df.setTimeZone( TimeZone.getTimeZone( "GMT" ) );
                 return df.parse( indexTimestamp );
             }
             catch ( ParseException ex )
             {
             }
         }
         return null;
     }
 
     /**
      * @param is an input stream to unpack index data from
      * @param threads thread count to use
      * @param d
      * @param context
      */
     public static IndexDataReadResult unpackIndexData( final InputStream is, final int threads, final Directory d,
                                                        final IndexingContext context )
         throws IOException
     {
         IndexWriterConfig config = new IndexWriterConfig( new NexusAnalyzer() );
         config.setUseCompoundFile( false );
         NexusIndexWriter w = new NexusIndexWriter( d, config );
         try
         {
             IndexDataReader dr = new IndexDataReader( is, threads );
 
             return dr.readIndex( w, context );
         }
         finally
         {
             IndexUtils.close( w );
         }
     }
 
     /**
      * Filesystem-based ResourceFetcher implementation
      */
     public static class FileFetcher
         implements ResourceFetcher
     {
         private final File basedir;
 
         public FileFetcher( File basedir )
         {
             this.basedir = basedir;
         }
 
         public void connect( String id, String url )
             throws IOException
         {
             // don't need to do anything
         }
 
         public void disconnect()
             throws IOException
         {
             // don't need to do anything
         }
 
         public void retrieve( String name, File targetFile )
             throws IOException, FileNotFoundException
         {
             FileUtils.copyFile( getFile( name ), targetFile );
 
         }
 
         public InputStream retrieve( String name )
             throws IOException, FileNotFoundException
         {
             return new FileInputStream( getFile( name ) );
         }
 
         private File getFile( String name )
         {
             return new File( basedir, name );
         }
 
     }
 
     private abstract class IndexAdaptor
     {
         protected final File dir;
 
         protected Properties properties;
 
         protected IndexAdaptor( File dir )
         {
             this.dir = dir;
         }
 
         public abstract Properties getProperties();
 
         public abstract void storeProperties()
             throws IOException;
 
         public abstract void addIndexChunk( ResourceFetcher source, String filename )
             throws IOException;
 
         public abstract Date setIndexFile( ResourceFetcher source, String string )
             throws IOException;
 
         public Properties setProperties( ResourceFetcher source )
             throws IOException
         {
             this.properties = downloadIndexProperties( source );
             return properties;
         }
 
         public abstract Date getTimestamp();
 
         public void commit()
             throws IOException
         {
             storeProperties();
         }
     }
 
     private class LuceneIndexAdaptor
         extends IndexAdaptor
     {
         private final IndexUpdateRequest updateRequest;
 
         LuceneIndexAdaptor( IndexUpdateRequest updateRequest )
         {
             super( updateRequest.getIndexingContext().getIndexDirectoryFile() );
             this.updateRequest = updateRequest;
         }
 
         public Properties getProperties()
         {
             if ( properties == null )
             {
                 properties = loadIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE );
             }
             return properties;
         }
 
         public void storeProperties()
             throws IOException
         {
             storeIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE, properties );
         }
 
         public Date getTimestamp()
         {
             return updateRequest.getIndexingContext().getTimestamp();
         }
 
         public void addIndexChunk( ResourceFetcher source, String filename )
             throws IOException
         {
             loadIndexDirectory( updateRequest, source, true, filename );
         }
 
         public Date setIndexFile( ResourceFetcher source, String filename )
             throws IOException
         {
             return loadIndexDirectory( updateRequest, source, false, filename );
         }
 
         public void commit()
             throws IOException
         {
             super.commit();
 
             updateRequest.getIndexingContext().commit();
         }
 
     }
 
     private class LocalCacheIndexAdaptor
         extends IndexAdaptor
     {
         private static final String CHUNKS_FILENAME = "chunks.lst";
 
         private final IndexUpdateResult result;
 
         private final ArrayList<String> newChunks = new ArrayList<>();
 
         LocalCacheIndexAdaptor( File dir, IndexUpdateResult result )
         {
             super( dir );
             this.result = result;
         }
 
         public Properties getProperties()
         {
             if ( properties == null )
             {
                 properties = loadIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
             }
             return properties;
         }
 
         public void storeProperties()
             throws IOException
         {
             storeIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE, properties );
         }
 
         public Date getTimestamp()
         {
             Properties properties = getProperties();
             if ( properties == null )
             {
                 return null;
             }
 
             Date timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_TIMESTAMP );
 
             if ( timestamp == null )
             {
                 timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
             }
 
             return timestamp;
         }
 
         public void addIndexChunk( ResourceFetcher source, String filename )
             throws IOException
         {
             File chunk = new File( dir, filename );
             FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), chunk );
             newChunks.add( filename );
         }
 
         public Date setIndexFile( ResourceFetcher source, String filename )
             throws IOException
         {
             cleanCacheDirectory( dir );
 
             result.setFullUpdate( true );
 
             File target = new File( dir, filename );
             FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), target );
 
             return null;
         }
 
         @Override
         public void commit()
             throws IOException
         {
             File chunksFile = new File( dir, CHUNKS_FILENAME );
             try ( BufferedOutputStream os = new BufferedOutputStream( new FileOutputStream( chunksFile, true ) ); //
                             Writer w = new OutputStreamWriter( os, StandardCharsets.UTF_8 ) )
             {
                 for ( String filename : newChunks )
                 {
                     w.write( filename + "\n" );
                 }
                 w.flush();
             }
             super.commit();
         }
 
         public List<String> getChunks()
             throws IOException
         {
             ArrayList<String> chunks = new ArrayList<>();
 
             File chunksFile = new File( dir, CHUNKS_FILENAME );
             try ( BufferedReader r =
                 new BufferedReader( new InputStreamReader(
                         new FileInputStream( chunksFile ), StandardCharsets.UTF_8 ) ) )
             {
                 String str;
                 while ( ( str = r.readLine() ) != null )
                 {
                     chunks.add( str );
                 }
             }
             return chunks;
         }
 
         public ResourceFetcher getFetcher()
         {
             return new LocalIndexCacheFetcher( dir )
             {
                 @Override
                 public List<String> getChunks()
                     throws IOException
                 {
                     return LocalCacheIndexAdaptor.this.getChunks();
                 }
             };
         }
     }
 
     abstract static class LocalIndexCacheFetcher
         extends FileFetcher
     {
         LocalIndexCacheFetcher( File basedir )
         {
             super( basedir );
         }
 
         public abstract List<String> getChunks()
             throws IOException;
     }
 
     private IndexUpdateResult fetchAndUpdateIndex( final IndexUpdateRequest updateRequest, ResourceFetcher source,
                                       IndexAdaptor target )
         throws IOException
     {
         IndexUpdateResult result = new IndexUpdateResult();
         
         if ( !updateRequest.isForceFullUpdate() )
         {
             Properties localProperties = target.getProperties();
             Date localTimestamp = null;
 
             if ( localProperties != null )
             {
                 localTimestamp = getTimestamp( localProperties, IndexingContext.INDEX_TIMESTAMP );
             }
 
             // this will download and store properties in the target, so next run
             // target.getProperties() will retrieve it
             Properties remoteProperties = target.setProperties( source );
 
             Date updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_TIMESTAMP );
 
             // If new timestamp is missing, dont bother checking incremental, we have an old file
             if ( updateTimestamp != null )
             {
                 List<String> filenames =
                     incrementalHandler.loadRemoteIncrementalUpdates( updateRequest, localProperties, remoteProperties );
 
                 // if we have some incremental files, merge them in
                 if ( filenames != null )
                 {
                     for ( String filename : filenames )
                     {
                         target.addIndexChunk( source, filename );
                     }
 
                     result.setTimestamp( updateTimestamp );
                     result.setSuccessful( true );
                     return result;
                 }
             }
             else
             {
                 updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
             }
 
             // fallback to timestamp comparison, but try with one coming from local properties, and if not possible (is
             // null)
             // fallback to context timestamp
             if ( localTimestamp != null )
             {
                 // if we have localTimestamp
                 // if incremental can't be done for whatever reason, simply use old logic of
                 // checking the timestamp, if the same, nothing to do
                 if ( updateTimestamp != null && localTimestamp != null && !updateTimestamp.after( localTimestamp ) )
                 {
                     //Index is up to date
                     result.setSuccessful( true );
                     return result;
                 }
             }
         }
         else
         {
             // create index properties during forced full index download
             target.setProperties( source );
         }
 
         if ( !updateRequest.isIncrementalOnly() )
         {
             Date timestamp;
             try
             {
                 timestamp = target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
                 if ( source instanceof LocalIndexCacheFetcher )
                 {
                     // local cache has inverse organization compared to remote indexes,
                     // i.e. initial index file and delta chunks to apply on top of it
                     for ( String filename : ( (LocalIndexCacheFetcher) source ).getChunks() )
                     {
                         target.addIndexChunk( source, filename );
                     }
                 }
             }
             catch ( IOException ex )
             {
                 // try to look for legacy index transfer format
                 try
                 {
                     timestamp = target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".zip" );
                 }
                 catch ( IOException ex2 )
                 {
                     getLogger().error( "Fallback to *.zip also failed: " + ex2 ); // do not bother with stack trace
                     
                     throw ex; // original exception more likely to be interesting
                 }
             }
             
             result.setTimestamp( timestamp );
             result.setSuccessful( true );
             result.setFullUpdate( true );
         }
         
         return result;
     }
 
     /**
      * Cleans specified cache directory. If present, Locker.LOCK_FILE will not be deleted.
      */
     protected void cleanCacheDirectory( File dir )
         throws IOException
     {
         File[] members = dir.listFiles();
         if ( members == null )
         {
             return;
         }
 
         for ( File member : members )
         {
             if ( !Locker.LOCK_FILE.equals( member.getName() ) )
             {
                 FileUtils.forceDelete( member );
             }
         }
     }
 
 }