package org.apache.maven.wagon.shared.http;
   
   /*
    * Licensed to the Apache Software Foundation (ASF) under one
    * or more contributor license agreements.  See the NOTICE file
    * distributed with this work for additional information
    * regarding copyright ownership.  The ASF licenses this file
    * to you under the Apache License, Version 2.0 (the
    * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
   *   http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing,
   * software distributed under the License is distributed on an
   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   * KIND, either express or implied.  See the License for the
   * specific language governing permissions and limitations
   * under the License.
   */
  
  import org.apache.maven.wagon.TransferFailedException;
  import org.apache.xerces.xni.Augmentations;
  import org.apache.xerces.xni.QName;
  import org.apache.xerces.xni.XMLAttributes;
  import org.apache.xerces.xni.parser.XMLInputSource;
  import org.apache.xerces.xni.parser.XMLParserConfiguration;
  import org.codehaus.plexus.util.StringUtils;
  import org.cyberneko.html.HTMLConfiguration;
  import org.cyberneko.html.filters.DefaultFilter;
  
  import java.io.IOException;
  import java.io.InputStream;
  import java.io.UnsupportedEncodingException;
  import java.net.URI;
  import java.net.URISyntaxException;
  import java.net.URLDecoder;
  import java.util.ArrayList;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Set;
  import java.util.regex.Pattern;
  
  /**
   * Html File List Parser.
   */
  public class HtmlFileListParser
  {
      /**
       * Fetches a raw HTML from a provided InputStream, parses it, and returns the file list.
       *
       * @return the file list.
       * @throws TransferFailedException if there was a problem fetching the raw html.
       */
      public static List<String> parseFileList( String baseurl, InputStream stream )
          throws TransferFailedException
      {
          try
          {
              // Use URI object to get benefits of proper absolute and relative path resolution for free
              URI baseURI = new URI( baseurl );
  
              Parser handler = new Parser( baseURI );
  
              XMLParserConfiguration parser = new HTMLConfiguration();
              parser.setDocumentHandler( handler );
              parser.setFeature( "http://cyberneko.org/html/features/augmentations", true );
              parser.setProperty( "http://cyberneko.org/html/properties/names/elems", "upper" );
              parser.setProperty( "http://cyberneko.org/html/properties/names/attrs", "upper" );
              parser.parse( new XMLInputSource( null, baseurl, baseURI.toString(), stream, "UTF-8" ) );
  
              return new ArrayList<String>( handler.getLinks() );
  
          }
          catch ( URISyntaxException e )
          {
              throw new TransferFailedException( "Unable to parse as URI: " + baseurl, e );
          }
          catch ( IOException e )
          {
              throw new TransferFailedException( "I/O error: " + e.getMessage(), e );
          }
      }
  
      private static class Parser
          extends DefaultFilter
      {
          // Apache Fancy Index Sort Headers
          private static final Pattern APACHE_INDEX_SKIP = Pattern.compile( "\\?[CDMNS]=.*" );
  
          // URLs with excessive paths.
          private static final Pattern URLS_WITH_PATHS = Pattern.compile( "/[^/]*/" );
  
          // URLs that to a parent directory.
          private static final Pattern URLS_TO_PARENT = Pattern.compile( "\\.\\./" );
  
          // mailto urls
          private static final Pattern MAILTO_URLS = Pattern.compile( "mailto:.*" );
  
         private static final Pattern[] SKIPS =
             new Pattern[]{ APACHE_INDEX_SKIP, URLS_WITH_PATHS, URLS_TO_PARENT, MAILTO_URLS };
 
         private Set<String> links = new HashSet<String>();
 
         private URI baseURI;
 
         public Parser( URI baseURI )
         {
             this.baseURI = baseURI.normalize();
         }
 
         public Set<String> getLinks()
         {
             return links;
         }
 
         public void startElement( QName element, XMLAttributes attrs, Augmentations augs )
         {
             if ( "A".equals( element.rawname ) )
             {
                 String href = attrs.getValue( "HREF" );
                 if ( href != null )
                 {
                     String link = cleanLink( baseURI, href );
                     if ( isAcceptableLink( link ) )
                     {
                         links.add( link );
                     }
                 }
             }
         }
 
         private static String cleanLink( URI baseURI, String link )
         {
             if ( StringUtils.isEmpty( link ) )
             {
                 return "";
             }
 
             String ret = link;
 
             try
             {
                 URI linkuri = new URI( ret );
                 URI relativeURI = baseURI.relativize( linkuri ).normalize();
                 ret = relativeURI.toASCIIString();
                 if ( ret.startsWith( baseURI.getPath() ) )
                 {
                     ret = ret.substring( baseURI.getPath().length() );
                 }
 
                 ret = URLDecoder.decode( ret, "UTF-8" );
             }
             catch ( URISyntaxException e )
             {
             }
             catch ( UnsupportedEncodingException e )
             {
             }
 
             return ret;
         }
 
         private static boolean isAcceptableLink( String link )
         {
             if ( StringUtils.isEmpty( link ) )
             {
                 return false;
             }
 
             for ( int i = 0; i < SKIPS.length; i++ )
             {
                 if ( SKIPS[i].matcher( link ).find() )
                 {
                     return false;
                 }
             }
 
             return true;
         }
     }
 }