001package org.apache.maven.wagon.shared.http; 002 003/* 004 * Licensed to the Apache Software Foundation (ASF) under one 005 * or more contributor license agreements. See the NOTICE file 006 * distributed with this work for additional information 007 * regarding copyright ownership. The ASF licenses this file 008 * to you under the Apache License, Version 2.0 (the 009 * "License"); you may not use this file except in compliance 010 * with the License. You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, 015 * software distributed under the License is distributed on an 016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 017 * KIND, either express or implied. See the License for the 018 * specific language governing permissions and limitations 019 * under the License. 020 */ 021 022import org.apache.commons.io.IOUtils; 023import org.apache.maven.wagon.TransferFailedException; 024import org.jsoup.Jsoup; 025import org.jsoup.nodes.Document; 026import org.jsoup.nodes.Element; 027import org.jsoup.select.Elements; 028 029import java.io.IOException; 030import java.io.InputStream; 031import java.io.UnsupportedEncodingException; 032import java.net.URI; 033import java.net.URISyntaxException; 034import java.net.URLDecoder; 035import java.util.ArrayList; 036import java.util.HashSet; 037import java.util.List; 038import java.util.Set; 039import java.util.regex.Pattern; 040 041/** 042 * Html File List Parser. 043 */ 044public class HtmlFileListParser 045{ 046 // Apache Fancy Index Sort Headers 047 private static final Pattern APACHE_INDEX_SKIP = Pattern.compile( "\\?[CDMNS]=.*" ); 048 049 // URLs with excessive paths. 050 private static final Pattern URLS_WITH_PATHS = Pattern.compile( "/[^/]*/" ); 051 052 // URLs that to a parent directory. 053 private static final Pattern URLS_TO_PARENT = Pattern.compile( "\\.\\./" ); 054 055 // mailto urls 056 private static final Pattern MAILTO_URLS = Pattern.compile( "mailto:.*" ); 057 058 private static final Pattern[] SKIPS = 059 new Pattern[]{ APACHE_INDEX_SKIP, URLS_WITH_PATHS, URLS_TO_PARENT, MAILTO_URLS }; 060 061 /** 062 * Fetches a raw HTML from a provided InputStream, parses it, and returns the file list. 063 * 064 * @param stream the input stream. 065 * @return the file list. 066 * @throws TransferFailedException if there was a problem fetching the raw html. 067 */ 068 public static List<String> parseFileList( String baseurl, InputStream stream ) 069 throws TransferFailedException 070 { 071 try 072 { 073 URI baseURI = new URI( baseurl ); 074 // to make debugging easier, start with a string. This is assuming UTF-8, which might not be a safe 075 // assumption. 076 String content = IOUtils.toString( stream, "utf-8" ); 077 Document doc = Jsoup.parse( content, baseurl ); 078 Elements links = doc.select( "a[href]" ); 079 Set<String> results = new HashSet<String>(); 080 for ( Element link : links ) 081 { 082 /* 083 * The abs:href loses directories, so we deal with absolute paths ourselves below in cleanLink 084 */ 085 String target = link.attr( "href" ); 086 if ( target != null ) 087 { 088 String clean = cleanLink( baseURI, target ); 089 if ( isAcceptableLink( clean ) ) 090 { 091 results.add( clean ); 092 } 093 } 094 095 } 096 097 return new ArrayList<String>( results ); 098 } 099 catch ( URISyntaxException e ) 100 { 101 throw new TransferFailedException( "Unable to parse as base URI: " + baseurl, e ); 102 } 103 catch ( IOException e ) 104 { 105 throw new TransferFailedException( "I/O error reading HTML listing of artifacts: " + e.getMessage(), e ); 106 } 107 } 108 109 private static String cleanLink( URI baseURI, String link ) 110 { 111 if ( link == null || link.length() == 0 ) 112 { 113 return ""; 114 } 115 116 String ret = link; 117 118 try 119 { 120 URI linkuri = new URI( ret ); 121 if ( link.startsWith( "/" ) ) 122 { 123 linkuri = baseURI.resolve( linkuri ); 124 } 125 URI relativeURI = baseURI.relativize( linkuri ).normalize(); 126 ret = relativeURI.toASCIIString(); 127 if ( ret.startsWith( baseURI.getPath() ) ) 128 { 129 ret = ret.substring( baseURI.getPath().length() ); 130 } 131 132 ret = URLDecoder.decode( ret, "UTF-8" ); 133 } 134 catch ( URISyntaxException e ) 135 { 136 // ignore 137 } 138 catch ( UnsupportedEncodingException e ) 139 { 140 // ignore 141 } 142 143 return ret; 144 } 145 146 private static boolean isAcceptableLink( String link ) 147 { 148 if ( link == null || link.length() == 0 ) 149 { 150 return false; 151 } 152 153 for ( Pattern pattern : SKIPS ) 154 { 155 if ( pattern.matcher( link ).find() ) 156 { 157 return false; 158 } 159 } 160 161 return true; 162 } 163 164}