Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.apache.maven.doxia.module.markdown;
020
021import javax.inject.Inject;
022import javax.inject.Named;
023import javax.inject.Singleton;
024
025import java.io.IOException;
026import java.io.Reader;
027import java.util.Arrays;
028import java.util.Collections;
029import java.util.LinkedHashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.Map.Entry;
033import java.util.regex.Matcher;
034import java.util.regex.Pattern;
035import java.util.stream.Collectors;
036
037import com.vladsch.flexmark.ast.Heading;
038import com.vladsch.flexmark.ast.HtmlCommentBlock;
039import com.vladsch.flexmark.ext.abbreviation.AbbreviationExtension;
040import com.vladsch.flexmark.ext.autolink.AutolinkExtension;
041import com.vladsch.flexmark.ext.definition.DefinitionExtension;
042import com.vladsch.flexmark.ext.escaped.character.EscapedCharacterExtension;
043import com.vladsch.flexmark.ext.footnotes.FootnoteExtension;
044import com.vladsch.flexmark.ext.gfm.strikethrough.StrikethroughExtension;
045import com.vladsch.flexmark.ext.tables.TablesExtension;
046import com.vladsch.flexmark.ext.typographic.TypographicExtension;
047import com.vladsch.flexmark.ext.wikilink.WikiLinkExtension;
048import com.vladsch.flexmark.ext.yaml.front.matter.YamlFrontMatterExtension;
049import com.vladsch.flexmark.html.HtmlRenderer;
050import com.vladsch.flexmark.util.ast.Node;
051import com.vladsch.flexmark.util.ast.TextCollectingVisitor;
052import com.vladsch.flexmark.util.data.MutableDataSet;
053import org.apache.commons.io.IOUtils;
054import org.apache.maven.doxia.markup.HtmlMarkup;
055import org.apache.maven.doxia.markup.TextMarkup;
056import org.apache.maven.doxia.module.xhtml5.Xhtml5Parser;
057import org.apache.maven.doxia.parser.AbstractTextParser;
058import org.apache.maven.doxia.parser.ParseException;
059import org.apache.maven.doxia.sink.Sink;
060import org.apache.maven.doxia.util.HtmlTools;
061import org.codehaus.plexus.util.xml.pull.XmlPullParser;
062import org.jsoup.Jsoup;
063import org.jsoup.nodes.Document;
064
065/**
066 * <p>
067 * Implementation of {@link org.apache.maven.doxia.parser.Parser} for Markdown documents.
068 * </p>
069 * <p>
070 * Defers effective parsing to the <a href="https://github.com/vsch/flexmark-java">flexmark-java library</a>,
071 * which generates HTML content then delegates parsing of this content to a slightly modified Doxia Xhtml5 parser.
072 * (before 1.8, the <a href="http://pegdown.org">PegDown library</a> was used)
073 * </p>
074 *
075 * @author Vladimir Schneider
076 * @author Julien Nicoulaud
077 * @since 1.3
078 */
079@Singleton
080@Named("markdown")
081public class MarkdownParser extends AbstractTextParser implements TextMarkup {
082
083    /**
084     * Regex that identifies a multimarkdown-style metadata section at the start of the document
085     *
086     * In order to ensure that we have minimal risk of false positives when slurping metadata sections, the
087     * first key in the metadata section must be one of these standard keys or else the entire metadata section is
088     * ignored.
089     * @see <a href="https://fletcher.github.io/MultiMarkdown-5/metadata.html">Multimarkdown Metadata</a>
090     */
091    private static final Pattern METADATA_SECTION_PATTERN = Pattern.compile(
092            "\\A^"
093                    + "(?:title|author|date|address|affiliation|copyright|email|keywords|language|phone|subtitle)"
094                    + "[ \\t]*:[\\S\\s]+?^[ \\t]*$",
095            Pattern.MULTILINE | Pattern.CASE_INSENSITIVE);
096
097    /**
098     * Regex that captures the key and value of a multimarkdown-style metadata entry.
099     * Group 1 captures the key, group 2 captures the value. Multivalues are not supported in the syntax!
100     * Multiline values need to be normalized
101     * @see <a href="https://fletcher.github.io/MultiMarkdown-5/metadata.html">Multimarkdown Metadata</a>
102     *
103     */
104    private static final Pattern METADATA_ENTRY_PATTERN = Pattern.compile(
105            "^([^:\\r\\n]+?)[ \\t]*:([\\S\\s]+?)(?=(?:^(?:[^:\\r\\n]+?)[ \\t]*:)|^[ \\t]*$)", Pattern.MULTILINE);
106
107    /**
108     * The parser of the HTML produced by Flexmark, that we will
109     * use to convert this HTML to Sink events
110     */
111    @Inject
112    private MarkdownHtmlParser parser;
113
114    /**
115     * Flexmark's Markdown parser (one static instance fits all)
116     */
117    private static final com.vladsch.flexmark.parser.Parser FLEXMARK_PARSER;
118
119    /**
120     * Flexmark's Markdown Metadata parser
121     */
122    private static final com.vladsch.flexmark.parser.Parser FLEXMARK_METADATA_PARSER;
123
124    /**
125     * Flexmark's HTML renderer (its output will be re-parsed and converted to Sink events)
126     */
127    private static final HtmlRenderer FLEXMARK_HTML_RENDERER;
128
129    // Initialize the Flexmark parser and renderer, once and for all
130    static {
131        MutableDataSet flexmarkOptions = new MutableDataSet();
132
133        // Enable the extensions that we used to have in Pegdown
134        flexmarkOptions.set(
135                com.vladsch.flexmark.parser.Parser.EXTENSIONS,
136                Arrays.asList(
137                        EscapedCharacterExtension.create(),
138                        AbbreviationExtension.create(),
139                        AutolinkExtension.create(),
140                        DefinitionExtension.create(),
141                        TypographicExtension.create(),
142                        TablesExtension.create(),
143                        WikiLinkExtension.create(),
144                        FootnoteExtension.create(),
145                        StrikethroughExtension.create()));
146
147        // Disable wrong apostrophe replacement
148        flexmarkOptions.set(TypographicExtension.SINGLE_QUOTE_UNMATCHED, "&apos;");
149
150        // Additional options on the HTML rendering
151        flexmarkOptions.set(HtmlRenderer.HTML_BLOCK_OPEN_TAG_EOL, false);
152        flexmarkOptions.set(HtmlRenderer.HTML_BLOCK_CLOSE_TAG_EOL, false);
153        flexmarkOptions.set(HtmlRenderer.MAX_TRAILING_BLANK_LINES, -1);
154        flexmarkOptions.set(HtmlRenderer.FENCED_CODE_NO_LANGUAGE_CLASS, "nohighlight nocode");
155
156        // Build the Markdown parser
157        FLEXMARK_PARSER =
158                com.vladsch.flexmark.parser.Parser.builder(flexmarkOptions).build();
159
160        MutableDataSet flexmarkMetadataOptions = new MutableDataSet();
161        flexmarkMetadataOptions.set(
162                com.vladsch.flexmark.parser.Parser.EXTENSIONS, Arrays.asList(YamlFrontMatterExtension.create()));
163        FLEXMARK_METADATA_PARSER = com.vladsch.flexmark.parser.Parser.builder(flexmarkMetadataOptions)
164                .build();
165
166        // Build the HTML renderer
167        FLEXMARK_HTML_RENDERER = HtmlRenderer.builder(flexmarkOptions)
168                .linkResolverFactory(new FlexmarkDoxiaLinkResolver.Factory())
169                .build();
170    }
171
172    @Override
173    public void parse(Reader source, Sink sink, String reference) throws ParseException {
174        try {
175            // Markdown to HTML (using flexmark-java library)
176            String xhtml = toXhtml(source);
177
178            // TODO: add locator for the markdown source (not the intermediate HTML format)
179            // this requires writing a custom renderer not leveraging the XHTML parser
180
181            // then HTML to Sink API
182            parser.setEmitComments(isEmitComments());
183            parser.parse(xhtml, getWrappedSink(sink), "Intermediate HTML from " + reference);
184        } catch (IOException e) {
185            throw new ParseException("Failed reading Markdown source document", e);
186        }
187    }
188
189    private boolean processMetadataForHtml(StringBuilder html, StringBuilder source) {
190        final Map<String, List<String>> metadata;
191        final int endOffset; // end of metadata within source
192        // support two types of metadata:
193        if (source.toString().startsWith("---")) {
194            // 1. YAML front matter (https://github.com/vsch/flexmark-java/wiki/Extensions#yaml-front-matter)
195            Node documentRoot = FLEXMARK_METADATA_PARSER.parse(source.toString());
196            YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor();
197            visitor.visit(documentRoot);
198            metadata = visitor.getData();
199            endOffset = visitor.getEndOffset();
200        } else {
201            // 2. Multimarkdown metadata (https://fletcher.github.io/MultiMarkdown-5/metadata.html), not yet supported
202            // by Flexmark (https://github.com/vsch/flexmark-java/issues/550)
203            metadata = new LinkedHashMap<>();
204            Matcher metadataMatcher = METADATA_SECTION_PATTERN.matcher(source);
205            if (metadataMatcher.find()) {
206                String entry = metadataMatcher.group(0) + EOL;
207                Matcher entryMatcher = METADATA_ENTRY_PATTERN.matcher(entry);
208                while (entryMatcher.find()) {
209                    String key = entryMatcher.group(1);
210                    String value = normalizeMultilineValue(entryMatcher.group(2));
211                    metadata.put(key, Collections.singletonList(value));
212                }
213                endOffset = metadataMatcher.end(0);
214            } else {
215                endOffset = 0;
216            }
217        }
218        if (endOffset > 0) {
219            // Trim the metadata from the source
220            source.delete(0, endOffset);
221        }
222        return writeHtmlMetadata(html, metadata);
223    }
224
225    static String normalizeMultilineValue(String value) {
226        return value.trim().replaceAll("[ \\t]*[\\r\\n]+[ \\t]*", " ");
227    }
228
229    private boolean writeHtmlMetadata(StringBuilder html, Map<String, List<String>> data) {
230        boolean containsTitle = false;
231        for (Entry<String, List<String>> entry : data.entrySet()) {
232            if (writeHtmlMetadata(html, entry.getKey(), entry.getValue())) {
233                containsTitle = true;
234            }
235        }
236        return containsTitle;
237    }
238
239    private boolean writeHtmlMetadata(StringBuilder html, String key, List<String> values) {
240        if ("title".equalsIgnoreCase(key)) {
241            html.append("<title>");
242            html.append(HtmlTools.escapeHTML(values.stream().collect(Collectors.joining(", ")), false));
243            html.append("</title>");
244            return true;
245        } else {
246            if (key.equalsIgnoreCase("author") && values.size() > 1) {
247                // for multiple authors emit multiple meta tags
248                for (String value : values) {
249                    writeHtmlMetadata(html, key, Collections.singletonList(value));
250                }
251            } else {
252                // every other multi-value should just be concatenated and emitted in a single meta tag
253                final String separator;
254                if (key.equalsIgnoreCase("keywords")) {
255                    separator = ",";
256                } else {
257                    separator = EOL;
258                }
259                html.append("<meta name='");
260                html.append(HtmlTools.escapeHTML(key));
261                html.append("' content='");
262                html.append(HtmlTools.escapeHTML(values.stream().collect(Collectors.joining(separator))));
263                html.append("' />");
264            }
265            return false;
266        }
267    }
268
269    /**
270     * uses flexmark-java library to parse content and generate HTML output.
271     *
272     * @param source the Markdown source
273     * @return HTML content generated by flexmark-java
274     * @throws IOException passed through
275     */
276    String toXhtml(Reader source) throws IOException {
277        // Read the source
278        StringBuilder markdownText = new StringBuilder(IOUtils.toString(source));
279
280        // Now, build the HTML document
281        StringBuilder html = new StringBuilder(1000);
282        html.append("<html>");
283        html.append("<head>");
284
285        boolean haveTitle = processMetadataForHtml(html, markdownText);
286
287        // Now is the time to parse the Markdown document
288        // (after we've trimmed out the metadatas, and before we check for its headings)
289        Node documentRoot = FLEXMARK_PARSER.parse(markdownText.toString());
290
291        // Special trick: if there is no title specified as a metadata in the header, we will use the first
292        // heading as the document title
293        if (!haveTitle && documentRoot.hasChildren()) {
294            // Skip the comment nodes
295            Node firstNode = documentRoot.getFirstChild();
296            while (firstNode != null && firstNode instanceof HtmlCommentBlock) {
297                firstNode = firstNode.getNext();
298            }
299
300            // If this first non-comment node is a heading, we use it as the document title
301            if (firstNode != null && firstNode instanceof Heading) {
302                html.append("<title>");
303                TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
304                String headingText = collectingVisitor.collectAndGetText(firstNode);
305                html.append(HtmlTools.escapeHTML(headingText, false));
306                html.append("</title>");
307            }
308        }
309        html.append("</head>");
310        html.append("<body>");
311
312        // Convert our Markdown document to HTML and append it to our HTML
313        FLEXMARK_HTML_RENDERER.render(documentRoot, html);
314
315        html.append("</body>");
316        html.append("</html>");
317
318        return toXhtml(html.toString());
319    }
320
321    private String toXhtml(String html) {
322        final Document document = Jsoup.parse(html);
323        document.outputSettings().syntax(Document.OutputSettings.Syntax.xml).prettyPrint(false);
324        return document.html();
325    }
326
327    /**
328     * Internal parser for HTML generated by the Markdown library.
329     *
330     * 2 special things:
331     * <ul>
332     * <li> DIV elements are translated as Unknown Sink events
333     * </ul>
334     * PRE elements need to be "source" because the Xhtml5Sink will surround the
335     * corresponding verbatim() Sink event with a DIV element with class="source",
336     * which is how most Maven Skin (incl. Fluido) recognize a block of code, which
337     * needs to be highlighted accordingly.
338     */
339    @Named
340    public static class MarkdownHtmlParser extends Xhtml5Parser {
341        public MarkdownHtmlParser() {
342            super();
343        }
344
345        @Override
346        protected void init() {
347            super.init();
348        }
349
350        @Override
351        protected boolean baseEndTag(XmlPullParser parser, Sink sink) {
352            boolean visited = super.baseEndTag(parser, sink);
353            if (!visited) {
354                if (parser.getName().equals(HtmlMarkup.DIV.toString())) {
355                    handleUnknown(parser, sink, TAG_TYPE_END);
356                    visited = true;
357                }
358            }
359            return visited;
360        }
361
362        @Override
363        protected boolean baseStartTag(XmlPullParser parser, Sink sink) {
364            boolean visited = super.baseStartTag(parser, sink);
365            if (!visited) {
366                if (parser.getName().equals(HtmlMarkup.DIV.toString())) {
367                    handleUnknown(parser, sink, TAG_TYPE_START);
368                    visited = true;
369                }
370            }
371            return visited;
372        }
373    }
374}