001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *   http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing,
013 * software distributed under the License is distributed on an
014 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015 * KIND, either express or implied.  See the License for the
016 * specific language governing permissions and limitations
017 * under the License.
018 */
019package org.eclipse.aether.tools;
020
021import javax.tools.DiagnosticCollector;
022import javax.tools.DocumentationTool;
023import javax.tools.JavaFileObject;
024import javax.tools.StandardJavaFileManager;
025import javax.tools.StandardLocation;
026import javax.tools.ToolProvider;
027
028import java.io.File;
029import java.io.IOException;
030import java.io.PrintWriter;
031import java.io.Reader;
032import java.io.Writer;
033import java.net.URISyntaxException;
034import java.net.URL;
035import java.net.URLClassLoader;
036import java.nio.charset.StandardCharsets;
037import java.nio.file.Files;
038import java.nio.file.Path;
039import java.util.ArrayList;
040import java.util.Arrays;
041import java.util.Comparator;
042import java.util.LinkedHashMap;
043import java.util.List;
044import java.util.Map;
045import java.util.Properties;
046import java.util.concurrent.Callable;
047import java.util.stream.Collectors;
048import java.util.stream.Stream;
049
050import org.apache.velocity.VelocityContext;
051import org.apache.velocity.app.VelocityEngine;
052import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
053import org.codehaus.plexus.util.io.CachingWriter;
054import picocli.CommandLine;
055
056/**
057 * This tool is used both from <a href="https://github.com/apache/maven-resolver/blob/79d102b66235f33ad1e6134e18451ac3ee91b44a/maven-resolver-tools/pom.xml#L185">Resolver</a>
058 * as well as from <a href="https://github.com/apache/maven/blob/7aa3c8a37b091a5a86d3dae3a7d99ce910fd6caa/pom.xml#L1043">Maven</a>
059 * to generate documentation for configuration keys. It scans the source files under a given root directory, collects the configuration keys declared in them and renders them into Velocity templates.
060 * It relies on javadoc with a custom doclet to extract the configuration keys from the source files.
061 * The doclet writes the discovered keys into an intermediate properties file, which is then read back and used to render the Velocity templates.
062 */
063@CommandLine.Command(name = "docgen", description = "Configuration Documentation Generator")
064public class CollectConfiguration implements Callable<Integer> {
065    public static void main(String[] args) {
066        new CommandLine(new CollectConfiguration()).execute(args);
067    }
068
069    protected static final String KEY = "key";
070
071    /**
072     * The metadata fields collected per configuration key and written to / read from the intermediate properties file.
073     */
074    protected static final List<String> FIELDS = List.of(
075            KEY,
076            "defaultValue",
077            "fqName",
078            "description",
079            "since",
080            "configurationSource",
081            "configurationType",
082            "supportRepoIdSuffix");
083
084    /**
085     * Javadoc block tag marking a constant field as a configuration key.
086     */
087    protected static final String CONFIGURATION_MARKER = "@configurationSource";
088
089    /**
090     * Text marker used to pre-select the source files to feed the doclet when scanning Maven sources. Maven declares
091     * configuration keys via the {@code org.apache.maven.api.annotations.Config} annotation.
092     */
093    protected static final String MAVEN_CONFIGURATION_MARKER = "@Config";
094
095    /**
096     * The mode of the generator, i.e. what kind of sources are being scanned.
097     */
098    public enum Mode {
099        maven,
100        resolver
101    }
102
103    @CommandLine.Option(
104            names = {"-m", "--mode"},
105            arity = "1",
106            paramLabel = "mode",
107            description = "The mode of generator (what is being scanned?), supported modes are 'maven', 'resolver'")
108    protected Mode mode = Mode.resolver;
109
110    @CommandLine.Option(
111            names = {"-t", "--templates"},
112            arity = "1",
113            split = ",",
114            paramLabel = "template",
115            description = "The template names to write content out without '.vm' extension")
116    protected List<String> templates;
117
118    @CommandLine.Parameters(index = "0", description = "The root directory to process sources from")
119    protected Path rootDirectory;
120
121    @CommandLine.Parameters(index = "1", description = "The directory to generate output(s) to")
122    protected Path outputDirectory;
123
124    @Override
125    public Integer call() {
126        try {
127            rootDirectory = rootDirectory.toAbsolutePath().normalize();
128            outputDirectory = outputDirectory.toAbsolutePath().normalize();
129
130            System.out.println("Processing sources from " + rootDirectory);
131            Path intermediateFile = Files.createTempFile("configuration-keys", ".properties");
132            try {
133                runDoclet(intermediateFile);
134                List<Map<String, String>> discoveredKeys = readDiscoveredKeys(intermediateFile);
135                discoveredKeys.sort(Comparator.comparing(e -> e.get(KEY)));
136                render(discoveredKeys);
137            } finally {
138                Files.deleteIfExists(intermediateFile);
139            }
140            return 0;
141        } catch (Exception e) {
142            e.printStackTrace(System.err);
143            return 1;
144        }
145    }
146
147    /**
148     * Collects the source files under {@link #rootDirectory} and runs {@link ConfigurationCollectorDoclet} against them,
149     * having it write the discovered configuration keys into the given intermediate properties file.
150     */
151    protected void runDoclet(Path intermediateFile) throws Exception {
152        // Only feed javadoc the files that actually declare configuration keys. This keeps the set of types that
153        // javadoc must resolve small, avoiding failures caused by unrelated sources referencing dependencies that
154        // are not on this module's classpath (e.g. gson, jetty).
155        String marker = mode == Mode.maven ? MAVEN_CONFIGURATION_MARKER : CONFIGURATION_MARKER;
156        List<File> sourceFiles;
157        try (Stream<Path> stream = Files.walk(rootDirectory)) {
158            sourceFiles = stream.map(Path::toAbsolutePath)
159                    .filter(p -> p.getFileName().toString().endsWith(".java"))
160                    .filter(p -> p.toString().contains("/src/main/java/"))
161                    .filter(p -> !p.toString().endsWith("/module-info.java"))
162                    .filter(p -> !p.toString().contains("/maven-resolver-tools/"))
163                    .filter(p -> fileContains(p, marker))
164                    .map(Path::toFile)
165                    .collect(Collectors.toList());
166        }
167        if (sourceFiles.isEmpty()) {
168            throw new IllegalStateException(
169                    "No Java sources declaring configuration keys found under " + rootDirectory);
170        }
171
172        DocumentationTool documentationTool = ToolProvider.getSystemDocumentationTool();
173        DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>();
174        try (StandardJavaFileManager fileManager =
175                documentationTool.getStandardFileManager(diagnostics, null, StandardCharsets.UTF_8)) {
176            // Configure the classpath on the file manager (the -classpath option is not honored when a file manager
177            // is supplied to getTask()). Note that under exec:java the project dependencies are on the context
178            // classloader, not on the JVM's java.class.path.
179            fileManager.setLocation(StandardLocation.CLASS_PATH, resolveClasspath());
180
181            Iterable<? extends JavaFileObject> compilationUnits = fileManager.getJavaFileObjectsFromFiles(sourceFiles);
182
183            List<String> options = new ArrayList<>(Arrays.asList(
184                    "--output", intermediateFile.toString(), "--mode", mode.name(), "-encoding", "UTF-8"));
185
186            Writer out = new PrintWriter(System.err);
187            DocumentationTool.DocumentationTask task = documentationTool.getTask(
188                    out, fileManager, diagnostics, ConfigurationCollectorDoclet.class, options, compilationUnits);
189            boolean ok = task.call();
190            out.flush();
191            if (!ok) {
192                diagnostics.getDiagnostics().forEach(d -> System.err.println(d));
193                throw new IllegalStateException("Javadoc doclet execution failed");
194            }
195        }
196    }
197
198    private static boolean fileContains(Path path, String marker) {
199        try {
200            return Files.readString(path, StandardCharsets.UTF_8).contains(marker);
201        } catch (IOException e) {
202            return false;
203        }
204    }
205
206    /**
207     * Resolves the classpath to use for symbol resolution during the javadoc run. Under {@code exec:java} the project
208     * dependencies live on the context classloader (a {@link URLClassLoader}), not on the JVM's
209     * {@code java.class.path}, so both sources are combined.
210     */
211    private static List<File> resolveClasspath() {
212        List<File> classpath = new ArrayList<>();
213        for (ClassLoader cl = Thread.currentThread().getContextClassLoader(); cl != null; cl = cl.getParent()) {
214            if (cl instanceof URLClassLoader) {
215                for (URL url : ((URLClassLoader) cl).getURLs()) {
216                    if ("file".equals(url.getProtocol())) {
217                        try {
218                            classpath.add(new File(url.toURI()));
219                        } catch (URISyntaxException e) {
220                            classpath.add(new File(url.getPath()));
221                        }
222                    }
223                }
224            }
225        }
226        for (String element : System.getProperty("java.class.path").split(File.pathSeparator)) {
227            classpath.add(new File(element));
228        }
229        return classpath;
230    }
231
232    /**
233     * Reads back the intermediate properties file produced by {@link ConfigurationCollectorDoclet} into the list of
234     * maps consumed by the Velocity templates.
235     */
236    protected List<Map<String, String>> readDiscoveredKeys(Path intermediateFile) throws Exception {
237        Properties properties = new Properties();
238        try (Reader reader = Files.newBufferedReader(intermediateFile, StandardCharsets.UTF_8)) {
239            properties.load(reader);
240        }
241        int count = Integer.parseInt(properties.getProperty("keys.count", "0"));
242        List<Map<String, String>> discoveredKeys = new ArrayList<>(count);
243        for (int i = 0; i < count; i++) {
244            Map<String, String> entry = new LinkedHashMap<>();
245            for (String field : FIELDS) {
246                entry.put(field, properties.getProperty("keys." + i + "." + field, ""));
247            }
248            discoveredKeys.add(entry);
249        }
250        return discoveredKeys;
251    }
252
253    protected void render(List<Map<String, String>> discoveredKeys) throws Exception {
254        Properties properties = new Properties();
255        properties.setProperty("resource.loaders", "classpath");
256        properties.setProperty("resource.loader.classpath.class", ClasspathResourceLoader.class.getName());
257        VelocityEngine velocityEngine = new VelocityEngine();
258        velocityEngine.init(properties);
259
260        VelocityContext context = new VelocityContext();
261        context.put("keys", discoveredKeys);
262
263        for (String template : templates) {
264            Path output = outputDirectory.resolve(template);
265            Files.createDirectories(output.getParent());
266            System.out.println("Writing out to " + output);
267            try (Writer fileWriter = new CachingWriter(output, StandardCharsets.UTF_8)) {
268                velocityEngine.getTemplate(template + ".vm").merge(context, fileWriter);
269            }
270        }
271    }
272}