1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.plugins.shade;
20
21 import javax.inject.Named;
22 import javax.inject.Singleton;
23
24 import java.io.BufferedOutputStream;
25 import java.io.ByteArrayInputStream;
26 import java.io.File;
27 import java.io.FileInputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.InputStreamReader;
31 import java.io.OutputStreamWriter;
32 import java.io.PushbackInputStream;
33 import java.io.Writer;
34 import java.nio.charset.StandardCharsets;
35 import java.util.ArrayList;
36 import java.util.Arrays;
37 import java.util.Collection;
38 import java.util.Collections;
39 import java.util.Enumeration;
40 import java.util.HashSet;
41 import java.util.Iterator;
42 import java.util.LinkedList;
43 import java.util.List;
44 import java.util.Objects;
45 import java.util.Set;
46 import java.util.TimeZone;
47 import java.util.concurrent.Callable;
48 import java.util.jar.JarEntry;
49 import java.util.jar.JarFile;
50 import java.util.jar.JarOutputStream;
51 import java.util.regex.Matcher;
52 import java.util.regex.Pattern;
53 import java.util.zip.CRC32;
54 import java.util.zip.ZipEntry;
55 import java.util.zip.ZipException;
56
57 import org.apache.commons.collections4.MultiValuedMap;
58 import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
59 import org.apache.commons.compress.archivers.zip.ExtraFieldUtils;
60 import org.apache.commons.compress.archivers.zip.X5455_ExtendedTimestamp;
61 import org.apache.commons.compress.archivers.zip.ZipExtraField;
62 import org.apache.maven.plugin.MojoExecutionException;
63 import org.apache.maven.plugins.shade.filter.Filter;
64 import org.apache.maven.plugins.shade.relocation.Relocator;
65 import org.apache.maven.plugins.shade.resource.ManifestResourceTransformer;
66 import org.apache.maven.plugins.shade.resource.ReproducibleResourceTransformer;
67 import org.apache.maven.plugins.shade.resource.ResourceTransformer;
68 import org.codehaus.plexus.util.IOUtil;
69 import org.codehaus.plexus.util.io.CachingOutputStream;
70 import org.objectweb.asm.ClassReader;
71 import org.objectweb.asm.ClassVisitor;
72 import org.objectweb.asm.ClassWriter;
73 import org.objectweb.asm.commons.ClassRemapper;
74 import org.objectweb.asm.commons.Remapper;
75 import org.slf4j.Logger;
76 import org.slf4j.LoggerFactory;
77
78
79
80
81 @Singleton
82 @Named
83 public class DefaultShader implements Shader {
84 private static final int BUFFER_SIZE = 32 * 1024;
85
86 private final Logger logger;
87
88 public DefaultShader() {
89 this(LoggerFactory.getLogger(DefaultShader.class));
90 }
91
92 public DefaultShader(final Logger logger) {
93 this.logger = Objects.requireNonNull(logger);
94 }
95
96
97 private long getTime(ZipEntry entry) {
98 if (entry.getExtra() != null) {
99 try {
100 ZipExtraField[] fields =
101 ExtraFieldUtils.parse(entry.getExtra(), true, ExtraFieldUtils.UnparseableExtraField.SKIP);
102 for (ZipExtraField field : fields) {
103 if (X5455_ExtendedTimestamp.HEADER_ID.equals(field.getHeaderId())) {
104
105 return entry.getTime() - TimeZone.getDefault().getRawOffset();
106 }
107 }
108 } catch (ZipException ze) {
109
110 }
111 }
112 return entry.getTime();
113 }
114
115 public void shade(ShadeRequest shadeRequest) throws IOException, MojoExecutionException {
116 Set<String> resources = new HashSet<>();
117
118 ManifestResourceTransformer manifestTransformer = null;
119 List<ResourceTransformer> transformers = new ArrayList<>(shadeRequest.getResourceTransformers());
120 for (Iterator<ResourceTransformer> it = transformers.iterator(); it.hasNext(); ) {
121 ResourceTransformer transformer = it.next();
122 if (transformer instanceof ManifestResourceTransformer) {
123 manifestTransformer = (ManifestResourceTransformer) transformer;
124 it.remove();
125 }
126 }
127
128 final DefaultPackageMapper packageMapper = new DefaultPackageMapper(shadeRequest.getRelocators());
129
130
131 shadeRequest.getUberJar().getParentFile().mkdirs();
132
133 try (JarOutputStream out =
134 new JarOutputStream(new BufferedOutputStream(new CachingOutputStream(shadeRequest.getUberJar())))) {
135 goThroughAllJarEntriesForManifestTransformer(shadeRequest, resources, manifestTransformer, out);
136
137
138 MultiValuedMap<String, File> duplicates = new HashSetValuedHashMap<>(10000, 3);
139
140
141 shadeJars(shadeRequest, resources, transformers, out, duplicates, packageMapper);
142
143
144 MultiValuedMap<Collection<File>, String> overlapping = new HashSetValuedHashMap<>(20, 15);
145
146
147 for (String clazz : duplicates.keySet()) {
148 Collection<File> jarz = duplicates.get(clazz);
149 if (jarz.size() > 1) {
150 overlapping.put(jarz, clazz);
151 }
152 }
153
154
155 logSummaryOfDuplicates(overlapping);
156
157 if (overlapping.keySet().size() > 0) {
158 showOverlappingWarning();
159 }
160
161 for (ResourceTransformer transformer : transformers) {
162 if (transformer.hasTransformedResource()) {
163 transformer.modifyOutputStream(out);
164 }
165 }
166 }
167
168 for (Filter filter : shadeRequest.getFilters()) {
169 filter.finished();
170 }
171 }
172
173
174
175
176 private static class ZipHeaderPeekInputStream extends PushbackInputStream {
177
178 private static final byte[] ZIP_HEADER = new byte[] {0x50, 0x4b, 0x03, 0x04};
179
180 private static final int HEADER_LEN = 4;
181
182 protected ZipHeaderPeekInputStream(InputStream in) {
183 super(in, HEADER_LEN);
184 }
185
186 public boolean hasZipHeader() throws IOException {
187 final byte[] header = new byte[HEADER_LEN];
188 int len = super.read(header, 0, HEADER_LEN);
189 if (len != -1) {
190 super.unread(header, 0, len);
191 }
192 return Arrays.equals(header, ZIP_HEADER);
193 }
194 }
195
196
197
198
199 private static class CrcAndSize {
200
201 private final CRC32 crc = new CRC32();
202
203 private long size;
204
205 CrcAndSize(InputStream inputStream) throws IOException {
206 load(inputStream);
207 }
208
209 private void load(InputStream inputStream) throws IOException {
210 byte[] buffer = new byte[BUFFER_SIZE];
211 int bytesRead;
212 while ((bytesRead = inputStream.read(buffer)) != -1) {
213 this.crc.update(buffer, 0, bytesRead);
214 this.size += bytesRead;
215 }
216 }
217
218 public void setupStoredEntry(JarEntry entry) {
219 entry.setSize(this.size);
220 entry.setCompressedSize(this.size);
221 entry.setCrc(this.crc.getValue());
222 entry.setMethod(ZipEntry.STORED);
223 }
224 }
225
226 private void shadeJars(
227 ShadeRequest shadeRequest,
228 Set<String> resources,
229 List<ResourceTransformer> transformers,
230 JarOutputStream jos,
231 MultiValuedMap<String, File> duplicates,
232 DefaultPackageMapper packageMapper)
233 throws IOException {
234 for (File jar : shadeRequest.getJars()) {
235
236 logger.debug("Processing JAR " + jar);
237
238 List<Filter> jarFilters = getFilters(jar, shadeRequest.getFilters());
239 if (jar.isDirectory()) {
240 shadeDir(
241 shadeRequest,
242 resources,
243 transformers,
244 packageMapper,
245 jos,
246 duplicates,
247 jar,
248 jar,
249 "",
250 jarFilters);
251 } else {
252 shadeJar(shadeRequest, resources, transformers, packageMapper, jos, duplicates, jar, jarFilters);
253 }
254 }
255 }
256
257 private void shadeDir(
258 ShadeRequest shadeRequest,
259 Set<String> resources,
260 List<ResourceTransformer> transformers,
261 DefaultPackageMapper packageMapper,
262 JarOutputStream jos,
263 MultiValuedMap<String, File> duplicates,
264 File jar,
265 File current,
266 String prefix,
267 List<Filter> jarFilters)
268 throws IOException {
269 final File[] children = current.listFiles();
270 if (children == null) {
271 return;
272 }
273 for (final File file : children) {
274 final String name = prefix + file.getName();
275 if (file.isDirectory()) {
276 try {
277 shadeDir(
278 shadeRequest,
279 resources,
280 transformers,
281 packageMapper,
282 jos,
283 duplicates,
284 jar,
285 file,
286 prefix + file.getName() + '/',
287 jarFilters);
288 continue;
289 } catch (Exception e) {
290 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", current, name, e), e);
291 }
292 }
293 if (isFiltered(jarFilters, name) || isExcludedEntry(name)) {
294 continue;
295 }
296
297 try {
298 shadeJarEntry(
299 shadeRequest,
300 resources,
301 transformers,
302 packageMapper,
303 jos,
304 duplicates,
305 jar,
306 new Callable<InputStream>() {
307 @Override
308 public InputStream call() throws Exception {
309 return new FileInputStream(file);
310 }
311 },
312 name,
313 file.lastModified(),
314 -1 );
315 } catch (Exception e) {
316 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", current, name, e), e);
317 }
318 }
319 }
320
321 private void shadeJar(
322 ShadeRequest shadeRequest,
323 Set<String> resources,
324 List<ResourceTransformer> transformers,
325 DefaultPackageMapper packageMapper,
326 JarOutputStream jos,
327 MultiValuedMap<String, File> duplicates,
328 File jar,
329 List<Filter> jarFilters)
330 throws IOException {
331 try (JarFile jarFile = newJarFile(jar)) {
332
333 for (Enumeration<JarEntry> j = jarFile.entries(); j.hasMoreElements(); ) {
334 final JarEntry entry = j.nextElement();
335
336 String name = entry.getName();
337
338 if (entry.isDirectory() || isFiltered(jarFilters, name) || isExcludedEntry(name)) {
339 continue;
340 }
341
342 try {
343 shadeJarEntry(
344 shadeRequest,
345 resources,
346 transformers,
347 packageMapper,
348 jos,
349 duplicates,
350 jar,
351 new Callable<InputStream>() {
352 @Override
353 public InputStream call() throws Exception {
354 return jarFile.getInputStream(entry);
355 }
356 },
357 name,
358 getTime(entry),
359 entry.getMethod());
360 } catch (Exception e) {
361 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", jar, name, e), e);
362 }
363 }
364 }
365 }
366
367 private boolean isExcludedEntry(final String name) {
368 if ("META-INF/INDEX.LIST".equals(name)) {
369
370
371
372 return true;
373 }
374
375 if ("module-info.class".equals(name)) {
376 logger.warn("Discovered module-info.class. " + "Shading will break its strong encapsulation.");
377 return true;
378 }
379 return false;
380 }
381
382 private void shadeJarEntry(
383 ShadeRequest shadeRequest,
384 Set<String> resources,
385 List<ResourceTransformer> transformers,
386 DefaultPackageMapper packageMapper,
387 JarOutputStream jos,
388 MultiValuedMap<String, File> duplicates,
389 File jar,
390 Callable<InputStream> inputProvider,
391 String name,
392 long time,
393 int method)
394 throws Exception {
395 try (InputStream in = inputProvider.call()) {
396 String mappedName = packageMapper.map(name, true, false);
397
398 int idx = mappedName.lastIndexOf('/');
399 if (idx != -1) {
400
401 String dir = mappedName.substring(0, idx);
402 if (!resources.contains(dir)) {
403 addDirectory(resources, jos, dir, time);
404 }
405 }
406
407 duplicates.put(name, jar);
408 if (name.endsWith(".class")) {
409 addRemappedClass(jos, jar, name, time, in, packageMapper);
410 } else if (shadeRequest.isShadeSourcesContent() && name.endsWith(".java")) {
411
412 if (resources.contains(mappedName)) {
413 return;
414 }
415
416 addJavaSource(resources, jos, mappedName, time, in, shadeRequest.getRelocators());
417 } else {
418 if (!resourceTransformed(transformers, mappedName, in, shadeRequest.getRelocators(), time)) {
419
420 if (resources.contains(mappedName)) {
421 logger.debug("We have a duplicate " + name + " in " + jar);
422 return;
423 }
424
425 addResource(resources, jos, mappedName, inputProvider, time, method);
426 } else {
427 duplicates.removeMapping(name, jar);
428 }
429 }
430 }
431 }
432
433 private void goThroughAllJarEntriesForManifestTransformer(
434 ShadeRequest shadeRequest,
435 Set<String> resources,
436 ManifestResourceTransformer manifestTransformer,
437 JarOutputStream jos)
438 throws IOException {
439 if (manifestTransformer != null) {
440 for (File jar : shadeRequest.getJars()) {
441 try (JarFile jarFile = newJarFile(jar)) {
442 for (Enumeration<JarEntry> en = jarFile.entries(); en.hasMoreElements(); ) {
443 JarEntry entry = en.nextElement();
444 String resource = entry.getName();
445 if (manifestTransformer.canTransformResource(resource)) {
446 resources.add(resource);
447 try (InputStream inputStream = jarFile.getInputStream(entry)) {
448 manifestTransformer.processResource(
449 resource, inputStream, shadeRequest.getRelocators(), getTime(entry));
450 }
451 break;
452 }
453 }
454 }
455 }
456 if (manifestTransformer.hasTransformedResource()) {
457 manifestTransformer.modifyOutputStream(jos);
458 }
459 }
460 }
461
462 private void showOverlappingWarning() {
463 logger.warn("maven-shade-plugin has detected that some files are");
464 logger.warn("present in two or more JARs. When this happens, only one");
465 logger.warn("single version of the file is copied to the uber jar.");
466 logger.warn("Usually this is not harmful and you can skip these warnings,");
467 logger.warn("otherwise try to manually exclude artifacts based on");
468 logger.warn("mvn dependency:tree -Ddetail=true and the above output.");
469 logger.warn("See https://maven.apache.org/plugins/maven-shade-plugin/");
470 }
471
472 private void logSummaryOfDuplicates(MultiValuedMap<Collection<File>, String> overlapping) {
473 for (Collection<File> jarz : overlapping.keySet()) {
474 List<String> jarzS = new ArrayList<>();
475
476 for (File jjar : jarz) {
477 jarzS.add(jjar.getName());
478 }
479
480 Collections.sort(jarzS);
481
482 List<String> classes = new LinkedList<>();
483 List<String> resources = new LinkedList<>();
484
485 for (String name : overlapping.get(jarz)) {
486 if (name.endsWith(".class")) {
487 classes.add(name.replace(".class", "").replace("/", "."));
488 } else {
489 resources.add(name);
490 }
491 }
492
493
494 final Collection<String> overlaps = new ArrayList<>();
495 if (!classes.isEmpty()) {
496 if (resources.size() == 1) {
497 overlaps.add("class");
498 } else {
499 overlaps.add("classes");
500 }
501 }
502 if (!resources.isEmpty()) {
503 if (resources.size() == 1) {
504 overlaps.add("resource");
505 } else {
506 overlaps.add("resources");
507 }
508 }
509
510 final List<String> all = new ArrayList<>(classes.size() + resources.size());
511 all.addAll(classes);
512 all.addAll(resources);
513
514 logger.warn(String.join(", ", jarzS) + " define " + all.size() + " overlapping "
515 + String.join(" and ", overlaps) + ": ");
516
517
518 Collections.sort(all);
519
520 int max = 10;
521
522 for (int i = 0; i < Math.min(max, all.size()); i++) {
523 logger.warn(" - " + all.get(i));
524 }
525
526 if (all.size() > max) {
527 logger.warn(" - " + (all.size() - max) + " more...");
528 }
529 }
530 }
531
532 private JarFile newJarFile(File jar) throws IOException {
533 try {
534 return new JarFile(jar);
535 } catch (ZipException zex) {
536
537
538 throw new ZipException("error in opening zip file " + jar);
539 }
540 }
541
542 private List<Filter> getFilters(File jar, List<Filter> filters) {
543 List<Filter> list = new ArrayList<>();
544
545 for (Filter filter : filters) {
546 if (filter.canFilter(jar)) {
547 list.add(filter);
548 }
549 }
550
551 return list;
552 }
553
554 private void addDirectory(Set<String> resources, JarOutputStream jos, String name, long time) throws IOException {
555 if (name.lastIndexOf('/') > 0) {
556 String parent = name.substring(0, name.lastIndexOf('/'));
557 if (!resources.contains(parent)) {
558 addDirectory(resources, jos, parent, time);
559 }
560 }
561
562
563 JarEntry entry = new JarEntry(name + "/");
564 entry.setTime(time);
565 jos.putNextEntry(entry);
566
567 resources.add(name);
568 }
569
570 private void addRemappedClass(
571 JarOutputStream jos, File jar, String name, long time, InputStream is, DefaultPackageMapper packageMapper)
572 throws IOException, MojoExecutionException {
573 if (packageMapper.relocators.isEmpty()) {
574 try {
575 JarEntry entry = new JarEntry(name);
576 entry.setTime(time);
577 jos.putNextEntry(entry);
578 IOUtil.copy(is, jos);
579 } catch (ZipException e) {
580 logger.debug("We have a duplicate " + name + " in " + jar);
581 }
582
583 return;
584 }
585
586
587
588
589 byte[] originalClass = IOUtil.toByteArray(is);
590
591 ClassReader cr = new ClassReader(new ByteArrayInputStream(originalClass));
592
593
594
595
596
597
598 ClassWriter cw = new ClassWriter(0);
599
600 final String pkg = name.substring(0, name.lastIndexOf('/') + 1);
601 final ShadeClassRemapper cv = new ShadeClassRemapper(cw, pkg, packageMapper);
602
603 try {
604 cr.accept(cv, ClassReader.EXPAND_FRAMES);
605 } catch (Throwable ise) {
606 throw new MojoExecutionException("Error in ASM processing class " + name, ise);
607 }
608
609
610 final byte[] renamedClass;
611 if (cv.remapped) {
612 logger.debug("Rewrote class bytecode: " + name);
613 renamedClass = cw.toByteArray();
614 } else {
615 logger.debug("Keeping original class bytecode: " + name);
616 renamedClass = originalClass;
617 }
618
619
620 String mappedName = packageMapper.map(name.substring(0, name.indexOf('.')), true, false);
621
622 try {
623
624 JarEntry entry = new JarEntry(mappedName + ".class");
625 entry.setTime(time);
626 jos.putNextEntry(entry);
627
628 jos.write(renamedClass);
629 } catch (ZipException e) {
630 logger.debug("We have a duplicate " + mappedName + " in " + jar);
631 }
632 }
633
634 private boolean isFiltered(List<Filter> filters, String name) {
635 for (Filter filter : filters) {
636 if (filter.isFiltered(name)) {
637 return true;
638 }
639 }
640
641 return false;
642 }
643
644 private boolean resourceTransformed(
645 List<ResourceTransformer> resourceTransformers,
646 String name,
647 InputStream is,
648 List<Relocator> relocators,
649 long time)
650 throws IOException {
651 boolean resourceTransformed = false;
652
653 for (ResourceTransformer transformer : resourceTransformers) {
654 if (transformer.canTransformResource(name)) {
655 logger.debug("Transforming " + name + " using "
656 + transformer.getClass().getName());
657
658 if (transformer instanceof ReproducibleResourceTransformer) {
659 ((ReproducibleResourceTransformer) transformer).processResource(name, is, relocators, time);
660 } else {
661 transformer.processResource(name, is, relocators);
662 }
663
664 resourceTransformed = true;
665
666 break;
667 }
668 }
669 return resourceTransformed;
670 }
671
672 private void addJavaSource(
673 Set<String> resources,
674 JarOutputStream jos,
675 String name,
676 long time,
677 InputStream is,
678 List<Relocator> relocators)
679 throws IOException {
680 JarEntry entry = new JarEntry(name);
681 entry.setTime(time);
682 jos.putNextEntry(entry);
683
684 String sourceContent = IOUtil.toString(new InputStreamReader(is, StandardCharsets.UTF_8));
685
686 for (Relocator relocator : relocators) {
687 sourceContent = relocator.applyToSourceContent(sourceContent);
688 }
689
690 final Writer writer = new OutputStreamWriter(jos, StandardCharsets.UTF_8);
691 writer.write(sourceContent);
692 writer.flush();
693
694 resources.add(name);
695 }
696
697 private void addResource(
698 Set<String> resources, JarOutputStream jos, String name, Callable<InputStream> input, long time, int method)
699 throws Exception {
700 ZipHeaderPeekInputStream inputStream = new ZipHeaderPeekInputStream(input.call());
701 try {
702 final JarEntry entry = new JarEntry(name);
703
704
705 if (inputStream.hasZipHeader() && method == ZipEntry.STORED) {
706 new CrcAndSize(inputStream).setupStoredEntry(entry);
707 inputStream.close();
708 inputStream = new ZipHeaderPeekInputStream(input.call());
709 }
710
711 entry.setTime(time);
712
713 jos.putNextEntry(entry);
714
715 IOUtil.copy(inputStream, jos);
716
717 resources.add(name);
718 } finally {
719 inputStream.close();
720 }
721 }
722
723 private interface PackageMapper {
724
725
726
727
728
729
730
731
732 String map(String entityName, boolean mapPaths, boolean mapPackages);
733 }
734
735
736
737
738 private static class DefaultPackageMapper implements PackageMapper {
739 private static final Pattern CLASS_PATTERN = Pattern.compile("(\\[*)?L(.+);");
740
741 private final List<Relocator> relocators;
742
743 private DefaultPackageMapper(final List<Relocator> relocators) {
744 this.relocators = relocators;
745 }
746
747 @Override
748 public String map(String entityName, boolean mapPaths, final boolean mapPackages) {
749 String value = entityName;
750
751 String prefix = "";
752 String suffix = "";
753
754 Matcher m = CLASS_PATTERN.matcher(entityName);
755 if (m.matches()) {
756 prefix = m.group(1) + "L";
757 suffix = ";";
758 entityName = m.group(2);
759 }
760
761 for (Relocator r : relocators) {
762 if (mapPackages && r.canRelocateClass(entityName)) {
763 value = prefix + r.relocateClass(entityName) + suffix;
764 break;
765 } else if (mapPaths && r.canRelocatePath(entityName)) {
766 value = prefix + r.relocatePath(entityName) + suffix;
767 break;
768 }
769 }
770 return value;
771 }
772 }
773
774 private static class LazyInitRemapper extends Remapper {
775 private PackageMapper relocators;
776
777 @Override
778 public Object mapValue(Object object) {
779 return object instanceof String ? relocators.map((String) object, true, true) : super.mapValue(object);
780 }
781
782 @Override
783 public String map(String name) {
784
785
786
787
788
789
790
791
792 return relocators.map(name, true, false);
793 }
794 }
795
796
797
798
799
800
801
802
803
804
805
806 private static class ShadeClassRemapper extends ClassRemapper implements PackageMapper {
807 private final String pkg;
808 private final PackageMapper packageMapper;
809 private boolean remapped;
810
811 ShadeClassRemapper(
812 final ClassVisitor classVisitor, final String pkg, final DefaultPackageMapper packageMapper) {
813 super(classVisitor, new LazyInitRemapper() );
814 this.pkg = pkg;
815 this.packageMapper = packageMapper;
816
817
818 LazyInitRemapper.class.cast(remapper).relocators = this;
819 }
820
821 @Override
822 public void visitSource(final String source, final String debug) {
823 if (source == null) {
824 super.visitSource(null, debug);
825 return;
826 }
827
828 final String fqSource = pkg + source;
829 final String mappedSource = map(fqSource, true, false);
830 final String filename = mappedSource.substring(mappedSource.lastIndexOf('/') + 1);
831 super.visitSource(filename, debug);
832 }
833
834 @Override
835 public String map(final String entityName, boolean mapPaths, final boolean mapPackages) {
836 final String mapped = packageMapper.map(entityName, true, mapPackages);
837 if (!remapped) {
838 remapped = !mapped.equals(entityName);
839 }
840 return mapped;
841 }
842 }
843 }