1 package org.apache.maven.plugins.shade;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.BufferedOutputStream;
23 import java.io.ByteArrayInputStream;
24 import java.io.File;
25 import java.io.FileOutputStream;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.InputStreamReader;
29 import java.io.OutputStreamWriter;
30 import java.io.PushbackInputStream;
31 import java.io.Writer;
32 import java.nio.charset.StandardCharsets;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.Enumeration;
38 import java.util.HashSet;
39 import java.util.Iterator;
40 import java.util.LinkedList;
41 import java.util.List;
42 import java.util.Objects;
43 import java.util.Set;
44 import java.util.jar.JarEntry;
45 import java.util.jar.JarFile;
46 import java.util.jar.JarOutputStream;
47 import java.util.regex.Matcher;
48 import java.util.regex.Pattern;
49 import java.util.zip.CRC32;
50 import java.util.zip.ZipEntry;
51 import java.util.zip.ZipException;
52
53 import javax.inject.Named;
54 import javax.inject.Singleton;
55
56 import org.apache.commons.collections4.MultiValuedMap;
57 import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
58 import org.apache.commons.lang3.StringUtils;
59 import org.apache.maven.plugin.MojoExecutionException;
60 import org.apache.maven.plugins.shade.filter.Filter;
61 import org.apache.maven.plugins.shade.relocation.Relocator;
62 import org.apache.maven.plugins.shade.resource.ManifestResourceTransformer;
63 import org.apache.maven.plugins.shade.resource.ReproducibleResourceTransformer;
64 import org.apache.maven.plugins.shade.resource.ResourceTransformer;
65 import org.codehaus.plexus.util.IOUtil;
66 import org.objectweb.asm.ClassReader;
67 import org.objectweb.asm.ClassVisitor;
68 import org.objectweb.asm.ClassWriter;
69 import org.objectweb.asm.commons.ClassRemapper;
70 import org.objectweb.asm.commons.Remapper;
71 import org.slf4j.Logger;
72 import org.slf4j.LoggerFactory;
73
74
75
76
77 @Singleton
78 @Named
79 public class DefaultShader
80 implements Shader
81 {
82 private static final int BUFFER_SIZE = 32 * 1024;
83
84 private final Logger logger;
85
86 public DefaultShader()
87 {
88 this( LoggerFactory.getLogger( DefaultShader.class ) );
89 }
90
91 public DefaultShader( final Logger logger )
92 {
93 this.logger = Objects.requireNonNull( logger );
94 }
95
96 public void shade( ShadeRequest shadeRequest )
97 throws IOException, MojoExecutionException
98 {
99 Set<String> resources = new HashSet<>();
100
101 ManifestResourceTransformer manifestTransformer = null;
102 List<ResourceTransformer> transformers =
103 new ArrayList<>( shadeRequest.getResourceTransformers() );
104 for ( Iterator<ResourceTransformer> it = transformers.iterator(); it.hasNext(); )
105 {
106 ResourceTransformer transformer = it.next();
107 if ( transformer instanceof ManifestResourceTransformer )
108 {
109 manifestTransformer = (ManifestResourceTransformer) transformer;
110 it.remove();
111 }
112 }
113
114 final DefaultPackageMapper packageMapper = new DefaultPackageMapper( shadeRequest.getRelocators() );
115
116
117 shadeRequest.getUberJar().getParentFile().mkdirs();
118
119 try ( JarOutputStream out =
120 new JarOutputStream( new BufferedOutputStream( new FileOutputStream( shadeRequest.getUberJar() ) ) ) )
121 {
122 goThroughAllJarEntriesForManifestTransformer( shadeRequest, resources, manifestTransformer, out );
123
124
125 MultiValuedMap<String, File> duplicates = new HashSetValuedHashMap<>( 10000, 3 );
126
127
128 shadeJars( shadeRequest, resources, transformers, out, duplicates, packageMapper );
129
130
131 MultiValuedMap<Collection<File>, String> overlapping = new HashSetValuedHashMap<>( 20, 15 );
132
133
134 for ( String clazz : duplicates.keySet() )
135 {
136 Collection<File> jarz = duplicates.get( clazz );
137 if ( jarz.size() > 1 )
138 {
139 overlapping.put( jarz, clazz );
140 }
141 }
142
143
144 logSummaryOfDuplicates( overlapping );
145
146 if ( overlapping.keySet().size() > 0 )
147 {
148 showOverlappingWarning();
149 }
150
151 for ( ResourceTransformer transformer : transformers )
152 {
153 if ( transformer.hasTransformedResource() )
154 {
155 transformer.modifyOutputStream( out );
156 }
157 }
158 }
159
160 for ( Filter filter : shadeRequest.getFilters() )
161 {
162 filter.finished();
163 }
164 }
165
166
167
168
169 private static class ZipHeaderPeekInputStream extends PushbackInputStream
170 {
171
172 private static final byte[] ZIP_HEADER = new byte[] {0x50, 0x4b, 0x03, 0x04};
173
174 private static final int HEADER_LEN = 4;
175
176 protected ZipHeaderPeekInputStream( InputStream in )
177 {
178 super( in, HEADER_LEN );
179 }
180
181 public boolean hasZipHeader() throws IOException
182 {
183 final byte[] header = new byte[HEADER_LEN];
184 super.read( header, 0, HEADER_LEN );
185 super.unread( header );
186 return Arrays.equals( header, ZIP_HEADER );
187 }
188 }
189
190
191
192
193 private static class CrcAndSize
194 {
195
196 private final CRC32 crc = new CRC32();
197
198 private long size;
199
200 CrcAndSize( InputStream inputStream ) throws IOException
201 {
202 load( inputStream );
203 }
204
205 private void load( InputStream inputStream ) throws IOException
206 {
207 byte[] buffer = new byte[BUFFER_SIZE];
208 int bytesRead;
209 while ( ( bytesRead = inputStream.read( buffer ) ) != -1 )
210 {
211 this.crc.update( buffer, 0, bytesRead );
212 this.size += bytesRead;
213 }
214 }
215
216 public void setupStoredEntry( JarEntry entry )
217 {
218 entry.setSize( this.size );
219 entry.setCompressedSize( this.size );
220 entry.setCrc( this.crc.getValue() );
221 entry.setMethod( ZipEntry.STORED );
222 }
223 }
224
225 private void shadeJars( ShadeRequest shadeRequest, Set<String> resources, List<ResourceTransformer> transformers,
226 JarOutputStream jos, MultiValuedMap<String, File> duplicates,
227 DefaultPackageMapper packageMapper )
228 throws IOException
229 {
230 for ( File jar : shadeRequest.getJars() )
231 {
232
233 logger.debug( "Processing JAR " + jar );
234
235 List<Filter> jarFilters = getFilters( jar, shadeRequest.getFilters() );
236
237 try ( JarFile jarFile = newJarFile( jar ) )
238 {
239
240 for ( Enumeration<JarEntry> j = jarFile.entries(); j.hasMoreElements(); )
241 {
242 JarEntry entry = j.nextElement();
243
244 String name = entry.getName();
245
246 if ( entry.isDirectory() || isFiltered( jarFilters, name ) )
247 {
248 continue;
249 }
250
251
252 if ( "META-INF/INDEX.LIST".equals( name ) )
253 {
254
255
256
257 continue;
258 }
259
260 if ( "module-info.class".equals( name ) )
261 {
262 logger.warn( "Discovered module-info.class. "
263 + "Shading will break its strong encapsulation." );
264 continue;
265 }
266
267 try
268 {
269 shadeJarEntry( shadeRequest, resources, transformers, packageMapper, jos, duplicates, jar,
270 jarFile, entry, name );
271 }
272 catch ( Exception e )
273 {
274 throw new IOException( String.format( "Problem shading JAR %s entry %s: %s", jar, name, e ),
275 e );
276 }
277 }
278
279 }
280 }
281 }
282
283 private void shadeJarEntry( ShadeRequest shadeRequest, Set<String> resources,
284 List<ResourceTransformer> transformers, DefaultPackageMapper packageMapper,
285 JarOutputStream jos, MultiValuedMap<String, File> duplicates, File jar,
286 JarFile jarFile, JarEntry entry, String name )
287 throws IOException, MojoExecutionException
288 {
289 try ( InputStream in = jarFile.getInputStream( entry ) )
290 {
291 String mappedName = packageMapper.map( name, true, false );
292
293 int idx = mappedName.lastIndexOf( '/' );
294 if ( idx != -1 )
295 {
296
297 String dir = mappedName.substring( 0, idx );
298 if ( !resources.contains( dir ) )
299 {
300 addDirectory( resources, jos, dir, entry.getTime() );
301 }
302 }
303
304 duplicates.put( name, jar );
305 if ( name.endsWith( ".class" ) )
306 {
307 addRemappedClass( jos, jar, name, entry.getTime(), in, packageMapper );
308 }
309 else if ( shadeRequest.isShadeSourcesContent() && name.endsWith( ".java" ) )
310 {
311
312 if ( resources.contains( mappedName ) )
313 {
314 return;
315 }
316
317 addJavaSource( resources, jos, mappedName, entry.getTime(), in, shadeRequest.getRelocators() );
318 }
319 else
320 {
321 if ( !resourceTransformed( transformers, mappedName, in, shadeRequest.getRelocators(),
322 entry.getTime() ) )
323 {
324
325 if ( resources.contains( mappedName ) )
326 {
327 logger.debug( "We have a duplicate " + name + " in " + jar );
328 return;
329 }
330
331 addResource( resources, jos, mappedName, entry, jarFile );
332 }
333 else
334 {
335 duplicates.removeMapping( name, jar );
336 }
337 }
338 }
339 }
340
341 private void goThroughAllJarEntriesForManifestTransformer( ShadeRequest shadeRequest, Set<String> resources,
342 ManifestResourceTransformer manifestTransformer,
343 JarOutputStream jos )
344 throws IOException
345 {
346 if ( manifestTransformer != null )
347 {
348 for ( File jar : shadeRequest.getJars() )
349 {
350 try ( JarFile jarFile = newJarFile( jar ) )
351 {
352 for ( Enumeration<JarEntry> en = jarFile.entries(); en.hasMoreElements(); )
353 {
354 JarEntry entry = en.nextElement();
355 String resource = entry.getName();
356 if ( manifestTransformer.canTransformResource( resource ) )
357 {
358 resources.add( resource );
359 try ( InputStream inputStream = jarFile.getInputStream( entry ) )
360 {
361 manifestTransformer.processResource( resource, inputStream,
362 shadeRequest.getRelocators(), entry.getTime() );
363 }
364 break;
365 }
366 }
367 }
368 }
369 if ( manifestTransformer.hasTransformedResource() )
370 {
371 manifestTransformer.modifyOutputStream( jos );
372 }
373 }
374 }
375
376 private void showOverlappingWarning()
377 {
378 logger.warn( "maven-shade-plugin has detected that some class files are" );
379 logger.warn( "present in two or more JARs. When this happens, only one" );
380 logger.warn( "single version of the class is copied to the uber jar." );
381 logger.warn( "Usually this is not harmful and you can skip these warnings," );
382 logger.warn( "otherwise try to manually exclude artifacts based on" );
383 logger.warn( "mvn dependency:tree -Ddetail=true and the above output." );
384 logger.warn( "See https://maven.apache.org/plugins/maven-shade-plugin/" );
385 }
386
387 private void logSummaryOfDuplicates( MultiValuedMap<Collection<File>, String> overlapping )
388 {
389 for ( Collection<File> jarz : overlapping.keySet() )
390 {
391 List<String> jarzS = new ArrayList<>();
392
393 for ( File jjar : jarz )
394 {
395 jarzS.add( jjar.getName() );
396 }
397
398 Collections.sort( jarzS );
399
400 List<String> classes = new LinkedList<>();
401 List<String> resources = new LinkedList<>();
402
403 for ( String name : overlapping.get( jarz ) )
404 {
405 if ( name.endsWith( ".class" ) )
406 {
407 classes.add( name.replace( ".class", "" ).replace( "/", "." ) );
408 }
409 else
410 {
411 resources.add( name );
412 }
413 }
414
415
416 final Collection<String> overlaps = new ArrayList<>();
417 if ( !classes.isEmpty() )
418 {
419 if ( resources.size() == 1 )
420 {
421 overlaps.add( "class" );
422 }
423 else
424 {
425 overlaps.add( "classes" );
426 }
427 }
428 if ( !resources.isEmpty() )
429 {
430 if ( resources.size() == 1 )
431 {
432 overlaps.add( "resource" );
433 }
434 else
435 {
436 overlaps.add( "resources" );
437 }
438 }
439
440 final List<String> all = new ArrayList<>( classes.size() + resources.size() );
441 all.addAll( classes );
442 all.addAll( resources );
443
444 logger.warn(
445 StringUtils.join( jarzS, ", " ) + " define " + all.size()
446 + " overlapping " + StringUtils.join( overlaps, " and " ) + ": " );
447
448
449 Collections.sort( all );
450
451 int max = 10;
452
453 for ( int i = 0; i < Math.min( max, all.size() ); i++ )
454 {
455 logger.warn( " - " + all.get( i ) );
456 }
457
458 if ( all.size() > max )
459 {
460 logger.warn( " - " + ( all.size() - max ) + " more..." );
461 }
462
463 }
464 }
465
466 private JarFile newJarFile( File jar )
467 throws IOException
468 {
469 try
470 {
471 return new JarFile( jar );
472 }
473 catch ( ZipException zex )
474 {
475
476
477 throw new ZipException( "error in opening zip file " + jar );
478 }
479 }
480
481 private List<Filter> getFilters( File jar, List<Filter> filters )
482 {
483 List<Filter> list = new ArrayList<>();
484
485 for ( Filter filter : filters )
486 {
487 if ( filter.canFilter( jar ) )
488 {
489 list.add( filter );
490 }
491
492 }
493
494 return list;
495 }
496
497 private void addDirectory( Set<String> resources, JarOutputStream jos, String name, long time )
498 throws IOException
499 {
500 if ( name.lastIndexOf( '/' ) > 0 )
501 {
502 String parent = name.substring( 0, name.lastIndexOf( '/' ) );
503 if ( !resources.contains( parent ) )
504 {
505 addDirectory( resources, jos, parent, time );
506 }
507 }
508
509
510 JarEntry entry = new JarEntry( name + "/" );
511 entry.setTime( time );
512 jos.putNextEntry( entry );
513
514 resources.add( name );
515 }
516
517 private void addRemappedClass( JarOutputStream jos, File jar, String name,
518 long time, InputStream is, DefaultPackageMapper packageMapper )
519 throws IOException, MojoExecutionException
520 {
521 if ( packageMapper.relocators.isEmpty() )
522 {
523 try
524 {
525 JarEntry entry = new JarEntry( name );
526 entry.setTime( time );
527 jos.putNextEntry( entry );
528 IOUtil.copy( is, jos );
529 }
530 catch ( ZipException e )
531 {
532 logger.debug( "We have a duplicate " + name + " in " + jar );
533 }
534
535 return;
536 }
537
538
539
540
541 byte[] originalClass = IOUtil.toByteArray( is );
542
543 ClassReader cr = new ClassReader( new ByteArrayInputStream( originalClass ) );
544
545
546
547
548
549
550 ClassWriter cw = new ClassWriter( 0 );
551
552 final String pkg = name.substring( 0, name.lastIndexOf( '/' ) + 1 );
553 final ShadeClassRemapper cv = new ShadeClassRemapper( cw, pkg, packageMapper );
554
555 try
556 {
557 cr.accept( cv, ClassReader.EXPAND_FRAMES );
558 }
559 catch ( Throwable ise )
560 {
561 throw new MojoExecutionException( "Error in ASM processing class " + name, ise );
562 }
563
564
565 final byte[] renamedClass;
566 if ( cv.remapped )
567 {
568 logger.debug( "Rewrote class bytecode: " + name );
569 renamedClass = cw.toByteArray();
570 }
571 else
572 {
573 logger.debug( "Keeping original class bytecode: " + name );
574 renamedClass = originalClass;
575 }
576
577
578 String mappedName = packageMapper.map( name.substring( 0, name.indexOf( '.' ) ), true, false );
579
580 try
581 {
582
583 JarEntry entry = new JarEntry( mappedName + ".class" );
584 entry.setTime( time );
585 jos.putNextEntry( entry );
586
587 jos.write( renamedClass );
588 }
589 catch ( ZipException e )
590 {
591 logger.debug( "We have a duplicate " + mappedName + " in " + jar );
592 }
593 }
594
595 private boolean isFiltered( List<Filter> filters, String name )
596 {
597 for ( Filter filter : filters )
598 {
599 if ( filter.isFiltered( name ) )
600 {
601 return true;
602 }
603 }
604
605 return false;
606 }
607
608 private boolean resourceTransformed( List<ResourceTransformer> resourceTransformers, String name, InputStream is,
609 List<Relocator> relocators, long time )
610 throws IOException
611 {
612 boolean resourceTransformed = false;
613
614 for ( ResourceTransformer transformer : resourceTransformers )
615 {
616 if ( transformer.canTransformResource( name ) )
617 {
618 logger.debug( "Transforming " + name + " using " + transformer.getClass().getName() );
619
620 if ( transformer instanceof ReproducibleResourceTransformer )
621 {
622 ( (ReproducibleResourceTransformer) transformer ).processResource( name, is, relocators, time );
623 }
624 else
625 {
626 transformer.processResource( name, is, relocators );
627 }
628
629 resourceTransformed = true;
630
631 break;
632 }
633 }
634 return resourceTransformed;
635 }
636
637 private void addJavaSource( Set<String> resources, JarOutputStream jos, String name, long time, InputStream is,
638 List<Relocator> relocators )
639 throws IOException
640 {
641 JarEntry entry = new JarEntry( name );
642 entry.setTime( time );
643 jos.putNextEntry( entry );
644
645 String sourceContent = IOUtil.toString( new InputStreamReader( is, StandardCharsets.UTF_8 ) );
646
647 for ( Relocator relocator : relocators )
648 {
649 sourceContent = relocator.applyToSourceContent( sourceContent );
650 }
651
652 final Writer writer = new OutputStreamWriter( jos, StandardCharsets.UTF_8 );
653 writer.write( sourceContent );
654 writer.flush();
655
656 resources.add( name );
657 }
658
659 private void addResource( Set<String> resources, JarOutputStream jos, String name, JarEntry originalEntry,
660 JarFile jarFile ) throws IOException
661 {
662 ZipHeaderPeekInputStream inputStream = new ZipHeaderPeekInputStream( jarFile.getInputStream( originalEntry ) );
663 try
664 {
665 final JarEntry entry = new JarEntry( name );
666
667
668 if ( inputStream.hasZipHeader() && originalEntry.getMethod() == ZipEntry.STORED )
669 {
670 new CrcAndSize( inputStream ).setupStoredEntry( entry );
671 inputStream.close();
672 inputStream = new ZipHeaderPeekInputStream( jarFile.getInputStream( originalEntry ) );
673 }
674
675
676 entry.setTime( originalEntry.getTime() );
677
678 jos.putNextEntry( entry );
679
680 IOUtil.copy( inputStream, jos );
681
682 resources.add( name );
683 }
684 finally
685 {
686 inputStream.close();
687 }
688 }
689
690 private interface PackageMapper
691 {
692
693
694
695
696
697
698
699
700 String map( String entityName, boolean mapPaths, boolean mapPackages );
701 }
702
703
704
705
706 private static class DefaultPackageMapper implements PackageMapper
707 {
708 private static final Pattern CLASS_PATTERN = Pattern.compile( "(\\[*)?L(.+);" );
709
710 private final List<Relocator> relocators;
711
712 private DefaultPackageMapper( final List<Relocator> relocators )
713 {
714 this.relocators = relocators;
715 }
716
717 @Override
718 public String map( String entityName, boolean mapPaths, final boolean mapPackages )
719 {
720 String value = entityName;
721
722 String prefix = "";
723 String suffix = "";
724
725 Matcher m = CLASS_PATTERN.matcher( entityName );
726 if ( m.matches() )
727 {
728 prefix = m.group( 1 ) + "L";
729 suffix = ";";
730 entityName = m.group( 2 );
731 }
732
733 for ( Relocator r : relocators )
734 {
735 if ( mapPackages && r.canRelocateClass( entityName ) )
736 {
737 value = prefix + r.relocateClass( entityName ) + suffix;
738 break;
739 }
740 else if ( mapPaths && r.canRelocatePath( entityName ) )
741 {
742 value = prefix + r.relocatePath( entityName ) + suffix;
743 break;
744 }
745 }
746 return value;
747 }
748 }
749
750 private static class LazyInitRemapper extends Remapper
751 {
752 private PackageMapper relocators;
753
754 @Override
755 public Object mapValue( Object object )
756 {
757 return object instanceof String
758 ? relocators.map( (String) object, true, true )
759 : super.mapValue( object );
760 }
761
762 @Override
763 public String map( String name )
764 {
765
766
767
768
769
770
771
772
773 return relocators.map( name, true, false );
774 }
775 }
776
777
778
779
780
781
782
783
784
785
786
787 private static class ShadeClassRemapper extends ClassRemapper implements PackageMapper
788 {
789 private final String pkg;
790 private final PackageMapper packageMapper;
791 private boolean remapped;
792
793 ShadeClassRemapper( final ClassVisitor classVisitor, final String pkg,
794 final DefaultPackageMapper packageMapper )
795 {
796 super( classVisitor, new LazyInitRemapper() );
797 this.pkg = pkg;
798 this.packageMapper = packageMapper;
799
800
801 LazyInitRemapper.class.cast( remapper ).relocators = this;
802 }
803
804 @Override
805 public void visitSource( final String source, final String debug )
806 {
807 if ( source == null )
808 {
809 super.visitSource( null, debug );
810 return;
811 }
812
813 final String fqSource = pkg + source;
814 final String mappedSource = map( fqSource, true, false );
815 final String filename = mappedSource.substring( mappedSource.lastIndexOf( '/' ) + 1 );
816 super.visitSource( filename, debug );
817 }
818
819 @Override
820 public String map( final String entityName, boolean mapPaths, final boolean mapPackages )
821 {
822 final String mapped = packageMapper.map( entityName, true, mapPackages );
823 if ( !remapped )
824 {
825 remapped = !mapped.equals( entityName );
826 }
827 return mapped;
828 }
829 }
830 }