1 package org.apache.maven.plugins.shade;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.BufferedOutputStream;
23 import java.io.ByteArrayInputStream;
24 import java.io.File;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.io.InputStreamReader;
28 import java.io.OutputStreamWriter;
29 import java.io.PushbackInputStream;
30 import java.io.Writer;
31 import java.nio.charset.StandardCharsets;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collection;
35 import java.util.Collections;
36 import java.util.Enumeration;
37 import java.util.HashSet;
38 import java.util.Iterator;
39 import java.util.LinkedList;
40 import java.util.List;
41 import java.util.Objects;
42 import java.util.Set;
43 import java.util.jar.JarEntry;
44 import java.util.jar.JarFile;
45 import java.util.jar.JarOutputStream;
46 import java.util.regex.Matcher;
47 import java.util.regex.Pattern;
48 import java.util.zip.CRC32;
49 import java.util.zip.ZipEntry;
50 import java.util.zip.ZipException;
51
52 import javax.inject.Named;
53 import javax.inject.Singleton;
54
55 import org.apache.commons.collections4.MultiValuedMap;
56 import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
57 import org.apache.maven.plugin.MojoExecutionException;
58 import org.apache.maven.plugins.shade.filter.Filter;
59 import org.apache.maven.plugins.shade.relocation.Relocator;
60 import org.apache.maven.plugins.shade.resource.ManifestResourceTransformer;
61 import org.apache.maven.plugins.shade.resource.ReproducibleResourceTransformer;
62 import org.apache.maven.plugins.shade.resource.ResourceTransformer;
63 import org.codehaus.plexus.util.IOUtil;
64 import org.codehaus.plexus.util.io.CachingOutputStream;
65 import org.objectweb.asm.ClassReader;
66 import org.objectweb.asm.ClassVisitor;
67 import org.objectweb.asm.ClassWriter;
68 import org.objectweb.asm.commons.ClassRemapper;
69 import org.objectweb.asm.commons.Remapper;
70 import org.slf4j.Logger;
71 import org.slf4j.LoggerFactory;
72
73
74
75
76 @Singleton
77 @Named
78 public class DefaultShader
79 implements Shader
80 {
81 private static final int BUFFER_SIZE = 32 * 1024;
82
83 private final Logger logger;
84
85 public DefaultShader()
86 {
87 this( LoggerFactory.getLogger( DefaultShader.class ) );
88 }
89
90 public DefaultShader( final Logger logger )
91 {
92 this.logger = Objects.requireNonNull( logger );
93 }
94
95 public void shade( ShadeRequest shadeRequest )
96 throws IOException, MojoExecutionException
97 {
98 Set<String> resources = new HashSet<>();
99
100 ManifestResourceTransformer manifestTransformer = null;
101 List<ResourceTransformer> transformers =
102 new ArrayList<>( shadeRequest.getResourceTransformers() );
103 for ( Iterator<ResourceTransformer> it = transformers.iterator(); it.hasNext(); )
104 {
105 ResourceTransformer transformer = it.next();
106 if ( transformer instanceof ManifestResourceTransformer )
107 {
108 manifestTransformer = (ManifestResourceTransformer) transformer;
109 it.remove();
110 }
111 }
112
113 final DefaultPackageMapper packageMapper = new DefaultPackageMapper( shadeRequest.getRelocators() );
114
115
116 shadeRequest.getUberJar().getParentFile().mkdirs();
117
118 try ( JarOutputStream out =
119 new JarOutputStream( new BufferedOutputStream(
120 new CachingOutputStream( shadeRequest.getUberJar() ) ) ) )
121 {
122 goThroughAllJarEntriesForManifestTransformer( shadeRequest, resources, manifestTransformer, out );
123
124
125 MultiValuedMap<String, File> duplicates = new HashSetValuedHashMap<>( 10000, 3 );
126
127
128 shadeJars( shadeRequest, resources, transformers, out, duplicates, packageMapper );
129
130
131 MultiValuedMap<Collection<File>, String> overlapping = new HashSetValuedHashMap<>( 20, 15 );
132
133
134 for ( String clazz : duplicates.keySet() )
135 {
136 Collection<File> jarz = duplicates.get( clazz );
137 if ( jarz.size() > 1 )
138 {
139 overlapping.put( jarz, clazz );
140 }
141 }
142
143
144 logSummaryOfDuplicates( overlapping );
145
146 if ( overlapping.keySet().size() > 0 )
147 {
148 showOverlappingWarning();
149 }
150
151 for ( ResourceTransformer transformer : transformers )
152 {
153 if ( transformer.hasTransformedResource() )
154 {
155 transformer.modifyOutputStream( out );
156 }
157 }
158 }
159
160 for ( Filter filter : shadeRequest.getFilters() )
161 {
162 filter.finished();
163 }
164 }
165
166
167
168
169 private static class ZipHeaderPeekInputStream extends PushbackInputStream
170 {
171
172 private static final byte[] ZIP_HEADER = new byte[] {0x50, 0x4b, 0x03, 0x04};
173
174 private static final int HEADER_LEN = 4;
175
176 protected ZipHeaderPeekInputStream( InputStream in )
177 {
178 super( in, HEADER_LEN );
179 }
180
181 public boolean hasZipHeader() throws IOException
182 {
183 final byte[] header = new byte[HEADER_LEN];
184 int len = super.read( header, 0, HEADER_LEN );
185 if ( len != -1 )
186 {
187 super.unread( header, 0, len );
188 }
189 return Arrays.equals( header, ZIP_HEADER );
190 }
191 }
192
193
194
195
196 private static class CrcAndSize
197 {
198
199 private final CRC32 crc = new CRC32();
200
201 private long size;
202
203 CrcAndSize( InputStream inputStream ) throws IOException
204 {
205 load( inputStream );
206 }
207
208 private void load( InputStream inputStream ) throws IOException
209 {
210 byte[] buffer = new byte[BUFFER_SIZE];
211 int bytesRead;
212 while ( ( bytesRead = inputStream.read( buffer ) ) != -1 )
213 {
214 this.crc.update( buffer, 0, bytesRead );
215 this.size += bytesRead;
216 }
217 }
218
219 public void setupStoredEntry( JarEntry entry )
220 {
221 entry.setSize( this.size );
222 entry.setCompressedSize( this.size );
223 entry.setCrc( this.crc.getValue() );
224 entry.setMethod( ZipEntry.STORED );
225 }
226 }
227
228 private void shadeJars( ShadeRequest shadeRequest, Set<String> resources, List<ResourceTransformer> transformers,
229 JarOutputStream jos, MultiValuedMap<String, File> duplicates,
230 DefaultPackageMapper packageMapper )
231 throws IOException
232 {
233 for ( File jar : shadeRequest.getJars() )
234 {
235
236 logger.debug( "Processing JAR " + jar );
237
238 List<Filter> jarFilters = getFilters( jar, shadeRequest.getFilters() );
239
240 try ( JarFile jarFile = newJarFile( jar ) )
241 {
242
243 for ( Enumeration<JarEntry> j = jarFile.entries(); j.hasMoreElements(); )
244 {
245 JarEntry entry = j.nextElement();
246
247 String name = entry.getName();
248
249 if ( entry.isDirectory() || isFiltered( jarFilters, name ) )
250 {
251 continue;
252 }
253
254
255 if ( "META-INF/INDEX.LIST".equals( name ) )
256 {
257
258
259
260 continue;
261 }
262
263 if ( "module-info.class".equals( name ) )
264 {
265 logger.warn( "Discovered module-info.class. "
266 + "Shading will break its strong encapsulation." );
267 continue;
268 }
269
270 try
271 {
272 shadeJarEntry( shadeRequest, resources, transformers, packageMapper, jos, duplicates, jar,
273 jarFile, entry, name );
274 }
275 catch ( Exception e )
276 {
277 throw new IOException( String.format( "Problem shading JAR %s entry %s: %s", jar, name, e ),
278 e );
279 }
280 }
281
282 }
283 }
284 }
285
286 private void shadeJarEntry( ShadeRequest shadeRequest, Set<String> resources,
287 List<ResourceTransformer> transformers, DefaultPackageMapper packageMapper,
288 JarOutputStream jos, MultiValuedMap<String, File> duplicates, File jar,
289 JarFile jarFile, JarEntry entry, String name )
290 throws IOException, MojoExecutionException
291 {
292 try ( InputStream in = jarFile.getInputStream( entry ) )
293 {
294 String mappedName = packageMapper.map( name, true, false );
295
296 int idx = mappedName.lastIndexOf( '/' );
297 if ( idx != -1 )
298 {
299
300 String dir = mappedName.substring( 0, idx );
301 if ( !resources.contains( dir ) )
302 {
303 addDirectory( resources, jos, dir, entry.getTime() );
304 }
305 }
306
307 duplicates.put( name, jar );
308 if ( name.endsWith( ".class" ) )
309 {
310 addRemappedClass( jos, jar, name, entry.getTime(), in, packageMapper );
311 }
312 else if ( shadeRequest.isShadeSourcesContent() && name.endsWith( ".java" ) )
313 {
314
315 if ( resources.contains( mappedName ) )
316 {
317 return;
318 }
319
320 addJavaSource( resources, jos, mappedName, entry.getTime(), in, shadeRequest.getRelocators() );
321 }
322 else
323 {
324 if ( !resourceTransformed( transformers, mappedName, in, shadeRequest.getRelocators(),
325 entry.getTime() ) )
326 {
327
328 if ( resources.contains( mappedName ) )
329 {
330 logger.debug( "We have a duplicate " + name + " in " + jar );
331 return;
332 }
333
334 addResource( resources, jos, mappedName, entry, jarFile );
335 }
336 else
337 {
338 duplicates.removeMapping( name, jar );
339 }
340 }
341 }
342 }
343
344 private void goThroughAllJarEntriesForManifestTransformer( ShadeRequest shadeRequest, Set<String> resources,
345 ManifestResourceTransformer manifestTransformer,
346 JarOutputStream jos )
347 throws IOException
348 {
349 if ( manifestTransformer != null )
350 {
351 for ( File jar : shadeRequest.getJars() )
352 {
353 try ( JarFile jarFile = newJarFile( jar ) )
354 {
355 for ( Enumeration<JarEntry> en = jarFile.entries(); en.hasMoreElements(); )
356 {
357 JarEntry entry = en.nextElement();
358 String resource = entry.getName();
359 if ( manifestTransformer.canTransformResource( resource ) )
360 {
361 resources.add( resource );
362 try ( InputStream inputStream = jarFile.getInputStream( entry ) )
363 {
364 manifestTransformer.processResource( resource, inputStream,
365 shadeRequest.getRelocators(), entry.getTime() );
366 }
367 break;
368 }
369 }
370 }
371 }
372 if ( manifestTransformer.hasTransformedResource() )
373 {
374 manifestTransformer.modifyOutputStream( jos );
375 }
376 }
377 }
378
379 private void showOverlappingWarning()
380 {
381 logger.warn( "maven-shade-plugin has detected that some class files are" );
382 logger.warn( "present in two or more JARs. When this happens, only one" );
383 logger.warn( "single version of the class is copied to the uber jar." );
384 logger.warn( "Usually this is not harmful and you can skip these warnings," );
385 logger.warn( "otherwise try to manually exclude artifacts based on" );
386 logger.warn( "mvn dependency:tree -Ddetail=true and the above output." );
387 logger.warn( "See https://maven.apache.org/plugins/maven-shade-plugin/" );
388 }
389
390 private void logSummaryOfDuplicates( MultiValuedMap<Collection<File>, String> overlapping )
391 {
392 for ( Collection<File> jarz : overlapping.keySet() )
393 {
394 List<String> jarzS = new ArrayList<>();
395
396 for ( File jjar : jarz )
397 {
398 jarzS.add( jjar.getName() );
399 }
400
401 Collections.sort( jarzS );
402
403 List<String> classes = new LinkedList<>();
404 List<String> resources = new LinkedList<>();
405
406 for ( String name : overlapping.get( jarz ) )
407 {
408 if ( name.endsWith( ".class" ) )
409 {
410 classes.add( name.replace( ".class", "" ).replace( "/", "." ) );
411 }
412 else
413 {
414 resources.add( name );
415 }
416 }
417
418
419 final Collection<String> overlaps = new ArrayList<>();
420 if ( !classes.isEmpty() )
421 {
422 if ( resources.size() == 1 )
423 {
424 overlaps.add( "class" );
425 }
426 else
427 {
428 overlaps.add( "classes" );
429 }
430 }
431 if ( !resources.isEmpty() )
432 {
433 if ( resources.size() == 1 )
434 {
435 overlaps.add( "resource" );
436 }
437 else
438 {
439 overlaps.add( "resources" );
440 }
441 }
442
443 final List<String> all = new ArrayList<>( classes.size() + resources.size() );
444 all.addAll( classes );
445 all.addAll( resources );
446
447 logger.warn(
448 String.join( ", ", jarzS ) + " define " + all.size()
449 + " overlapping " + String.join( " and ", overlaps ) + ": " );
450
451
452 Collections.sort( all );
453
454 int max = 10;
455
456 for ( int i = 0; i < Math.min( max, all.size() ); i++ )
457 {
458 logger.warn( " - " + all.get( i ) );
459 }
460
461 if ( all.size() > max )
462 {
463 logger.warn( " - " + ( all.size() - max ) + " more..." );
464 }
465
466 }
467 }
468
469 private JarFile newJarFile( File jar )
470 throws IOException
471 {
472 try
473 {
474 return new JarFile( jar );
475 }
476 catch ( ZipException zex )
477 {
478
479
480 throw new ZipException( "error in opening zip file " + jar );
481 }
482 }
483
484 private List<Filter> getFilters( File jar, List<Filter> filters )
485 {
486 List<Filter> list = new ArrayList<>();
487
488 for ( Filter filter : filters )
489 {
490 if ( filter.canFilter( jar ) )
491 {
492 list.add( filter );
493 }
494
495 }
496
497 return list;
498 }
499
500 private void addDirectory( Set<String> resources, JarOutputStream jos, String name, long time )
501 throws IOException
502 {
503 if ( name.lastIndexOf( '/' ) > 0 )
504 {
505 String parent = name.substring( 0, name.lastIndexOf( '/' ) );
506 if ( !resources.contains( parent ) )
507 {
508 addDirectory( resources, jos, parent, time );
509 }
510 }
511
512
513 JarEntry entry = new JarEntry( name + "/" );
514 entry.setTime( time );
515 jos.putNextEntry( entry );
516
517 resources.add( name );
518 }
519
520 private void addRemappedClass( JarOutputStream jos, File jar, String name,
521 long time, InputStream is, DefaultPackageMapper packageMapper )
522 throws IOException, MojoExecutionException
523 {
524 if ( packageMapper.relocators.isEmpty() )
525 {
526 try
527 {
528 JarEntry entry = new JarEntry( name );
529 entry.setTime( time );
530 jos.putNextEntry( entry );
531 IOUtil.copy( is, jos );
532 }
533 catch ( ZipException e )
534 {
535 logger.debug( "We have a duplicate " + name + " in " + jar );
536 }
537
538 return;
539 }
540
541
542
543
544 byte[] originalClass = IOUtil.toByteArray( is );
545
546 ClassReader cr = new ClassReader( new ByteArrayInputStream( originalClass ) );
547
548
549
550
551
552
553 ClassWriter cw = new ClassWriter( 0 );
554
555 final String pkg = name.substring( 0, name.lastIndexOf( '/' ) + 1 );
556 final ShadeClassRemapper cv = new ShadeClassRemapper( cw, pkg, packageMapper );
557
558 try
559 {
560 cr.accept( cv, ClassReader.EXPAND_FRAMES );
561 }
562 catch ( Throwable ise )
563 {
564 throw new MojoExecutionException( "Error in ASM processing class " + name, ise );
565 }
566
567
568 final byte[] renamedClass;
569 if ( cv.remapped )
570 {
571 logger.debug( "Rewrote class bytecode: " + name );
572 renamedClass = cw.toByteArray();
573 }
574 else
575 {
576 logger.debug( "Keeping original class bytecode: " + name );
577 renamedClass = originalClass;
578 }
579
580
581 String mappedName = packageMapper.map( name.substring( 0, name.indexOf( '.' ) ), true, false );
582
583 try
584 {
585
586 JarEntry entry = new JarEntry( mappedName + ".class" );
587 entry.setTime( time );
588 jos.putNextEntry( entry );
589
590 jos.write( renamedClass );
591 }
592 catch ( ZipException e )
593 {
594 logger.debug( "We have a duplicate " + mappedName + " in " + jar );
595 }
596 }
597
598 private boolean isFiltered( List<Filter> filters, String name )
599 {
600 for ( Filter filter : filters )
601 {
602 if ( filter.isFiltered( name ) )
603 {
604 return true;
605 }
606 }
607
608 return false;
609 }
610
611 private boolean resourceTransformed( List<ResourceTransformer> resourceTransformers, String name, InputStream is,
612 List<Relocator> relocators, long time )
613 throws IOException
614 {
615 boolean resourceTransformed = false;
616
617 for ( ResourceTransformer transformer : resourceTransformers )
618 {
619 if ( transformer.canTransformResource( name ) )
620 {
621 logger.debug( "Transforming " + name + " using " + transformer.getClass().getName() );
622
623 if ( transformer instanceof ReproducibleResourceTransformer )
624 {
625 ( (ReproducibleResourceTransformer) transformer ).processResource( name, is, relocators, time );
626 }
627 else
628 {
629 transformer.processResource( name, is, relocators );
630 }
631
632 resourceTransformed = true;
633
634 break;
635 }
636 }
637 return resourceTransformed;
638 }
639
640 private void addJavaSource( Set<String> resources, JarOutputStream jos, String name, long time, InputStream is,
641 List<Relocator> relocators )
642 throws IOException
643 {
644 JarEntry entry = new JarEntry( name );
645 entry.setTime( time );
646 jos.putNextEntry( entry );
647
648 String sourceContent = IOUtil.toString( new InputStreamReader( is, StandardCharsets.UTF_8 ) );
649
650 for ( Relocator relocator : relocators )
651 {
652 sourceContent = relocator.applyToSourceContent( sourceContent );
653 }
654
655 final Writer writer = new OutputStreamWriter( jos, StandardCharsets.UTF_8 );
656 writer.write( sourceContent );
657 writer.flush();
658
659 resources.add( name );
660 }
661
662 private void addResource( Set<String> resources, JarOutputStream jos, String name, JarEntry originalEntry,
663 JarFile jarFile ) throws IOException
664 {
665 ZipHeaderPeekInputStream inputStream = new ZipHeaderPeekInputStream( jarFile.getInputStream( originalEntry ) );
666 try
667 {
668 final JarEntry entry = new JarEntry( name );
669
670
671 if ( inputStream.hasZipHeader() && originalEntry.getMethod() == ZipEntry.STORED )
672 {
673 new CrcAndSize( inputStream ).setupStoredEntry( entry );
674 inputStream.close();
675 inputStream = new ZipHeaderPeekInputStream( jarFile.getInputStream( originalEntry ) );
676 }
677
678
679 entry.setTime( originalEntry.getTime() );
680
681 jos.putNextEntry( entry );
682
683 IOUtil.copy( inputStream, jos );
684
685 resources.add( name );
686 }
687 finally
688 {
689 inputStream.close();
690 }
691 }
692
693 private interface PackageMapper
694 {
695
696
697
698
699
700
701
702
703 String map( String entityName, boolean mapPaths, boolean mapPackages );
704 }
705
706
707
708
709 private static class DefaultPackageMapper implements PackageMapper
710 {
711 private static final Pattern CLASS_PATTERN = Pattern.compile( "(\\[*)?L(.+);" );
712
713 private final List<Relocator> relocators;
714
715 private DefaultPackageMapper( final List<Relocator> relocators )
716 {
717 this.relocators = relocators;
718 }
719
720 @Override
721 public String map( String entityName, boolean mapPaths, final boolean mapPackages )
722 {
723 String value = entityName;
724
725 String prefix = "";
726 String suffix = "";
727
728 Matcher m = CLASS_PATTERN.matcher( entityName );
729 if ( m.matches() )
730 {
731 prefix = m.group( 1 ) + "L";
732 suffix = ";";
733 entityName = m.group( 2 );
734 }
735
736 for ( Relocator r : relocators )
737 {
738 if ( mapPackages && r.canRelocateClass( entityName ) )
739 {
740 value = prefix + r.relocateClass( entityName ) + suffix;
741 break;
742 }
743 else if ( mapPaths && r.canRelocatePath( entityName ) )
744 {
745 value = prefix + r.relocatePath( entityName ) + suffix;
746 break;
747 }
748 }
749 return value;
750 }
751 }
752
753 private static class LazyInitRemapper extends Remapper
754 {
755 private PackageMapper relocators;
756
757 @Override
758 public Object mapValue( Object object )
759 {
760 return object instanceof String
761 ? relocators.map( (String) object, true, true )
762 : super.mapValue( object );
763 }
764
765 @Override
766 public String map( String name )
767 {
768
769
770
771
772
773
774
775
776 return relocators.map( name, true, false );
777 }
778 }
779
780
781
782
783
784
785
786
787
788
789
790 private static class ShadeClassRemapper extends ClassRemapper implements PackageMapper
791 {
792 private final String pkg;
793 private final PackageMapper packageMapper;
794 private boolean remapped;
795
796 ShadeClassRemapper( final ClassVisitor classVisitor, final String pkg,
797 final DefaultPackageMapper packageMapper )
798 {
799 super( classVisitor, new LazyInitRemapper() );
800 this.pkg = pkg;
801 this.packageMapper = packageMapper;
802
803
804 LazyInitRemapper.class.cast( remapper ).relocators = this;
805 }
806
807 @Override
808 public void visitSource( final String source, final String debug )
809 {
810 if ( source == null )
811 {
812 super.visitSource( null, debug );
813 return;
814 }
815
816 final String fqSource = pkg + source;
817 final String mappedSource = map( fqSource, true, false );
818 final String filename = mappedSource.substring( mappedSource.lastIndexOf( '/' ) + 1 );
819 super.visitSource( filename, debug );
820 }
821
822 @Override
823 public String map( final String entityName, boolean mapPaths, final boolean mapPackages )
824 {
825 final String mapped = packageMapper.map( entityName, true, mapPackages );
826 if ( !remapped )
827 {
828 remapped = !mapped.equals( entityName );
829 }
830 return mapped;
831 }
832 }
833 }