1 package org.apache.maven.plugins.shade;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.BufferedOutputStream;
23 import java.io.ByteArrayInputStream;
24 import java.io.File;
25 import java.io.FileOutputStream;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.InputStreamReader;
29 import java.io.OutputStreamWriter;
30 import java.io.PushbackInputStream;
31 import java.io.Writer;
32 import java.nio.charset.StandardCharsets;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.Enumeration;
38 import java.util.HashSet;
39 import java.util.Iterator;
40 import java.util.LinkedList;
41 import java.util.List;
42 import java.util.Objects;
43 import java.util.Set;
44 import java.util.jar.JarEntry;
45 import java.util.jar.JarFile;
46 import java.util.jar.JarOutputStream;
47 import java.util.regex.Matcher;
48 import java.util.regex.Pattern;
49 import java.util.zip.CRC32;
50 import java.util.zip.ZipEntry;
51 import java.util.zip.ZipException;
52
53 import javax.inject.Named;
54 import javax.inject.Singleton;
55
56 import org.apache.commons.collections4.MultiValuedMap;
57 import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
58 import org.apache.maven.plugin.MojoExecutionException;
59 import org.apache.maven.plugins.shade.filter.Filter;
60 import org.apache.maven.plugins.shade.relocation.Relocator;
61 import org.apache.maven.plugins.shade.resource.ManifestResourceTransformer;
62 import org.apache.maven.plugins.shade.resource.ReproducibleResourceTransformer;
63 import org.apache.maven.plugins.shade.resource.ResourceTransformer;
64 import org.codehaus.plexus.util.IOUtil;
65 import org.objectweb.asm.ClassReader;
66 import org.objectweb.asm.ClassVisitor;
67 import org.objectweb.asm.ClassWriter;
68 import org.objectweb.asm.commons.ClassRemapper;
69 import org.objectweb.asm.commons.Remapper;
70 import org.slf4j.Logger;
71 import org.slf4j.LoggerFactory;
72
73
74
75
76 @Singleton
77 @Named
78 public class DefaultShader
79 implements Shader
80 {
81 private static final int BUFFER_SIZE = 32 * 1024;
82
83 private final Logger logger;
84
85 public DefaultShader()
86 {
87 this( LoggerFactory.getLogger( DefaultShader.class ) );
88 }
89
90 public DefaultShader( final Logger logger )
91 {
92 this.logger = Objects.requireNonNull( logger );
93 }
94
95 public void shade( ShadeRequest shadeRequest )
96 throws IOException, MojoExecutionException
97 {
98 Set<String> resources = new HashSet<>();
99
100 ManifestResourceTransformer manifestTransformer = null;
101 List<ResourceTransformer> transformers =
102 new ArrayList<>( shadeRequest.getResourceTransformers() );
103 for ( Iterator<ResourceTransformer> it = transformers.iterator(); it.hasNext(); )
104 {
105 ResourceTransformer transformer = it.next();
106 if ( transformer instanceof ManifestResourceTransformer )
107 {
108 manifestTransformer = (ManifestResourceTransformer) transformer;
109 it.remove();
110 }
111 }
112
113 final DefaultPackageMapper packageMapper = new DefaultPackageMapper( shadeRequest.getRelocators() );
114
115
116 shadeRequest.getUberJar().getParentFile().mkdirs();
117
118 try ( JarOutputStream out =
119 new JarOutputStream( new BufferedOutputStream( new FileOutputStream( shadeRequest.getUberJar() ) ) ) )
120 {
121 goThroughAllJarEntriesForManifestTransformer( shadeRequest, resources, manifestTransformer, out );
122
123
124 MultiValuedMap<String, File> duplicates = new HashSetValuedHashMap<>( 10000, 3 );
125
126
127 shadeJars( shadeRequest, resources, transformers, out, duplicates, packageMapper );
128
129
130 MultiValuedMap<Collection<File>, String> overlapping = new HashSetValuedHashMap<>( 20, 15 );
131
132
133 for ( String clazz : duplicates.keySet() )
134 {
135 Collection<File> jarz = duplicates.get( clazz );
136 if ( jarz.size() > 1 )
137 {
138 overlapping.put( jarz, clazz );
139 }
140 }
141
142
143 logSummaryOfDuplicates( overlapping );
144
145 if ( overlapping.keySet().size() > 0 )
146 {
147 showOverlappingWarning();
148 }
149
150 for ( ResourceTransformer transformer : transformers )
151 {
152 if ( transformer.hasTransformedResource() )
153 {
154 transformer.modifyOutputStream( out );
155 }
156 }
157 }
158
159 for ( Filter filter : shadeRequest.getFilters() )
160 {
161 filter.finished();
162 }
163 }
164
165
166
167
168 private static class ZipHeaderPeekInputStream extends PushbackInputStream
169 {
170
171 private static final byte[] ZIP_HEADER = new byte[] {0x50, 0x4b, 0x03, 0x04};
172
173 private static final int HEADER_LEN = 4;
174
175 protected ZipHeaderPeekInputStream( InputStream in )
176 {
177 super( in, HEADER_LEN );
178 }
179
180 public boolean hasZipHeader() throws IOException
181 {
182 final byte[] header = new byte[HEADER_LEN];
183 super.read( header, 0, HEADER_LEN );
184 super.unread( header );
185 return Arrays.equals( header, ZIP_HEADER );
186 }
187 }
188
189
190
191
192 private static class CrcAndSize
193 {
194
195 private final CRC32 crc = new CRC32();
196
197 private long size;
198
199 CrcAndSize( InputStream inputStream ) throws IOException
200 {
201 load( inputStream );
202 }
203
204 private void load( InputStream inputStream ) throws IOException
205 {
206 byte[] buffer = new byte[BUFFER_SIZE];
207 int bytesRead;
208 while ( ( bytesRead = inputStream.read( buffer ) ) != -1 )
209 {
210 this.crc.update( buffer, 0, bytesRead );
211 this.size += bytesRead;
212 }
213 }
214
215 public void setupStoredEntry( JarEntry entry )
216 {
217 entry.setSize( this.size );
218 entry.setCompressedSize( this.size );
219 entry.setCrc( this.crc.getValue() );
220 entry.setMethod( ZipEntry.STORED );
221 }
222 }
223
224 private void shadeJars( ShadeRequest shadeRequest, Set<String> resources, List<ResourceTransformer> transformers,
225 JarOutputStream jos, MultiValuedMap<String, File> duplicates,
226 DefaultPackageMapper packageMapper )
227 throws IOException
228 {
229 for ( File jar : shadeRequest.getJars() )
230 {
231
232 logger.debug( "Processing JAR " + jar );
233
234 List<Filter> jarFilters = getFilters( jar, shadeRequest.getFilters() );
235
236 try ( JarFile jarFile = newJarFile( jar ) )
237 {
238
239 for ( Enumeration<JarEntry> j = jarFile.entries(); j.hasMoreElements(); )
240 {
241 JarEntry entry = j.nextElement();
242
243 String name = entry.getName();
244
245 if ( entry.isDirectory() || isFiltered( jarFilters, name ) )
246 {
247 continue;
248 }
249
250
251 if ( "META-INF/INDEX.LIST".equals( name ) )
252 {
253
254
255
256 continue;
257 }
258
259 if ( "module-info.class".equals( name ) )
260 {
261 logger.warn( "Discovered module-info.class. "
262 + "Shading will break its strong encapsulation." );
263 continue;
264 }
265
266 try
267 {
268 shadeJarEntry( shadeRequest, resources, transformers, packageMapper, jos, duplicates, jar,
269 jarFile, entry, name );
270 }
271 catch ( Exception e )
272 {
273 throw new IOException( String.format( "Problem shading JAR %s entry %s: %s", jar, name, e ),
274 e );
275 }
276 }
277
278 }
279 }
280 }
281
282 private void shadeJarEntry( ShadeRequest shadeRequest, Set<String> resources,
283 List<ResourceTransformer> transformers, DefaultPackageMapper packageMapper,
284 JarOutputStream jos, MultiValuedMap<String, File> duplicates, File jar,
285 JarFile jarFile, JarEntry entry, String name )
286 throws IOException, MojoExecutionException
287 {
288 try ( InputStream in = jarFile.getInputStream( entry ) )
289 {
290 String mappedName = packageMapper.map( name, true, false );
291
292 int idx = mappedName.lastIndexOf( '/' );
293 if ( idx != -1 )
294 {
295
296 String dir = mappedName.substring( 0, idx );
297 if ( !resources.contains( dir ) )
298 {
299 addDirectory( resources, jos, dir, entry.getTime() );
300 }
301 }
302
303 duplicates.put( name, jar );
304 if ( name.endsWith( ".class" ) )
305 {
306 addRemappedClass( jos, jar, name, entry.getTime(), in, packageMapper );
307 }
308 else if ( shadeRequest.isShadeSourcesContent() && name.endsWith( ".java" ) )
309 {
310
311 if ( resources.contains( mappedName ) )
312 {
313 return;
314 }
315
316 addJavaSource( resources, jos, mappedName, entry.getTime(), in, shadeRequest.getRelocators() );
317 }
318 else
319 {
320 if ( !resourceTransformed( transformers, mappedName, in, shadeRequest.getRelocators(),
321 entry.getTime() ) )
322 {
323
324 if ( resources.contains( mappedName ) )
325 {
326 logger.debug( "We have a duplicate " + name + " in " + jar );
327 return;
328 }
329
330 addResource( resources, jos, mappedName, entry, jarFile );
331 }
332 else
333 {
334 duplicates.removeMapping( name, jar );
335 }
336 }
337 }
338 }
339
340 private void goThroughAllJarEntriesForManifestTransformer( ShadeRequest shadeRequest, Set<String> resources,
341 ManifestResourceTransformer manifestTransformer,
342 JarOutputStream jos )
343 throws IOException
344 {
345 if ( manifestTransformer != null )
346 {
347 for ( File jar : shadeRequest.getJars() )
348 {
349 try ( JarFile jarFile = newJarFile( jar ) )
350 {
351 for ( Enumeration<JarEntry> en = jarFile.entries(); en.hasMoreElements(); )
352 {
353 JarEntry entry = en.nextElement();
354 String resource = entry.getName();
355 if ( manifestTransformer.canTransformResource( resource ) )
356 {
357 resources.add( resource );
358 try ( InputStream inputStream = jarFile.getInputStream( entry ) )
359 {
360 manifestTransformer.processResource( resource, inputStream,
361 shadeRequest.getRelocators(), entry.getTime() );
362 }
363 break;
364 }
365 }
366 }
367 }
368 if ( manifestTransformer.hasTransformedResource() )
369 {
370 manifestTransformer.modifyOutputStream( jos );
371 }
372 }
373 }
374
375 private void showOverlappingWarning()
376 {
377 logger.warn( "maven-shade-plugin has detected that some class files are" );
378 logger.warn( "present in two or more JARs. When this happens, only one" );
379 logger.warn( "single version of the class is copied to the uber jar." );
380 logger.warn( "Usually this is not harmful and you can skip these warnings," );
381 logger.warn( "otherwise try to manually exclude artifacts based on" );
382 logger.warn( "mvn dependency:tree -Ddetail=true and the above output." );
383 logger.warn( "See https://maven.apache.org/plugins/maven-shade-plugin/" );
384 }
385
386 private void logSummaryOfDuplicates( MultiValuedMap<Collection<File>, String> overlapping )
387 {
388 for ( Collection<File> jarz : overlapping.keySet() )
389 {
390 List<String> jarzS = new ArrayList<>();
391
392 for ( File jjar : jarz )
393 {
394 jarzS.add( jjar.getName() );
395 }
396
397 Collections.sort( jarzS );
398
399 List<String> classes = new LinkedList<>();
400 List<String> resources = new LinkedList<>();
401
402 for ( String name : overlapping.get( jarz ) )
403 {
404 if ( name.endsWith( ".class" ) )
405 {
406 classes.add( name.replace( ".class", "" ).replace( "/", "." ) );
407 }
408 else
409 {
410 resources.add( name );
411 }
412 }
413
414
415 final Collection<String> overlaps = new ArrayList<>();
416 if ( !classes.isEmpty() )
417 {
418 if ( resources.size() == 1 )
419 {
420 overlaps.add( "class" );
421 }
422 else
423 {
424 overlaps.add( "classes" );
425 }
426 }
427 if ( !resources.isEmpty() )
428 {
429 if ( resources.size() == 1 )
430 {
431 overlaps.add( "resource" );
432 }
433 else
434 {
435 overlaps.add( "resources" );
436 }
437 }
438
439 final List<String> all = new ArrayList<>( classes.size() + resources.size() );
440 all.addAll( classes );
441 all.addAll( resources );
442
443 logger.warn(
444 String.join( ", ", jarzS ) + " define " + all.size()
445 + " overlapping " + String.join( " and ", overlaps ) + ": " );
446
447
448 Collections.sort( all );
449
450 int max = 10;
451
452 for ( int i = 0; i < Math.min( max, all.size() ); i++ )
453 {
454 logger.warn( " - " + all.get( i ) );
455 }
456
457 if ( all.size() > max )
458 {
459 logger.warn( " - " + ( all.size() - max ) + " more..." );
460 }
461
462 }
463 }
464
465 private JarFile newJarFile( File jar )
466 throws IOException
467 {
468 try
469 {
470 return new JarFile( jar );
471 }
472 catch ( ZipException zex )
473 {
474
475
476 throw new ZipException( "error in opening zip file " + jar );
477 }
478 }
479
480 private List<Filter> getFilters( File jar, List<Filter> filters )
481 {
482 List<Filter> list = new ArrayList<>();
483
484 for ( Filter filter : filters )
485 {
486 if ( filter.canFilter( jar ) )
487 {
488 list.add( filter );
489 }
490
491 }
492
493 return list;
494 }
495
496 private void addDirectory( Set<String> resources, JarOutputStream jos, String name, long time )
497 throws IOException
498 {
499 if ( name.lastIndexOf( '/' ) > 0 )
500 {
501 String parent = name.substring( 0, name.lastIndexOf( '/' ) );
502 if ( !resources.contains( parent ) )
503 {
504 addDirectory( resources, jos, parent, time );
505 }
506 }
507
508
509 JarEntry entry = new JarEntry( name + "/" );
510 entry.setTime( time );
511 jos.putNextEntry( entry );
512
513 resources.add( name );
514 }
515
516 private void addRemappedClass( JarOutputStream jos, File jar, String name,
517 long time, InputStream is, DefaultPackageMapper packageMapper )
518 throws IOException, MojoExecutionException
519 {
520 if ( packageMapper.relocators.isEmpty() )
521 {
522 try
523 {
524 JarEntry entry = new JarEntry( name );
525 entry.setTime( time );
526 jos.putNextEntry( entry );
527 IOUtil.copy( is, jos );
528 }
529 catch ( ZipException e )
530 {
531 logger.debug( "We have a duplicate " + name + " in " + jar );
532 }
533
534 return;
535 }
536
537
538
539
540 byte[] originalClass = IOUtil.toByteArray( is );
541
542 ClassReader cr = new ClassReader( new ByteArrayInputStream( originalClass ) );
543
544
545
546
547
548
549 ClassWriter cw = new ClassWriter( 0 );
550
551 final String pkg = name.substring( 0, name.lastIndexOf( '/' ) + 1 );
552 final ShadeClassRemapper cv = new ShadeClassRemapper( cw, pkg, packageMapper );
553
554 try
555 {
556 cr.accept( cv, ClassReader.EXPAND_FRAMES );
557 }
558 catch ( Throwable ise )
559 {
560 throw new MojoExecutionException( "Error in ASM processing class " + name, ise );
561 }
562
563
564 final byte[] renamedClass;
565 if ( cv.remapped )
566 {
567 logger.debug( "Rewrote class bytecode: " + name );
568 renamedClass = cw.toByteArray();
569 }
570 else
571 {
572 logger.debug( "Keeping original class bytecode: " + name );
573 renamedClass = originalClass;
574 }
575
576
577 String mappedName = packageMapper.map( name.substring( 0, name.indexOf( '.' ) ), true, false );
578
579 try
580 {
581
582 JarEntry entry = new JarEntry( mappedName + ".class" );
583 entry.setTime( time );
584 jos.putNextEntry( entry );
585
586 jos.write( renamedClass );
587 }
588 catch ( ZipException e )
589 {
590 logger.debug( "We have a duplicate " + mappedName + " in " + jar );
591 }
592 }
593
594 private boolean isFiltered( List<Filter> filters, String name )
595 {
596 for ( Filter filter : filters )
597 {
598 if ( filter.isFiltered( name ) )
599 {
600 return true;
601 }
602 }
603
604 return false;
605 }
606
607 private boolean resourceTransformed( List<ResourceTransformer> resourceTransformers, String name, InputStream is,
608 List<Relocator> relocators, long time )
609 throws IOException
610 {
611 boolean resourceTransformed = false;
612
613 for ( ResourceTransformer transformer : resourceTransformers )
614 {
615 if ( transformer.canTransformResource( name ) )
616 {
617 logger.debug( "Transforming " + name + " using " + transformer.getClass().getName() );
618
619 if ( transformer instanceof ReproducibleResourceTransformer )
620 {
621 ( (ReproducibleResourceTransformer) transformer ).processResource( name, is, relocators, time );
622 }
623 else
624 {
625 transformer.processResource( name, is, relocators );
626 }
627
628 resourceTransformed = true;
629
630 break;
631 }
632 }
633 return resourceTransformed;
634 }
635
636 private void addJavaSource( Set<String> resources, JarOutputStream jos, String name, long time, InputStream is,
637 List<Relocator> relocators )
638 throws IOException
639 {
640 JarEntry entry = new JarEntry( name );
641 entry.setTime( time );
642 jos.putNextEntry( entry );
643
644 String sourceContent = IOUtil.toString( new InputStreamReader( is, StandardCharsets.UTF_8 ) );
645
646 for ( Relocator relocator : relocators )
647 {
648 sourceContent = relocator.applyToSourceContent( sourceContent );
649 }
650
651 final Writer writer = new OutputStreamWriter( jos, StandardCharsets.UTF_8 );
652 writer.write( sourceContent );
653 writer.flush();
654
655 resources.add( name );
656 }
657
658 private void addResource( Set<String> resources, JarOutputStream jos, String name, JarEntry originalEntry,
659 JarFile jarFile ) throws IOException
660 {
661 ZipHeaderPeekInputStream inputStream = new ZipHeaderPeekInputStream( jarFile.getInputStream( originalEntry ) );
662 try
663 {
664 final JarEntry entry = new JarEntry( name );
665
666
667 if ( inputStream.hasZipHeader() && originalEntry.getMethod() == ZipEntry.STORED )
668 {
669 new CrcAndSize( inputStream ).setupStoredEntry( entry );
670 inputStream.close();
671 inputStream = new ZipHeaderPeekInputStream( jarFile.getInputStream( originalEntry ) );
672 }
673
674
675 entry.setTime( originalEntry.getTime() );
676
677 jos.putNextEntry( entry );
678
679 IOUtil.copy( inputStream, jos );
680
681 resources.add( name );
682 }
683 finally
684 {
685 inputStream.close();
686 }
687 }
688
689 private interface PackageMapper
690 {
691
692
693
694
695
696
697
698
699 String map( String entityName, boolean mapPaths, boolean mapPackages );
700 }
701
702
703
704
705 private static class DefaultPackageMapper implements PackageMapper
706 {
707 private static final Pattern CLASS_PATTERN = Pattern.compile( "(\\[*)?L(.+);" );
708
709 private final List<Relocator> relocators;
710
711 private DefaultPackageMapper( final List<Relocator> relocators )
712 {
713 this.relocators = relocators;
714 }
715
716 @Override
717 public String map( String entityName, boolean mapPaths, final boolean mapPackages )
718 {
719 String value = entityName;
720
721 String prefix = "";
722 String suffix = "";
723
724 Matcher m = CLASS_PATTERN.matcher( entityName );
725 if ( m.matches() )
726 {
727 prefix = m.group( 1 ) + "L";
728 suffix = ";";
729 entityName = m.group( 2 );
730 }
731
732 for ( Relocator r : relocators )
733 {
734 if ( mapPackages && r.canRelocateClass( entityName ) )
735 {
736 value = prefix + r.relocateClass( entityName ) + suffix;
737 break;
738 }
739 else if ( mapPaths && r.canRelocatePath( entityName ) )
740 {
741 value = prefix + r.relocatePath( entityName ) + suffix;
742 break;
743 }
744 }
745 return value;
746 }
747 }
748
749 private static class LazyInitRemapper extends Remapper
750 {
751 private PackageMapper relocators;
752
753 @Override
754 public Object mapValue( Object object )
755 {
756 return object instanceof String
757 ? relocators.map( (String) object, true, true )
758 : super.mapValue( object );
759 }
760
761 @Override
762 public String map( String name )
763 {
764
765
766
767
768
769
770
771
772 return relocators.map( name, true, false );
773 }
774 }
775
776
777
778
779
780
781
782
783
784
785
786 private static class ShadeClassRemapper extends ClassRemapper implements PackageMapper
787 {
788 private final String pkg;
789 private final PackageMapper packageMapper;
790 private boolean remapped;
791
792 ShadeClassRemapper( final ClassVisitor classVisitor, final String pkg,
793 final DefaultPackageMapper packageMapper )
794 {
795 super( classVisitor, new LazyInitRemapper() );
796 this.pkg = pkg;
797 this.packageMapper = packageMapper;
798
799
800 LazyInitRemapper.class.cast( remapper ).relocators = this;
801 }
802
803 @Override
804 public void visitSource( final String source, final String debug )
805 {
806 if ( source == null )
807 {
808 super.visitSource( null, debug );
809 return;
810 }
811
812 final String fqSource = pkg + source;
813 final String mappedSource = map( fqSource, true, false );
814 final String filename = mappedSource.substring( mappedSource.lastIndexOf( '/' ) + 1 );
815 super.visitSource( filename, debug );
816 }
817
818 @Override
819 public String map( final String entityName, boolean mapPaths, final boolean mapPackages )
820 {
821 final String mapped = packageMapper.map( entityName, true, mapPackages );
822 if ( !remapped )
823 {
824 remapped = !mapped.equals( entityName );
825 }
826 return mapped;
827 }
828 }
829 }