Skip to content

Commit

Permalink
Transform dirs and jars in parallel
Browse files Browse the repository at this point in the history
close #4
  • Loading branch information
yrom committed Jul 24, 2017
1 parent f6ebed3 commit fbe036d
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 76 deletions.
11 changes: 10 additions & 1 deletion shrinker/src/main/groovy/net/yrom/tools/ClassTransform.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import org.objectweb.asm.ClassReader
import org.objectweb.asm.ClassVisitor
import org.objectweb.asm.ClassWriter

import java.util.function.Function

/**
* @author yrom.
*/
Expand All @@ -31,7 +33,7 @@ class ClassTransform {
this.rSymbols = rSymbols
}

def byte[] transform(byte[] origin) {
def byte[] _transform(byte[] origin) {
ClassReader reader = new ClassReader(origin)
// don't pass reader to the writer.
// or it will copy 'CONSTANT POOL' that contains no used entries to lead proguard running failed!
Expand All @@ -40,4 +42,11 @@ class ClassTransform {
reader.accept visitor, 0
writer.toByteArray()
}
def transform = new Function<byte[], byte[]>() {
@Override
byte[] apply(byte[] origin) {
return ClassTransform.this._transform(origin)
}
}

}
89 changes: 14 additions & 75 deletions shrinker/src/main/groovy/net/yrom/tools/InlineRTransform.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,6 @@ import com.google.common.collect.ImmutableSet
import com.google.common.collect.Sets
import groovy.transform.PackageScope
import org.apache.commons.io.FileUtils
import org.apache.commons.io.IOUtils

import java.util.jar.JarEntry
import java.util.jar.JarOutputStream
import java.util.zip.ZipEntry
import java.util.zip.ZipInputStream

import static com.android.build.api.transform.QualifiedContent.DefaultContentType.CLASSES
/**
Expand Down Expand Up @@ -88,10 +82,19 @@ class InlineRTransform extends Transform {
if (config.inlineR && buildType != 'debug') {
def rSymbols = new RSymbols().from(inputs)
if (!rSymbols.isEmpty()) {
for (input in inputs) {
processAllClasses input, outputProvider, rSymbols
}
ShrinkerPlugin.logger.lifecycle "${transformInvocation.context.path} transform consume ${System.currentTimeMillis() - start}ms"
InlineRProcessor.proceed(inputs,
{ QualifiedContent input ->
def format
if (input instanceof DirectoryInput) format = Format.DIRECTORY
else if (input instanceof JarInput) format = Format.JAR
else throw new UnsupportedOperationException("Unknown format of input " + input)
def f = outputProvider.getContentLocation(input.name, input.contentTypes,
input.scopes, format)
if (!f.parentFile.exists()) f.parentFile.mkdirs()
return f.toPath()
},
new ClassTransform(rSymbols).transform)
ShrinkerPlugin.logger.lifecycle "${transformInvocation.context.path} consume ${System.currentTimeMillis() - start}ms"
return
}
}
Expand All @@ -112,71 +115,7 @@ class InlineRTransform extends Transform {
FileUtils.copyFile jarInput.file, dest
}
}
ShrinkerPlugin.logger.info "${transformInvocation.context.path} copy files take ${System.currentTimeMillis() - start} ms"
}

static void processAllClasses(TransformInput input, TransformOutputProvider outputProvider, rSymbols) {
def classTransform = new ClassTransform(rSymbols)
input.directoryInputs.each { DirectoryInput dir ->
File srcFolder = dir.file
ShrinkerPlugin.logger.info 'Processing folder ' + srcFolder
File destFolder = outputProvider.getContentLocation(dir.name, dir.contentTypes,
dir.scopes, Format.DIRECTORY);
copy(srcFolder, destFolder, classTransform)
}
input.jarInputs.each { JarInput jarInput ->
ShrinkerPlugin.logger.info 'Processing jar ' + jarInput.file.absolutePath
File dest = outputProvider.getContentLocation(jarInput.name, jarInput.contentTypes,
jarInput.scopes, Format.JAR)
ZipInputStream zis = null
JarOutputStream jarOutputStream = null
try {
zis = new ZipInputStream(FileUtils.openInputStream(jarInput.file))
jarOutputStream = new JarOutputStream(FileUtils.openOutputStream(dest))
ZipEntry entry
while ((entry = zis.getNextEntry()) != null) {
if (entry.isDirectory()) continue
String name = entry.name
if (!name.endsWith('.class')) {
continue
}
JarEntry newEntry
if (entry.method == ZipEntry.STORED) {
newEntry = new JarEntry(entry)
} else {
newEntry = new JarEntry(name)
}
jarOutputStream.putNextEntry newEntry
byte[] bytes = classTransform.transform IOUtils.toByteArray(zis)
jarOutputStream.write bytes
jarOutputStream.closeEntry()
zis.closeEntry()
}
} catch (Exception e) {
throw new IOException('Failed to process jar ' + jarInput.file.absolutePath, e)
} finally {
IOUtils.closeQuietly zis
IOUtils.closeQuietly jarOutputStream
}
}
}

static void copy(File src, File dest, ClassTransform classFilter) {
for (File file : src.listFiles()) {
if (file.isDirectory()) {
copy file, new File(dest, file.name), classFilter
} else {
String name = file.name
// find R.class or R$**.class
if (name ==~ /R\.class|R\$(?!styleable)[a-z]+\.class/) {
ShrinkerPlugin.logger.info ' ignored file ' + file.absolutePath
} else {
byte[] bytes = classFilter.transform(file.bytes)
File destFile = new File(dest, name)
FileUtils.writeByteArrayToFile destFile, bytes
}
}
}
ShrinkerPlugin.logger.info "${transformInvocation.context.path} copy files ${System.currentTimeMillis() - start} ms"
}

}
132 changes: 132 additions & 0 deletions shrinker/src/main/java/net/yrom/tools/InlineRProcessor.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
package net.yrom.tools;

import com.android.build.api.transform.DirectoryInput;
import com.android.build.api.transform.JarInput;
import com.android.build.api.transform.QualifiedContent;
import com.android.build.api.transform.TransformInput;

import org.apache.commons.io.IOUtils;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.Collection;
import java.util.function.Function;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import groovy.lang.Closure;

/**
* @author yrom
*/
final class InlineRProcessor {
private static final Logger log = Logging.getLogger(InlineRProcessor.class);
private InlineRProcessor() {}

static void proceed(Collection<TransformInput> inputs,
Closure<Path> getTargetPath,
Function<byte[], byte[]> transform) {
Stream.concat(
streamOf(inputs, TransformInput::getDirectoryInputs),
streamOf(inputs, TransformInput::getJarInputs))
.forEach((QualifiedContent input) -> {
long start = System.currentTimeMillis();
Path src = input.getFile().toPath();
Path dst = getTargetPath.call(input);
if (input instanceof DirectoryInput) {
transformDir(src, dst, transform);
} else if (input instanceof JarInput) {
transformJar(src, dst, transform);
} else {
throw new RuntimeException();
}
log.info((System.currentTimeMillis() - start) + "ms " + src);
});
}

private static <T extends QualifiedContent> Stream<T> streamOf(
Collection<TransformInput> inputs,
Function<TransformInput, Collection<T>> mapping) {
Collection<T> list = inputs.stream()
.map(mapping)
.flatMap(Collection::stream)
.collect(Collectors.toList());
if (list.size() >= Runtime.getRuntime().availableProcessors())
return list.parallelStream();
else
return list.stream();
}

private static PathMatcher CASE_R_FILE
= FileSystems.getDefault().getPathMatcher("regex:^R\\.class|R\\$(?!styleable)[a-z]+\\.class$");

private static DirectoryStream.Filter<Path> CLASS_TRANSFORM_FILTER
= path -> Files.isDirectory(path)
|| (Files.isRegularFile(path) && !CASE_R_FILE.matches(path.getFileName()));

private static void transformDir(Path src, Path dest, Function<byte[], byte[]> classTransform) {
try {
for (Path file : Files.newDirectoryStream(src, CLASS_TRANSFORM_FILTER)) {
String name = file.getFileName().toString();
Path target = dest.resolve(name);
if (Files.isDirectory(file)) {
transformDir(file, target, classTransform);
} else if (Files.isRegularFile(file)) {
log.debug("transform class {}...", file);
byte[] bytes = classTransform.apply(Files.readAllBytes(file));
if (Files.notExists(dest)) {
Files.createDirectories(dest);
}
Files.write(target, bytes);
}
}

} catch (IOException e) {
throw new UncheckedIOException(e);
}
}

private static void transformJar(Path src, Path dst, Function<byte[], byte[]> classTransform) {
if (Files.notExists(src)) throw new IllegalArgumentException("No such file " + src);
try (ZipInputStream in = new ZipInputStream(new BufferedInputStream(Files.newInputStream(src)));
JarOutputStream out = new JarOutputStream(new BufferedOutputStream(Files.newOutputStream(dst)))) {
ZipEntry entry;
while ((entry = in.getNextEntry()) != null) {
if (entry.isDirectory()) continue;
String name = entry.getName();
if (!name.endsWith(".class")) {
// skip
continue;
}
JarEntry newEntry;
if (entry.getMethod() == ZipEntry.STORED) {
newEntry = new JarEntry(entry);
} else {
newEntry = new JarEntry(name);
}
// put new entry
out.putNextEntry(newEntry);
byte[] bytes = classTransform.apply(IOUtils.toByteArray(in));
// write bytes of entry
out.write(bytes);
out.closeEntry();
in.closeEntry();
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}

0 comments on commit fbe036d

Please sign in to comment.