From 7f4f01e88ef15fd3a0ca82dacb34a30038a3301b Mon Sep 17 00:00:00 2001 From: Julien Roncaglia Date: Thu, 26 Oct 2023 17:21:03 +0200 Subject: [PATCH] build: build zip archive in parallel and in one pass * Use the apache compress library to zip in parallel * Avoid the temporary zip file and copy by adding the unknown files directly while building the apk --- brut.apktool/apktool-cli/build.gradle.kts | 3 + brut.apktool/apktool-lib/build.gradle.kts | 1 + .../main/java/brut/androlib/ApkBuilder.java | 132 +++++++----------- brut.j.dir/build.gradle.kts | 1 + .../main/java/brut/directory/ZipUtils.java | 56 +++++--- gradle/libs.versions.toml | 2 + 6 files changed, 91 insertions(+), 104 deletions(-) diff --git a/brut.apktool/apktool-cli/build.gradle.kts b/brut.apktool/apktool-cli/build.gradle.kts index f90708363d..3290b17dd4 100644 --- a/brut.apktool/apktool-cli/build.gradle.kts +++ b/brut.apktool/apktool-cli/build.gradle.kts @@ -66,6 +66,9 @@ tasks.register("proguard") { dontwarn("com.google.common.collect.**") dontwarn("com.google.common.util.**") dontwarn("javax.xml.xpath.**") + dontwarn("org.apache.commons.compress.harmony.**") + dontwarn("org.apache.commons.compress.compressors.**") + dontwarn("org.apache.commons.compress.archivers.**") dontnote("**") val outPath = "build/libs/apktool-$apktoolVersion.jar" diff --git a/brut.apktool/apktool-lib/build.gradle.kts b/brut.apktool/apktool-lib/build.gradle.kts index 6a0c803067..48eedb2b4b 100644 --- a/brut.apktool/apktool-lib/build.gradle.kts +++ b/brut.apktool/apktool-lib/build.gradle.kts @@ -33,6 +33,7 @@ dependencies { implementation(libs.smali) implementation(libs.xmlpull) implementation(libs.guava) + implementation(libs.commons.compress) implementation(libs.commons.lang3) implementation(libs.commons.io) implementation(libs.commons.text) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java index 39eecdb66b..c28bbd8efd 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java @@ -33,6 +33,8 @@ import brut.directory.ZipUtils; import brut.util.BrutIO; import brut.util.OS; +import org.apache.commons.compress.archivers.zip.*; +import org.apache.commons.compress.parallel.InputStreamSupplier; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.xml.sax.SAXException; @@ -44,9 +46,6 @@ import java.util.*; import java.util.logging.Logger; import java.util.zip.CRC32; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; public class ApkBuilder { private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName()); @@ -101,12 +100,8 @@ public void build(File outFile) throws BrutException { buildManifestFile(manifest, manifestOriginal); buildResources(); buildLibs(); - buildCopyOriginalFiles(); - buildApk(outFile); - // we must go after the Apk is built, and copy the files in via Zip - // this is because Aapt won't add files it doesn't know (ex unknown files) - buildUnknownFiles(outFile); + buildApk(outFile); // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it // lets restore the unedited one, to not change the original @@ -427,89 +422,54 @@ private void buildCopyOriginalFiles() throws AndrolibException { } } - private void buildUnknownFiles(File outFile) throws AndrolibException { - if (mApkInfo.unknownFiles != null) { - LOGGER.info("Copying unknown files/dir..."); - - Map files = mApkInfo.unknownFiles; - File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp"); - boolean renamed = outFile.renameTo(tempFile); - if (!renamed) { - throw new AndrolibException("Unable to rename temporary file"); - } - - try ( - ZipFile inputFile = new ZipFile(tempFile); - ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath())) - ) { - copyExistingFiles(inputFile, actualOutput); - copyUnknownFiles(actualOutput, files); - } catch (IOException | BrutException ex) { - throw new AndrolibException(ex); - } - - // Remove our temporary file. - //noinspection ResultOfMethodCallIgnored - tempFile.delete(); - } - } - - private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException { - // First, copy the contents from the existing outFile: - Enumeration entries = inputFile.entries(); - while (entries.hasMoreElements()) { - ZipEntry entry = new ZipEntry(entries.nextElement()); - - // We can't reuse the compressed size because it depends on compression sizes. - entry.setCompressedSize(-1); - outputFile.putNextEntry(entry); - - // No need to create directory entries in the final apk - if (!entry.isDirectory()) { - BrutIO.copy(inputFile, outputFile, entry); - } + private void copyUnknownFiles(ParallelScatterZipCreator zipCreator, Map files) + throws BrutException { + File unknownFileDir = new File(mApkDir, UNK_DIRNAME); - outputFile.closeEntry(); - } - } + try { + // loop through unknown files + for (Map.Entry unknownFileInfo : files.entrySet()) { + File inputFile; - private void copyUnknownFiles(ZipOutputStream outputFile, Map files) - throws BrutException, IOException { - File unknownFileDir = new File(mApkDir, UNK_DIRNAME); + try { + inputFile = new File(unknownFileDir, BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey())); + } catch (RootUnknownFileException | InvalidUnknownFileException | + TraversalUnknownFileException exception) { + LOGGER.warning(String.format("Skipping file %s (%s)", unknownFileInfo.getKey(), exception.getMessage())); + continue; + } - // loop through unknown files - for (Map.Entry unknownFileInfo : files.entrySet()) { - File inputFile; + if (inputFile.isDirectory()) { + continue; + } - try { - inputFile = new File(unknownFileDir, BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey())); - } catch (RootUnknownFileException | InvalidUnknownFileException | TraversalUnknownFileException exception) { - LOGGER.warning(String.format("Skipping file %s (%s)", unknownFileInfo.getKey(), exception.getMessage())); - continue; - } + ZipArchiveEntry newEntry = new ZipArchiveEntry(unknownFileInfo.getKey()); + int method = Integer.parseInt(unknownFileInfo.getValue()); + LOGGER.fine(String.format("Copying unknown file %s with method %d", unknownFileInfo.getKey(), method)); + if (method == ZipArchiveEntry.STORED) { + newEntry.setMethod(ZipArchiveEntry.STORED); + newEntry.setSize(inputFile.length()); + newEntry.setCompressedSize(-1); + BufferedInputStream unknownFile = new BufferedInputStream(Files.newInputStream(inputFile.toPath())); + CRC32 crc = BrutIO.calculateCrc(unknownFile); + newEntry.setCrc(crc.getValue()); + unknownFile.close(); + } else { + newEntry.setMethod(ZipArchiveEntry.DEFLATED); + } - if (inputFile.isDirectory()) { - continue; - } + InputStreamSupplier streamSupplier = () -> { + try { + return new FileInputStream(inputFile); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + }; - ZipEntry newEntry = new ZipEntry(unknownFileInfo.getKey()); - int method = Integer.parseInt(unknownFileInfo.getValue()); - LOGGER.fine(String.format("Copying unknown file %s with method %d", unknownFileInfo.getKey(), method)); - if (method == ZipEntry.STORED) { - newEntry.setMethod(ZipEntry.STORED); - newEntry.setSize(inputFile.length()); - newEntry.setCompressedSize(-1); - BufferedInputStream unknownFile = new BufferedInputStream(Files.newInputStream(inputFile.toPath())); - CRC32 crc = BrutIO.calculateCrc(unknownFile); - newEntry.setCrc(crc.getValue()); - unknownFile.close(); - } else { - newEntry.setMethod(ZipEntry.DEFLATED); + zipCreator.addArchiveEntry(newEntry, streamSupplier); } - outputFile.putNextEntry(newEntry); - - BrutIO.copy(inputFile, outputFile); - outputFile.closeEntry(); + }catch (IOException e){ + throw new BrutException(e); } } @@ -533,8 +493,10 @@ private void buildApk(File outApk) throws AndrolibException { } private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException { + Map files = mApkInfo.unknownFiles; + try { - ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress); + ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress, zipCreator -> copyUnknownFiles(zipCreator, files)); } catch (IOException | BrutException ex) { throw new AndrolibException(ex); } diff --git a/brut.j.dir/build.gradle.kts b/brut.j.dir/build.gradle.kts index d4865c8082..f46f1cd627 100644 --- a/brut.j.dir/build.gradle.kts +++ b/brut.j.dir/build.gradle.kts @@ -2,4 +2,5 @@ dependencies { implementation(project(":brut.j.common")) implementation(project(":brut.j.util")) implementation(libs.commons.io) + implementation(libs.commons.compress) } diff --git a/brut.j.dir/src/main/java/brut/directory/ZipUtils.java b/brut.j.dir/src/main/java/brut/directory/ZipUtils.java index 3e71990016..5e92c268b8 100644 --- a/brut.j.dir/src/main/java/brut/directory/ZipUtils.java +++ b/brut.j.dir/src/main/java/brut/directory/ZipUtils.java @@ -18,17 +18,20 @@ import brut.common.BrutException; import brut.util.BrutIO; +import org.apache.commons.compress.archivers.zip.*; +import org.apache.commons.compress.parallel.InputStreamSupplier; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; import java.io.*; import java.nio.file.Files; import java.util.Collection; +import java.util.concurrent.ExecutionException; import java.util.zip.CRC32; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; public class ZipUtils { + public interface AdditionalZipOperation { + void run(ParallelScatterZipCreator zipCreator) throws BrutException; + } private static Collection mDoNotCompress; @@ -36,53 +39,68 @@ private ZipUtils() { // Private constructor for utility class } - public static void zipFolders(final File folder, final File zip, final File assets, final Collection doNotCompress) + public static void zipFolders(final File folder, final File zip, final File assets, final Collection doNotCompress, final AdditionalZipOperation additionalZipOperation) throws BrutException, IOException { mDoNotCompress = doNotCompress; - ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(zip.toPath())); - zipFolders(folder, zipOutputStream); + ParallelScatterZipCreator zipCreator = new ParallelScatterZipCreator(); + zipFolders(folder, zipCreator); // We manually set the assets because we need to retain the folder structure if (assets != null) { - processFolder(assets, zipOutputStream, assets.getPath().length() - 6); + processFolder(assets, zipCreator, assets.getPath().length() - 6); + } + if (additionalZipOperation != null) { + additionalZipOperation.run(zipCreator); + } + + ZipArchiveOutputStream zipOutputStream = new ZipArchiveOutputStream(Files.newOutputStream(zip.toPath())); + zipOutputStream.setUseZip64(Zip64Mode.AsNeeded); + try { + zipCreator.writeTo(zipOutputStream); + } catch (InterruptedException | ExecutionException e) { + throw new BrutException(e); } zipOutputStream.close(); } - private static void zipFolders(final File folder, final ZipOutputStream outputStream) + private static void zipFolders(final File folder, final ParallelScatterZipCreator zipCreator) throws BrutException, IOException { - processFolder(folder, outputStream, folder.getPath().length() + 1); + processFolder(folder, zipCreator, folder.getPath().length() + 1); } - private static void processFolder(final File folder, final ZipOutputStream zipOutputStream, final int prefixLength) + private static void processFolder(final File folder, final ParallelScatterZipCreator zipCreator, final int prefixLength) throws BrutException, IOException { for (final File file : folder.listFiles()) { if (file.isFile()) { final String cleanedPath = BrutIO.sanitizeUnknownFile(folder, file.getPath().substring(prefixLength)); - final ZipEntry zipEntry = new ZipEntry(BrutIO.normalizePath(cleanedPath)); + final ZipArchiveEntry zipEntry = new ZipArchiveEntry(BrutIO.normalizePath(cleanedPath)); // aapt binary by default takes in parameters via -0 arsc to list extensions that shouldn't be // compressed. We will replicate that behavior final String extension = FilenameUtils.getExtension(file.getAbsolutePath()); if (mDoNotCompress != null && (mDoNotCompress.contains(extension) || mDoNotCompress.contains(zipEntry.getName()))) { - zipEntry.setMethod(ZipEntry.STORED); + zipEntry.setMethod(ZipArchiveEntry.STORED); zipEntry.setSize(file.length()); BufferedInputStream unknownFile = new BufferedInputStream(Files.newInputStream(file.toPath())); CRC32 crc = BrutIO.calculateCrc(unknownFile); zipEntry.setCrc(crc.getValue()); unknownFile.close(); } else { - zipEntry.setMethod(ZipEntry.DEFLATED); + zipEntry.setMethod(ZipArchiveEntry.DEFLATED); } - zipOutputStream.putNextEntry(zipEntry); - try (FileInputStream inputStream = new FileInputStream(file)) { - IOUtils.copy(inputStream, zipOutputStream); - } - zipOutputStream.closeEntry(); + InputStreamSupplier streamSupplier = () -> { + try { + return new FileInputStream(file); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + }; + + zipCreator.addArchiveEntry(zipEntry, streamSupplier); } else if (file.isDirectory()) { - processFolder(file, zipOutputStream, prefixLength); + processFolder(file, zipCreator, prefixLength); } } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1c5c3b3d50..508b3fecd6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,6 +2,7 @@ baksmali = "3.0.3" commons_io = "2.14.0" commons_cli = "1.5.0" +commons_compress = "1.24.0" commons_lang3 = "3.13.0" commons_text = "1.10.0" guava = "32.0.1-jre" @@ -16,6 +17,7 @@ xmlunit = "2.9.1" baksmali = { module = "com.android.tools.smali:smali-baksmali", version.ref = "baksmali" } commons_cli = { module = "commons-cli:commons-cli", version.ref = "commons_cli"} commons_io = { module = "commons-io:commons-io", version.ref = "commons_io" } +commons_compress = { module = "org.apache.commons:commons-compress", version.ref = "commons_compress" } commons_lang3 = { module = "org.apache.commons:commons-lang3", version.ref = "commons_lang3" } commons_text = { module = "org.apache.commons:commons-text", version.ref = "commons_text" } guava = { module = "com.google.guava:guava", version.ref = "guava" }