iBotPeaches · vbfox · Oct 26, 2023
diff --git a/brut.apktool/apktool-cli/build.gradle.kts b/brut.apktool/apktool-cli/build.gradle.kts
@@ -66,6 +66,9 @@ tasks.register<ProGuardTask>("proguard") {
     dontwarn("com.google.common.collect.**")
     dontwarn("com.google.common.util.**")
     dontwarn("javax.xml.xpath.**")
+    dontwarn("org.apache.commons.compress.harmony.**")
+    dontwarn("org.apache.commons.compress.compressors.**")
+    dontwarn("org.apache.commons.compress.archivers.**")
     dontnote("**")
 
     val outPath = "build/libs/apktool-$apktoolVersion.jar"

diff --git a/brut.apktool/apktool-lib/build.gradle.kts b/brut.apktool/apktool-lib/build.gradle.kts
@@ -33,6 +33,7 @@ dependencies {
     implementation(libs.smali)
     implementation(libs.xmlpull)
     implementation(libs.guava)
+    implementation(libs.commons.compress)
     implementation(libs.commons.lang3)
     implementation(libs.commons.io)
     implementation(libs.commons.text)

diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/ApkBuilder.java
@@ -33,6 +33,8 @@
 import brut.directory.ZipUtils;
 import brut.util.BrutIO;
 import brut.util.OS;
+import org.apache.commons.compress.archivers.zip.*;
+import org.apache.commons.compress.parallel.InputStreamSupplier;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.xml.sax.SAXException;
@@ -44,9 +46,6 @@
 import java.util.*;
 import java.util.logging.Logger;
 import java.util.zip.CRC32;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
-import java.util.zip.ZipOutputStream;
 
 public class ApkBuilder {
     private final static Logger LOGGER = Logger.getLogger(ApkBuilder.class.getName());
@@ -101,12 +100,8 @@ public void build(File outFile) throws BrutException {
         buildManifestFile(manifest, manifestOriginal);
         buildResources();
         buildLibs();
-        buildCopyOriginalFiles();
-        buildApk(outFile);
 
-        // we must go after the Apk is built, and copy the files in via Zip
-        // this is because Aapt won't add files it doesn't know (ex unknown files)
-        buildUnknownFiles(outFile);
+        buildApk(outFile);
 
         // we copied the AndroidManifest.xml to AndroidManifest.xml.orig so we can edit it
         // lets restore the unedited one, to not change the original
@@ -427,89 +422,54 @@ private void buildCopyOriginalFiles() throws AndrolibException {
         }
     }
 
-    private void buildUnknownFiles(File outFile) throws AndrolibException {
-        if (mApkInfo.unknownFiles != null) {
-            LOGGER.info("Copying unknown files/dir...");
-
-            Map<String, String> files = mApkInfo.unknownFiles;
-            File tempFile = new File(outFile.getParent(), outFile.getName() + ".apktool_temp");
-            boolean renamed = outFile.renameTo(tempFile);
-            if (!renamed) {
-                throw new AndrolibException("Unable to rename temporary file");
-            }
-
-            try (
-                ZipFile inputFile = new ZipFile(tempFile);
-                ZipOutputStream actualOutput = new ZipOutputStream(Files.newOutputStream(outFile.toPath()))
-            ) {
-                copyExistingFiles(inputFile, actualOutput);
-                copyUnknownFiles(actualOutput, files);
-            } catch (IOException | BrutException ex) {
-                throw new AndrolibException(ex);
-            }
-
-            // Remove our temporary file.
-            //noinspection ResultOfMethodCallIgnored
-            tempFile.delete();
-        }
-    }
-
-    private void copyExistingFiles(ZipFile inputFile, ZipOutputStream outputFile) throws IOException {
-        // First, copy the contents from the existing outFile:
-        Enumeration<? extends ZipEntry> entries = inputFile.entries();
-        while (entries.hasMoreElements()) {
-            ZipEntry entry = new ZipEntry(entries.nextElement());
-
-            // We can't reuse the compressed size because it depends on compression sizes.
-            entry.setCompressedSize(-1);
-            outputFile.putNextEntry(entry);
-
-            // No need to create directory entries in the final apk
-            if (!entry.isDirectory()) {
-                BrutIO.copy(inputFile, outputFile, entry);
-            }
+    private void copyUnknownFiles(ParallelScatterZipCreator zipCreator, Map<String, String> files)
+            throws BrutException {
+        File unknownFileDir = new File(mApkDir, UNK_DIRNAME);
 
-            outputFile.closeEntry();
-        }
-    }
+        try {
+            // loop through unknown files
+            for (Map.Entry<String, String> unknownFileInfo : files.entrySet()) {
+                File inputFile;
 
-    private void copyUnknownFiles(ZipOutputStream outputFile, Map<String, String> files)
-            throws BrutException, IOException {
-        File unknownFileDir = new File(mApkDir, UNK_DIRNAME);
+                try {
+                    inputFile = new File(unknownFileDir, BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey()));
+                } catch (RootUnknownFileException | InvalidUnknownFileException |
+                         TraversalUnknownFileException exception) {
+                    LOGGER.warning(String.format("Skipping file %s (%s)", unknownFileInfo.getKey(), exception.getMessage()));
+                    continue;
+                }
 
-        // loop through unknown files
-        for (Map.Entry<String,String> unknownFileInfo : files.entrySet()) {
-            File inputFile;
+                if (inputFile.isDirectory()) {
+                    continue;
+                }
 
-            try {
-                inputFile = new File(unknownFileDir, BrutIO.sanitizeUnknownFile(unknownFileDir, unknownFileInfo.getKey()));
-            } catch (RootUnknownFileException | InvalidUnknownFileException | TraversalUnknownFileException exception) {
-                LOGGER.warning(String.format("Skipping file %s (%s)", unknownFileInfo.getKey(), exception.getMessage()));
-                continue;
-            }
+                ZipArchiveEntry newEntry = new ZipArchiveEntry(unknownFileInfo.getKey());
+                int method = Integer.parseInt(unknownFileInfo.getValue());
+                LOGGER.fine(String.format("Copying unknown file %s with method %d", unknownFileInfo.getKey(), method));
+                if (method == ZipArchiveEntry.STORED) {
+                    newEntry.setMethod(ZipArchiveEntry.STORED);
+                    newEntry.setSize(inputFile.length());
+                    newEntry.setCompressedSize(-1);
+                    BufferedInputStream unknownFile = new BufferedInputStream(Files.newInputStream(inputFile.toPath()));
+                    CRC32 crc = BrutIO.calculateCrc(unknownFile);
+                    newEntry.setCrc(crc.getValue());
+                    unknownFile.close();
+                } else {
+                    newEntry.setMethod(ZipArchiveEntry.DEFLATED);
+                }
 
-            if (inputFile.isDirectory()) {
-                continue;
-            }
+                InputStreamSupplier streamSupplier = () -> {
+                    try {
+                        return new FileInputStream(inputFile);
+                    } catch (FileNotFoundException e) {
+                        throw new RuntimeException(e);
+                    }
+                };
 
-            ZipEntry newEntry = new ZipEntry(unknownFileInfo.getKey());
-            int method = Integer.parseInt(unknownFileInfo.getValue());
-            LOGGER.fine(String.format("Copying unknown file %s with method %d", unknownFileInfo.getKey(), method));
-            if (method == ZipEntry.STORED) {
-                newEntry.setMethod(ZipEntry.STORED);
-                newEntry.setSize(inputFile.length());
-                newEntry.setCompressedSize(-1);
-                BufferedInputStream unknownFile = new BufferedInputStream(Files.newInputStream(inputFile.toPath()));
-                CRC32 crc = BrutIO.calculateCrc(unknownFile);
-                newEntry.setCrc(crc.getValue());
-                unknownFile.close();
-            } else {
-                newEntry.setMethod(ZipEntry.DEFLATED);
+                zipCreator.addArchiveEntry(newEntry, streamSupplier);
             }
-            outputFile.putNextEntry(newEntry);
-
-            BrutIO.copy(inputFile, outputFile);
-            outputFile.closeEntry();
+        }catch (IOException e){
+            throw new BrutException(e);
         }
     }
 
@@ -533,8 +493,10 @@ private void buildApk(File outApk) throws AndrolibException {
     }
 
     private void zipPackage(File apkFile, File rawDir, File assetDir) throws AndrolibException {
+        Map<String, String> files = mApkInfo.unknownFiles;
+
         try {
-            ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress);
+            ZipUtils.zipFolders(rawDir, apkFile, assetDir, mApkInfo.doNotCompress, zipCreator -> copyUnknownFiles(zipCreator, files));
         } catch (IOException | BrutException ex) {
             throw new AndrolibException(ex);
         }

diff --git a/brut.j.dir/build.gradle.kts b/brut.j.dir/build.gradle.kts
@@ -2,4 +2,5 @@ dependencies {
   implementation(project(":brut.j.common"))
   implementation(project(":brut.j.util"))
   implementation(libs.commons.io)
+  implementation(libs.commons.compress)
 }
diff --git a/brut.j.dir/src/main/java/brut/directory/ZipUtils.java b/brut.j.dir/src/main/java/brut/directory/ZipUtils.java
@@ -18,71 +18,89 @@
 
 import brut.common.BrutException;
 import brut.util.BrutIO;
+import org.apache.commons.compress.archivers.zip.*;
+import org.apache.commons.compress.parallel.InputStreamSupplier;
 import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.io.IOUtils;
 
 import java.io.*;
 import java.nio.file.Files;
 import java.util.Collection;
+import java.util.concurrent.ExecutionException;
 import java.util.zip.CRC32;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 
 public class ZipUtils {
+    public interface AdditionalZipOperation {
+        void run(ParallelScatterZipCreator zipCreator) throws BrutException;
+    }
 
     private static Collection<String> mDoNotCompress;
 
     private ZipUtils() {
         // Private constructor for utility class
     }
 
-    public static void zipFolders(final File folder, final File zip, final File assets, final Collection<String> doNotCompress)
+    public static void zipFolders(final File folder, final File zip, final File assets, final Collection<String> doNotCompress, final AdditionalZipOperation additionalZipOperation)
             throws BrutException, IOException {
 
         mDoNotCompress = doNotCompress;
-        ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(zip.toPath()));
-        zipFolders(folder, zipOutputStream);
+        ParallelScatterZipCreator zipCreator = new ParallelScatterZipCreator();
+        zipFolders(folder, zipCreator);
 
         // We manually set the assets because we need to retain the folder structure
         if (assets != null) {
-            processFolder(assets, zipOutputStream, assets.getPath().length() - 6);
+            processFolder(assets, zipCreator, assets.getPath().length() - 6);
+        }
+        if (additionalZipOperation != null) {
+            additionalZipOperation.run(zipCreator);
+        }
+
+        ZipArchiveOutputStream zipOutputStream = new ZipArchiveOutputStream(Files.newOutputStream(zip.toPath()));
+        zipOutputStream.setUseZip64(Zip64Mode.AsNeeded);
+        try {
+            zipCreator.writeTo(zipOutputStream);
+        } catch (InterruptedException | ExecutionException e) {
+            throw new BrutException(e);
         }
         zipOutputStream.close();
     }
 
-    private static void zipFolders(final File folder, final ZipOutputStream outputStream)
+    private static void zipFolders(final File folder, final ParallelScatterZipCreator zipCreator)
             throws BrutException, IOException {
-        processFolder(folder, outputStream, folder.getPath().length() + 1);
+        processFolder(folder, zipCreator, folder.getPath().length() + 1);
     }
 
-    private static void processFolder(final File folder, final ZipOutputStream zipOutputStream, final int prefixLength)
+    private static void processFolder(final File folder, final ParallelScatterZipCreator zipCreator, final int prefixLength)
             throws BrutException, IOException {
         for (final File file : folder.listFiles()) {
             if (file.isFile()) {
                 final String cleanedPath = BrutIO.sanitizeUnknownFile(folder, file.getPath().substring(prefixLength));
-                final ZipEntry zipEntry = new ZipEntry(BrutIO.normalizePath(cleanedPath));
+                final ZipArchiveEntry zipEntry = new ZipArchiveEntry(BrutIO.normalizePath(cleanedPath));
 
                 // aapt binary by default takes in parameters via -0 arsc to list extensions that shouldn't be
                 // compressed. We will replicate that behavior
                 final String extension = FilenameUtils.getExtension(file.getAbsolutePath());
                 if (mDoNotCompress != null && (mDoNotCompress.contains(extension) || mDoNotCompress.contains(zipEntry.getName()))) {
-                    zipEntry.setMethod(ZipEntry.STORED);
+                    zipEntry.setMethod(ZipArchiveEntry.STORED);
                     zipEntry.setSize(file.length());
                     BufferedInputStream unknownFile = new BufferedInputStream(Files.newInputStream(file.toPath()));
                     CRC32 crc = BrutIO.calculateCrc(unknownFile);
                     zipEntry.setCrc(crc.getValue());
                     unknownFile.close();
                 } else {
-                    zipEntry.setMethod(ZipEntry.DEFLATED);
+                    zipEntry.setMethod(ZipArchiveEntry.DEFLATED);
                 }
 
-                zipOutputStream.putNextEntry(zipEntry);
-                try (FileInputStream inputStream = new FileInputStream(file)) {
-                    IOUtils.copy(inputStream, zipOutputStream);
-                }
-                zipOutputStream.closeEntry();
+                InputStreamSupplier streamSupplier = () -> {
+                    try {
+                        return new FileInputStream(file);
+                    } catch (FileNotFoundException e) {
+                        throw new RuntimeException(e);
+                    }
+                };
+
+                zipCreator.addArchiveEntry(zipEntry, streamSupplier);
             } else if (file.isDirectory()) {
-                processFolder(file, zipOutputStream, prefixLength);
+                processFolder(file, zipCreator, prefixLength);
             }
         }
     }

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -2,6 +2,7 @@
 baksmali = "3.0.3"
 commons_io = "2.14.0"
 commons_cli = "1.5.0"
+commons_compress = "1.24.0"
 commons_lang3 = "3.13.0"
 commons_text = "1.10.0"
 guava = "32.0.1-jre"
@@ -16,6 +17,7 @@ xmlunit = "2.9.1"
 baksmali = { module = "com.android.tools.smali:smali-baksmali", version.ref = "baksmali" }
 commons_cli = { module = "commons-cli:commons-cli", version.ref = "commons_cli"}
 commons_io = { module = "commons-io:commons-io", version.ref = "commons_io" }
+commons_compress = { module = "org.apache.commons:commons-compress", version.ref = "commons_compress" }
 commons_lang3 = { module = "org.apache.commons:commons-lang3", version.ref = "commons_lang3" }
 commons_text = { module = "org.apache.commons:commons-text", version.ref = "commons_text" }
 guava = { module = "com.google.guava:guava", version.ref = "guava" }