diff --git a/.gitignore b/.gitignore index 2080ab4..0f4e9a8 100644 --- a/.gitignore +++ b/.gitignore @@ -348,6 +348,7 @@ baserom_ntsc/ *.a *.z64 *.n64 +*.zip Extract/ tmp.txt diff --git a/OTRExporter/CMakeLists.txt b/OTRExporter/CMakeLists.txt index 517cb93..5debb0e 100644 --- a/OTRExporter/CMakeLists.txt +++ b/OTRExporter/CMakeLists.txt @@ -16,6 +16,8 @@ set(Header_Files "DisplayListExporter.h" "Exporter.h" "ExporterArchive.h" + "ExporterArchiveO2R.h" + "ExporterArchiveOTR.h" "Main.h" "MtxExporter.h" "PathExporter.h" @@ -46,6 +48,8 @@ set(Source_Files "DisplayListExporter.cpp" "Exporter.cpp" "ExporterArchive.cpp" + "ExporterArchiveO2R.cpp" + "ExporterArchiveOTR.cpp" "Main.cpp" "MtxExporter.cpp" "PathExporter.cpp" diff --git a/OTRExporter/ExporterArchive.cpp b/OTRExporter/ExporterArchive.cpp index a5d5c50..4fdb419 100644 --- a/OTRExporter/ExporterArchive.cpp +++ b/OTRExporter/ExporterArchive.cpp @@ -4,149 +4,8 @@ #include ExporterArchive::ExporterArchive(const std::string& path, bool enableWriting) : mPath(path) { - mMpq = nullptr; - Load(enableWriting); } ExporterArchive::~ExporterArchive() { - Unload(); } -bool ExporterArchive::Load(bool enableWriting) { - HANDLE mpqHandle = NULL; - - bool baseLoaded = false; - std::string fullPath = std::filesystem::absolute(mPath).string(); - - bool openArchiveSuccess; - { - const std::lock_guard lock(mMutex); - openArchiveSuccess = - SFileOpenArchive(fullPath.c_str(), 0, enableWriting ? 0 : MPQ_OPEN_READ_ONLY, &mpqHandle); - } - if (openArchiveSuccess) { - printf("Opened mpq file "); - printf(fullPath.c_str()); - printf("\n"); - mMpq = mpqHandle; - mPath = fullPath; - - - baseLoaded = true; - } - - if (!baseLoaded) { - printf("No valid OTR file was provided."); - return false; - } - - return true; -} - -bool ExporterArchive::Unload() { - bool success = true; - - bool closeArchiveSuccess; - { - const std::lock_guard lock(mMutex); - closeArchiveSuccess = SFileCloseArchive(mMpq); - } - if (!closeArchiveSuccess) { - printf("Failed to close mpq\n"); - success = false; - } - - mMpq = nullptr; - - return success; -} - -std::shared_ptr ExporterArchive::CreateArchive(const std::string& archivePath, size_t fileCapacity) { - auto archive = std::make_shared(archivePath, true); - - bool success; - { - const std::lock_guard lock(archive->mMutex); - success = SFileCreateArchive(archivePath.c_str(), MPQ_CREATE_LISTFILE | MPQ_CREATE_ATTRIBUTES | MPQ_CREATE_ARCHIVE_V2, - fileCapacity, &archive->mMpq); - } - int32_t error = GetLastError(); - - if (success) { - return archive; - } else { - printf("We tried to create an archive, but it has fallen and cannot get up.\n"); - return nullptr; - } -} - -bool ExporterArchive::AddFile(const std::string& filePath, uintptr_t fileData, DWORD fileSize) { - HANDLE hFile; -#ifdef _WIN32 - SYSTEMTIME sysTime; - GetSystemTime(&sysTime); - FILETIME t; - SystemTimeToFileTime(&sysTime, &t); - ULONGLONG theTime = static_cast(t.dwHighDateTime) << (sizeof(t.dwHighDateTime) * 8) | t.dwLowDateTime; -#else - time_t theTime; - time(&theTime); -#endif - - std::string updatedPath = filePath; - - StringHelper::ReplaceOriginal(updatedPath, "\\", "/"); - - bool createFileSuccess; - { - const std::lock_guard lock(mMutex); - createFileSuccess = - SFileCreateFile(mMpq, updatedPath.c_str(), theTime, fileSize, 0, MPQ_FILE_COMPRESS, &hFile); - } - if (!createFileSuccess) { - printf("Failed to create file.\n"); - return false; - } - - bool writeFileSuccess; - { - const std::lock_guard lock(mMutex); - writeFileSuccess = SFileWriteFile(hFile, (void*)fileData, fileSize, MPQ_COMPRESSION_ZLIB); - } - if (!writeFileSuccess) { - printf("Failed to write.\n"); - bool closeFileSuccess; - { - const std::lock_guard lock(mMutex); - closeFileSuccess = SFileCloseFile(hFile); - } - if (!closeFileSuccess) { - printf("Failed to close.\n"); - } - return false; - } - - bool finishFileSuccess; - { - const std::lock_guard lock(mMutex); - finishFileSuccess = SFileFinishFile(hFile); - } - if (!finishFileSuccess) { - printf("Failed to finish file.\n"); - bool closeFileSuccess; - { - const std::lock_guard lock(mMutex); - closeFileSuccess = SFileCloseFile(hFile); - } - if (!closeFileSuccess) { - printf("Failed to close after finish failure.\n"); - } - return false; - } - // SFileFinishFile already frees the handle, so no need to close it again. - - mAddedFiles.push_back(updatedPath); - mHashes[CRC64(updatedPath.c_str())] = updatedPath; - - return true; -} diff --git a/OTRExporter/ExporterArchive.h b/OTRExporter/ExporterArchive.h index fc65444..3070718 100644 --- a/OTRExporter/ExporterArchive.h +++ b/OTRExporter/ExporterArchive.h @@ -11,19 +11,16 @@ class ExporterArchive : public std::enable_shared_from_this { public: + ExporterArchive() {} ExporterArchive(const std::string& path, bool enableWriting); ~ExporterArchive(); - static std::shared_ptr CreateArchive(const std::string& archivePath, size_t fileCapacity); - bool AddFile(const std::string& filePath, uintptr_t fileData, DWORD fileSize); + virtual int CreateArchive(size_t fileCapacity) = 0; + virtual bool AddFile(const std::string& filePath, void* fileData, size_t fileSize) = 0; - private: std::string mPath; - HANDLE mMpq; - std::mutex mMutex; - std::vector mAddedFiles; - std::unordered_map mHashes; + std::mutex mMutex; - bool Load(bool enableWriting); - bool Unload(); + virtual bool Load(bool enableWriting) = 0; + virtual bool Unload() = 0; }; diff --git a/OTRExporter/ExporterArchiveO2R.cpp b/OTRExporter/ExporterArchiveO2R.cpp new file mode 100644 index 0000000..05fb981 --- /dev/null +++ b/OTRExporter/ExporterArchiveO2R.cpp @@ -0,0 +1,93 @@ +#include "ExporterArchiveO2R.h" +#include "Utils/StringHelper.h" +#include +#include +#include + +ExporterArchiveO2R::ExporterArchiveO2R(const std::string& path, bool enableWriting) { + mPath = path; + mZip = nullptr; +} + +ExporterArchiveO2R::~ExporterArchiveO2R() { + Unload(); +} + +bool ExporterArchiveO2R::Load(bool enableWriting) { + int openErr; + zip_t* archive = zip_open(mPath.c_str(), ZIP_CHECKCONS, &openErr); + + if (archive == nullptr) { + zip_error_t error; + zip_error_init_with_code(&error, openErr); + SPDLOG_ERROR("Failed to open ZIP (O2R) file. Error: {}", zip_error_strerror(&error)); + zip_error_fini(&error); + return false; + } + + SPDLOG_INFO("Loaded ZIP (O2R) archive: {}", mPath.c_str()); + mZip = archive; + + return true; +} + +bool ExporterArchiveO2R::Unload() { + printf("Unload\n"); + int err; + { + const std::lock_guard lock(mMutex); + err = zip_close(mZip); + } + if (err < 0) { + zip_error_t* zipError = zip_get_error(mZip); + SPDLOG_ERROR("Failed to close ZIP (O2R) file. Error: {}", zip_error_strerror(zipError)); + printf("fail\n"); + zip_error_fini(zipError); + return false; + } + + return true; +} + +int ExporterArchiveO2R::CreateArchive([[maybe_unused]] size_t fileCapacity) { + int openErr; + zip_t* zip; + + { + const std::lock_guard lock(mMutex); + zip = zip_open(mPath.c_str(), ZIP_CREATE, &openErr); + } + + if (zip == nullptr) { + zip_error_t error; + zip_error_init_with_code(&error, openErr); + SPDLOG_ERROR("Failed to create ZIP (O2R) file. Error: {}", zip_error_strerror(&error)); + zip_error_fini(&error); + return -1; + } + mZip = zip; + SPDLOG_INFO("Loaded ZIP (O2R) archive: {}", mPath.c_str()); + return 0; +} + +bool ExporterArchiveO2R::AddFile(const std::string& filePath, void* fileData, size_t fileSize) { + zip_source_t* source = zip_source_buffer(mZip, fileData, fileSize, 0); + + if (source == nullptr) { + zip_error_t* zipError = zip_get_error(mZip); + SPDLOG_ERROR("Failed to create ZIP source. Error: {}", zip_error_strerror(zipError)); + zip_source_free(source); + zip_error_fini(zipError); + return false; + } + + if (zip_file_add(mZip, filePath.c_str(), source, ZIP_FL_OVERWRITE | ZIP_FL_ENC_UTF_8) < 0) { + zip_error_t* zipError = zip_get_error(mZip); + SPDLOG_ERROR("Failed to add file to ZIP. Error: {}", zip_error_strerror(zipError)); + zip_source_free(source); + zip_error_fini(zipError); + return false; + } + + return true; +} diff --git a/OTRExporter/ExporterArchiveO2R.h b/OTRExporter/ExporterArchiveO2R.h new file mode 100644 index 0000000..2ef57cc --- /dev/null +++ b/OTRExporter/ExporterArchiveO2R.h @@ -0,0 +1,24 @@ +#pragma once + +#undef _DLL + +#include +#include +#include +#include +#include +#include +#include "ExporterArchive.h" + +class ExporterArchiveO2R : public ExporterArchive { + public: + ExporterArchiveO2R(const std::string& path, bool enableWriting); + ~ExporterArchiveO2R(); + + int CreateArchive(size_t fileCapacity) override; + bool AddFile(const std::string& filePath, void* fileData, size_t fileSize) override; + + zip_t* mZip; + bool Load(bool enableWriting) override; + bool Unload() override; +}; diff --git a/OTRExporter/ExporterArchiveOTR.cpp b/OTRExporter/ExporterArchiveOTR.cpp new file mode 100644 index 0000000..91952af --- /dev/null +++ b/OTRExporter/ExporterArchiveOTR.cpp @@ -0,0 +1,147 @@ +#include "ExporterArchiveOTR.h" +#include "Utils/StringHelper.h" +#include +#include + +ExporterArchiveOtr::ExporterArchiveOtr(const std::string& path, bool enableWriting) { + mPath = path; + mMpq = nullptr; +} + +ExporterArchiveOtr::~ExporterArchiveOtr() { + Unload(); +} + +bool ExporterArchiveOtr::Load(bool enableWriting) { + HANDLE mpqHandle = NULL; + + bool baseLoaded = false; + std::string fullPath = std::filesystem::absolute(mPath).string(); + + bool openArchiveSuccess; + { + const std::lock_guard lock(mMutex); + openArchiveSuccess = SFileOpenArchive(fullPath.c_str(), 0, enableWriting ? 0 : MPQ_OPEN_READ_ONLY, &mpqHandle); + } + if (openArchiveSuccess) { + printf("Opened mpq file "); + printf(fullPath.c_str()); + printf("\n"); + mMpq = mpqHandle; + mPath = fullPath; + baseLoaded = true; + } + + if (!baseLoaded) { + printf("No valid OTR file was provided."); + return false; + } + + return true; +} + +bool ExporterArchiveOtr::Unload() { + bool success = true; + + bool closeArchiveSuccess; + { + const std::lock_guard lock(mMutex); + closeArchiveSuccess = SFileCloseArchive(mMpq); + } + if (!closeArchiveSuccess) { + printf("Failed to close mpq\n"); + success = false; + } + + mMpq = nullptr; + + return success; +} + +int ExporterArchiveOtr::CreateArchive(size_t fileCapacity) { + bool success; + { + const std::lock_guard lock(mMutex); + success = SFileCreateArchive(mPath.c_str(), MPQ_CREATE_LISTFILE | MPQ_CREATE_ATTRIBUTES | MPQ_CREATE_ARCHIVE_V2, + static_cast(fileCapacity), &mMpq); + } + int32_t error = GetLastError(); + + if (success) { + return 0; + } else { + printf("We tried to create an archive, but it has fallen and cannot get up.\n"); + return -1; + } +} + +bool ExporterArchiveOtr::AddFile(const std::string& filePath, void* fileData, size_t fileSize) { + HANDLE hFile; +#ifdef _WIN32 + SYSTEMTIME sysTime; + GetSystemTime(&sysTime); + FILETIME t; + SystemTimeToFileTime(&sysTime, &t); + ULONGLONG theTime = static_cast(t.dwHighDateTime) << (sizeof(t.dwHighDateTime) * 8) | t.dwLowDateTime; +#else + time_t theTime; + time(&theTime); +#endif + + std::string updatedPath = filePath; + + StringHelper::ReplaceOriginal(updatedPath, "\\", "/"); + + bool createFileSuccess; + { + const std::lock_guard lock(mMutex); + createFileSuccess = + SFileCreateFile(mMpq, updatedPath.c_str(), theTime, static_cast(fileSize), 0, MPQ_FILE_COMPRESS, &hFile); + } + if (!createFileSuccess) { + printf("Failed to create file.\n"); + return false; + } + + bool writeFileSuccess; + { + const std::lock_guard lock(mMutex); + writeFileSuccess = SFileWriteFile(hFile, fileData, static_cast(fileSize), MPQ_COMPRESSION_ZLIB); + } + if (!writeFileSuccess) { + printf("Failed to write.\n"); + bool closeFileSuccess; + { + const std::lock_guard lock(mMutex); + closeFileSuccess = SFileCloseFile(hFile); + } + if (!closeFileSuccess) { + printf("Failed to close.\n"); + } + return false; + } + + bool finishFileSuccess; + { + const std::lock_guard lock(mMutex); + finishFileSuccess = SFileFinishFile(hFile); + } + if (!finishFileSuccess) { + printf("Failed to finish file.\n"); + bool closeFileSuccess; + { + const std::lock_guard lock(mMutex); + closeFileSuccess = SFileCloseFile(hFile); + } + if (!closeFileSuccess) { + printf("Failed to close after finish failure.\n"); + } + return false; + } + // SFileFinishFile already frees the handle, so no need to close it again. + + mAddedFiles.push_back(updatedPath); + mHashes[CRC64(updatedPath.c_str())] = updatedPath; + + return true; +} diff --git a/OTRExporter/ExporterArchiveOTR.h b/OTRExporter/ExporterArchiveOTR.h new file mode 100644 index 0000000..f9140ff --- /dev/null +++ b/OTRExporter/ExporterArchiveOTR.h @@ -0,0 +1,28 @@ +#pragma once + +#undef _DLL + +#include +#include +#include +#include +#include +#include +#include "ExporterArchive.h" + +class ExporterArchiveOtr : public ExporterArchive { + public: + ExporterArchiveOtr(const std::string& path, bool enableWriting); + ~ExporterArchiveOtr(); + + int CreateArchive(size_t fileCapacity) override; + bool AddFile(const std::string& filePath, void* fileData, size_t fileSize) override; + + bool Load(bool enableWriting) override; + bool Unload() override; + + private: + std::unordered_map mHashes; + std::vector mAddedFiles; + HANDLE mMpq; +}; diff --git a/OTRExporter/Main.cpp b/OTRExporter/Main.cpp index 99e8693..a5eaacc 100644 --- a/OTRExporter/Main.cpp +++ b/OTRExporter/Main.cpp @@ -28,8 +28,9 @@ #include #include #include +#include -std::string otrFileName = "mm.otr"; +std::string otrFileName = "mm.zip"; std::string customOtrFileName = ""; std::string customAssetsPath = ""; std::string portVersionString = "0.0.0"; @@ -44,295 +45,321 @@ void InitVersionInfo(); enum class ExporterFileMode { - BuildOTR = (int)ZFileMode::Custom + 1, + BuildOTR = (int)ZFileMode::Custom + 1, }; static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileMode) { - if (buildMode == "botr") - { - fileMode = (ZFileMode)ExporterFileMode::BuildOTR; + if (buildMode == "botr") + { + fileMode = (ZFileMode)ExporterFileMode::BuildOTR; - printf("BOTR: Generating OTR Archive...\n"); + printf("BOTR: Generating OTR Archive...\n"); - if (DiskFile::Exists(otrFileName)) - otrArchive = std::shared_ptr(new ExporterArchive(otrFileName, true)); - else - otrArchive = ExporterArchive::CreateArchive(otrFileName, 40000); + otrArchive = std::make_shared(otrFileName, true); - auto lst = Directory::ListFiles("Extract"); + if (DiskFile::Exists(otrFileName)) + otrArchive->Load(true); + else + otrArchive->CreateArchive(40000); - for (auto item : lst) - { - auto fileData = DiskFile::ReadAllBytes(item); - otrArchive->AddFile(StringHelper::Split(item, "Extract/")[1], (uintptr_t)fileData.data(), fileData.size()); - } - } + auto lst = Directory::ListFiles("Extract"); + + for (auto item : lst) + { + auto fileData = DiskFile::ReadAllBytes(item); + otrArchive->AddFile(StringHelper::Split(item, "Extract/")[1], fileData.data(), fileData.size()); + } + } } +typedef struct Data { + std::vector fileData; + std::string filePath; + size_t size; +} Data; + +typedef struct DataU { + std::vector fileData; + std::string filePath; + size_t size; +} DataU; + static void ExporterProgramEnd() { - uint32_t crc = 0xFFFFFFFF; - const uint8_t endianness = (uint8_t)Endianness::Big; - - std::vector portVersion = {}; - std::vector versionParts = StringHelper::Split(portVersionString, "."); - - // If a major.minor.patch string was not passed in, fallback to 0 0 0 - if (versionParts.size() != 3) { - portVersion = { 0, 0, 0 }; - } else { - // Parse version values to number - for (const auto& val : versionParts) { - uint16_t num = 0; - try { - num = (uint16_t)std::stoi(val, nullptr); - } catch (std::invalid_argument &e) { - num = 0; - } catch (std::out_of_range &e) { - num = 0; - } - - portVersion.push_back(num); - } - } - - MemoryStream *portVersionStream = new MemoryStream(); - BinaryWriter portVerWriter(portVersionStream); - portVerWriter.SetEndianness(Endianness::Big); - portVerWriter.Write(endianness); - portVerWriter.Write(portVersion[0]); // Major - portVerWriter.Write(portVersion[1]); // Minor - portVerWriter.Write(portVersion[2]); // Patch - portVerWriter.Close(); - - if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory) - { - std::string romPath = Globals::Instance->baseRomPath.string(); - std::vector romData = DiskFile::ReadAllBytes(romPath); - - BitConverter::RomToBigEndian(romData.data(), romData.size()); - - crc = BitConverter::ToUInt32BE(romData, 0x10); - printf("Creating version file...\n"); - - // Get crc from rom - - MemoryStream *versionStream = new MemoryStream(); - BinaryWriter writer(versionStream); - writer.SetEndianness(Endianness::Big); - writer.Write(endianness); - writer.Write(crc); - writer.Close(); - - printf("Created version file.\n"); - - printf("Generating OTR Archive...\n"); - otrArchive = ExporterArchive::CreateArchive(otrFileName, 40000); - - printf("Adding game version file.\n"); - otrArchive->AddFile("version", (uintptr_t)versionStream->ToVector().data(), versionStream->GetLength()); - - printf("Adding portVersion file.\n"); - otrArchive->AddFile("portVersion", (uintptr_t)portVersionStream->ToVector().data(), portVersionStream->GetLength()); - - for (const auto& item : files) - { - std::string fName = item.first; - if (fName.find("gTitleZeldaShieldLogoMQTex") != std::string::npos && !ZRom(romPath).IsMQ()) - { - size_t pos = 0; - if ((pos = fName.find("gTitleZeldaShieldLogoMQTex", 0)) != std::string::npos) - { - fName.replace(pos, 27, "gTitleZeldaShieldLogoTex"); - } - } - const auto& fileData = item.second; - otrArchive->AddFile(fName, (uintptr_t)fileData.data(), - fileData.size()); - } - } - - otrArchive = nullptr; - delete fileWriter; - files.clear(); - - // Generate custom otr file for extra assets - if (customAssetsPath == "" || customOtrFileName == "" || DiskFile::Exists(customOtrFileName)) { - printf("No Custom Assets path or otr file name provided, otr file already exists. Nothing to do.\n"); - return; - } - - if (!customAssetsPath.ends_with("/")) { - customAssetsPath += "/"; - } - - const auto& lst = Directory::ListFiles(customAssetsPath); - - printf("Generating Custom OTR Archive...\n"); - std::shared_ptr customOtr = ExporterArchive::CreateArchive(customOtrFileName, 4096); - - printf("Adding portVersion file.\n"); - customOtr->AddFile("portVersion", (uintptr_t)portVersionStream->ToVector().data(), portVersionStream->GetLength()); - - for (const auto& item : lst) - { - size_t filenameSepAt = item.find_last_of("/\\"); - const std::string filename = item.substr(filenameSepAt + 1); - - if (std::count(filename.begin(), filename.end(), '.') >= 2) - { - size_t extensionSepAt = filename.find_last_of("."); - size_t formatSepAt = filename.find_last_of(".", extensionSepAt - 1); - - const std::string extension = filename.substr(extensionSepAt + 1); - const std::string format = filename.substr(formatSepAt + 1, extensionSepAt - formatSepAt - 1); - std::string afterPath = item.substr(0, filenameSepAt + formatSepAt + 1); - - if (extension == "png" && (format == "rgba32" || format == "rgb5a1" || format == "i4" || format == "i8" || format == "ia4" || format == "ia8" || format == "ia16" || format == "ci4" || format == "ci8")) - { - ZTexture tex(nullptr); - Globals::Instance->buildRawTexture = true; - tex.FromPNG(item, ZTexture::GetTextureTypeFromString(format)); - printf("customOtr->AddFile(%s)\n", StringHelper::Split(afterPath, customAssetsPath)[1].c_str()); - - OTRExporter_Texture exporter; - - MemoryStream* stream = new MemoryStream(); - BinaryWriter writer(stream); - - exporter.Save(&tex, "", &writer); - - std::string src = tex.GetBodySourceCode(); - writer.Write((char *)src.c_str(), src.size()); - - std::vector fileData = stream->ToVector(); - customOtr->AddFile(StringHelper::Split(afterPath, customAssetsPath)[1], (uintptr_t)fileData.data(), fileData.size()); - continue; - } - } - - if (item.find("accessibility") != std::string::npos) - { - std::string extension = filename.substr(filename.find_last_of(".") + 1); - if (extension == "json") - { - const auto &fileData = DiskFile::ReadAllBytes(item); - printf("Adding accessibility texts %s\n", StringHelper::Split(item, customAssetsPath)[1].c_str()); - customOtr->AddFile(StringHelper::Split(item, customAssetsPath)[1], (uintptr_t)fileData.data(), fileData.size()); - } - continue; - } - - const auto& fileData = DiskFile::ReadAllBytes(item); - printf("customOtr->AddFile(%s)\n", StringHelper::Split(item, customAssetsPath)[1].c_str()); - customOtr->AddFile(StringHelper::Split(item, customAssetsPath)[1], (uintptr_t)fileData.data(), fileData.size()); - } - - customOtr = nullptr; + uint32_t crc = 0xFFFFFFFF; + const uint8_t endianness = (uint8_t)Endianness::Big; + + std::vector portVersion = {}; + std::vector versionParts = StringHelper::Split(portVersionString, "."); + + // If a major.minor.patch string was not passed in, fallback to 0 0 0 + if (versionParts.size() != 3) { + portVersion = { 0, 0, 0 }; + } else { + // Parse version values to number + for (const auto& val : versionParts) { + uint16_t num = 0; + try { + num = (uint16_t)std::stoi(val, nullptr); + } catch (std::invalid_argument &e) { + num = 0; + } catch (std::out_of_range &e) { + num = 0; + } + + portVersion.push_back(num); + } + } + + MemoryStream *portVersionStream = new MemoryStream(); + BinaryWriter portVerWriter(portVersionStream); + portVerWriter.SetEndianness(Endianness::Big); + portVerWriter.Write(endianness); + portVerWriter.Write(portVersion[0]); // Major + portVerWriter.Write(portVersion[1]); // Minor + portVerWriter.Write(portVersion[2]); // Patch + portVerWriter.Close(); + + if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory) + { + std::string romPath = Globals::Instance->baseRomPath.string(); + std::vector romData = DiskFile::ReadAllBytes(romPath); + + BitConverter::RomToBigEndian(romData.data(), romData.size()); + + crc = BitConverter::ToUInt32BE(romData, 0x10); + printf("Creating version file...\n"); + + // Get crc from rom + + MemoryStream *versionStream = new MemoryStream(); + BinaryWriter writer(versionStream); + writer.SetEndianness(Endianness::Big); + writer.Write(endianness); + writer.Write(crc); + writer.Close(); + + printf("Created version file.\n"); + + printf("Generating OTR Archive...\n"); + otrArchive = std::make_shared(otrFileName, true); + otrArchive->CreateArchive(40000); + + printf("Adding game version file.\n"); + otrArchive->AddFile("version", versionStream->ToVector().data(), versionStream->GetLength()); + + printf("Adding portVersion file.\n"); + otrArchive->AddFile("portVersion", portVersionStream->ToVector().data(), portVersionStream->GetLength()); + + for (const auto& item : files) + { + std::string fName = item.first; + if (fName.find("gTitleZeldaShieldLogoMQTex") != std::string::npos && !ZRom(romPath).IsMQ()) + { + size_t pos = 0; + if ((pos = fName.find("gTitleZeldaShieldLogoMQTex", 0)) != std::string::npos) + { + fName.replace(pos, 27, "gTitleZeldaShieldLogoTex"); + } + } + const auto& fileData = item.second; + otrArchive->AddFile(fName, (void*)fileData.data(), fileData.size()); + } + } + + otrArchive = nullptr; + delete fileWriter; + files.clear(); + + // Generate custom otr file for extra assets + if (customAssetsPath == "" || customOtrFileName == "" || DiskFile::Exists(customOtrFileName)) { + printf("No Custom Assets path or otr file name provided, otr file already exists. Nothing to do.\n"); + return; + } + + if (!customAssetsPath.ends_with("/")) { + customAssetsPath += "/"; + } + + const auto& lst = Directory::ListFiles(customAssetsPath); + + printf("Generating Custom OTR Archive...\n"); + auto customOtr = std::make_unique(customOtrFileName, true); + customOtr->CreateArchive(40000); + + printf("Adding portVersion file.\n"); + customOtr->AddFile("portVersion", portVersionStream->ToVector().data(), portVersionStream->GetLength()); + + std::vector dataVec; + std::vector dataVec2; + + + for (const auto& item : lst) + { + size_t filenameSepAt = item.find_last_of("/\\"); + const std::string filename = item.substr(filenameSepAt + 1); + + if (std::count(filename.begin(), filename.end(), '.') >= 2) + { + size_t extensionSepAt = filename.find_last_of("."); + size_t formatSepAt = filename.find_last_of(".", extensionSepAt - 1); + + const std::string extension = filename.substr(extensionSepAt + 1); + const std::string format = filename.substr(formatSepAt + 1, extensionSepAt - formatSepAt - 1); + std::string afterPath = item.substr(0, filenameSepAt + formatSepAt + 1); + + if (extension == "png" && (format == "rgba32" || format == "rgb5a1" || format == "i4" || format == "i8" || format == "ia4" || format == "ia8" || format == "ia16" || format == "ci4" || format == "ci8")) + { + ZTexture tex(nullptr); + Globals::Instance->buildRawTexture = true; + tex.FromPNG(item, ZTexture::GetTextureTypeFromString(format)); + printf("customOtr->AddFile(%s)\n", StringHelper::Split(afterPath, customAssetsPath)[1].c_str()); + + OTRExporter_Texture exporter; + + MemoryStream* stream = new MemoryStream(); + BinaryWriter writer(stream); + + exporter.Save(&tex, "", &writer); + + std::string src = tex.GetBodySourceCode(); + writer.Write((char *)src.c_str(), src.size()); + + std::vector fileData = stream->ToVector(); + dataVec.push_back({ fileData, StringHelper::Split(afterPath, customAssetsPath)[1], fileData.size() }); + continue; + } + } + + if (item.find("accessibility") != std::string::npos) + { + std::string extension = filename.substr(filename.find_last_of(".") + 1); + if (extension == "json") + { + const auto &fileData = DiskFile::ReadAllBytes(item); + printf("Adding accessibility texts %s\n", StringHelper::Split(item, customAssetsPath)[1].c_str()); + customOtr->AddFile(StringHelper::Split(item, customAssetsPath)[1], const_cast(fileData.data()), fileData.size()); + } + continue; + } + + const auto& fileData = DiskFile::ReadAllBytes(item); + printf("customOtr->AddFile(%s)\n", StringHelper::Split(item, customAssetsPath)[1].c_str()); + dataVec2.push_back({ fileData, StringHelper::Split(item, customAssetsPath)[1], fileData.size() }); + } + for (auto& d : dataVec) { + customOtr->AddFile(d.filePath, d.fileData.data(), d.size); + } + + for (auto& d : dataVec2) { + customOtr->AddFile(d.filePath, d.fileData.data(), d.size); + } + + printf("Done\n"); } static void ExporterParseArgs(int argc, char* argv[], int& i) { - std::string arg = argv[i]; - - if (arg == "--otrfile") { - otrFileName = argv[i + 1]; - i++; - } else if (arg == "--customOtrFile") { - customOtrFileName = argv[i + 1]; - i++; - } else if (arg == "--customAssetsPath") { - customAssetsPath = argv[i + 1]; - i++; - } else if (arg == "--portVer") { - portVersionString = argv[i + 1]; - i++; - } + std::string arg = argv[i]; + + if (arg == "--otrfile") { + otrFileName = argv[i + 1]; + i++; + } else if (arg == "--customOtrFile") { + customOtrFileName = argv[i + 1]; + i++; + } else if (arg == "--customAssetsPath") { + customAssetsPath = argv[i + 1]; + i++; + } else if (arg == "--portVer") { + portVersionString = argv[i + 1]; + i++; + } } static bool ExporterProcessFileMode(ZFileMode fileMode) { - // Do whatever work is associated with these custom file modes... - // Return true to indicate one of our own file modes is being processed - if (fileMode == (ZFileMode)ExporterFileMode::BuildOTR) - return true; + // Do whatever work is associated with these custom file modes... + // Return true to indicate one of our own file modes is being processed + if (fileMode == (ZFileMode)ExporterFileMode::BuildOTR) + return true; - return false; + return false; } static void ExporterFileBegin(ZFile* file) { - fileStart = std::chrono::steady_clock::now(); + fileStart = std::chrono::steady_clock::now(); - MemoryStream* stream = new MemoryStream(); - fileWriter = new BinaryWriter(stream); + MemoryStream* stream = new MemoryStream(); + fileWriter = new BinaryWriter(stream); } static void ExporterFileEnd(ZFile* file) { - // delete fileWriter; + // delete fileWriter; } static void ExporterResourceEnd(ZResource* res, BinaryWriter& writer) { - auto streamShared = writer.GetStream(); - MemoryStream* strem = (MemoryStream*)streamShared.get(); - - auto start = std::chrono::steady_clock::now(); - - if (res->GetName() != "") - { - std::string oName = res->parent->GetOutName(); - std::string rName = res->GetName(); - std::string prefix = OTRExporter_DisplayList::GetPrefix(res); - - //auto xmlFilePath = res->parent->GetXmlFilePath(); - //prefix = StringHelper::Split(StringHelper::Split(xmlFilePath.string(), "xml\\")[1], ".xml")[0]; - - if (StringHelper::Contains(oName, "_scene")) - { - auto split = StringHelper::Split(oName, "_"); - oName = ""; - for (size_t i = 0; i < split.size() - 1; i++) - oName += split[i] + "_"; - - oName += "scene"; - } - else if (StringHelper::Contains(oName, "_room")) - { - if (Globals::Instance->game != ZGame::MM_RETAIL) - oName = StringHelper::Split(oName, "_room")[0] + "_scene"; - else - oName = StringHelper::Split(oName, "_room")[0]; - } - - std::string fName = ""; - - if (prefix != "") - fName = StringHelper::Sprintf("%s/%s/%s", prefix.c_str(), oName.c_str(), rName.c_str()); - else - fName = StringHelper::Sprintf("%s/%s", oName.c_str(), rName.c_str()); - - if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory) - { - std::unique_lock Lock(fileMutex); - files[fName] = strem->ToVector(); - } - else - DiskFile::WriteAllBytes("Extract/" + fName, strem->ToVector()); - } - - auto end = std::chrono::steady_clock::now(); - size_t diff = std::chrono::duration_cast(end - start).count(); - - //if (diff > 10) - //printf("Exported Resource End %s in %zums\n", res->GetName().c_str(), diff); + auto streamShared = writer.GetStream(); + MemoryStream* strem = (MemoryStream*)streamShared.get(); + + auto start = std::chrono::steady_clock::now(); + + if (res->GetName() != "") + { + std::string oName = res->parent->GetOutName(); + std::string rName = res->GetName(); + std::string prefix = OTRExporter_DisplayList::GetPrefix(res); + + //auto xmlFilePath = res->parent->GetXmlFilePath(); + //prefix = StringHelper::Split(StringHelper::Split(xmlFilePath.string(), "xml\\")[1], ".xml")[0]; + + if (StringHelper::Contains(oName, "_scene")) + { + auto split = StringHelper::Split(oName, "_"); + oName = ""; + for (size_t i = 0; i < split.size() - 1; i++) + oName += split[i] + "_"; + + oName += "scene"; + } + else if (StringHelper::Contains(oName, "_room")) + { + if (Globals::Instance->game != ZGame::MM_RETAIL) + oName = StringHelper::Split(oName, "_room")[0] + "_scene"; + else + oName = StringHelper::Split(oName, "_room")[0]; + } + + std::string fName = ""; + + if (prefix != "") + fName = StringHelper::Sprintf("%s/%s/%s", prefix.c_str(), oName.c_str(), rName.c_str()); + else + fName = StringHelper::Sprintf("%s/%s", oName.c_str(), rName.c_str()); + + if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory) + { + std::unique_lock Lock(fileMutex); + files[fName] = strem->ToVector(); + } + else + DiskFile::WriteAllBytes("Extract/" + fName, strem->ToVector()); + } + + auto end = std::chrono::steady_clock::now(); + size_t diff = std::chrono::duration_cast(end - start).count(); + + //if (diff > 10) + //printf("Exported Resource End %s in %zums\n", res->GetName().c_str(), diff); } static void ExporterProcessCompilable(tinyxml2::XMLElement* reader) { - std::string nodeName = reader->Name(); + std::string nodeName = reader->Name(); } static void ExporterXMLBegin() @@ -345,55 +372,55 @@ static void ExporterXMLEnd() void AddFile(std::string fName, std::vector data) { - if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory) - DiskFile::WriteAllBytes("Extract/" + fName, data); - else - { - std::unique_lock Lock(fileMutex); - files[fName] = data; - } + if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory) + DiskFile::WriteAllBytes("Extract/" + fName, data); + else + { + std::unique_lock Lock(fileMutex); + files[fName] = data; + } } void ImportExporters() { - // In this example we set up a new exporter called "EXAMPLE". - // By running ZAPD with the argument -se EXAMPLE, we tell it that we want to use this exporter for our resources. - ExporterSet* exporterSet = new ExporterSet(); - exporterSet->processFileModeFunc = ExporterProcessFileMode; - exporterSet->parseFileModeFunc = ExporterParseFileMode; - exporterSet->processCompilableFunc = ExporterProcessCompilable; - exporterSet->parseArgsFunc = ExporterParseArgs; - exporterSet->beginFileFunc = ExporterFileBegin; - exporterSet->endFileFunc = ExporterFileEnd; - exporterSet->beginXMLFunc = ExporterXMLBegin; - exporterSet->endXMLFunc = ExporterXMLEnd; - exporterSet->resSaveFunc = ExporterResourceEnd; - exporterSet->endProgramFunc = ExporterProgramEnd; - - exporterSet->exporters[ZResourceType::Background] = new OTRExporter_Background(); - exporterSet->exporters[ZResourceType::Texture] = new OTRExporter_Texture(); - exporterSet->exporters[ZResourceType::Room] = new OTRExporter_Room(); - exporterSet->exporters[ZResourceType::AltHeader] = new OTRExporter_Room(); - exporterSet->exporters[ZResourceType::Scene] = new OTRExporter_Room(); - exporterSet->exporters[ZResourceType::CollisionHeader] = new OTRExporter_Collision(); - exporterSet->exporters[ZResourceType::DisplayList] = new OTRExporter_DisplayList(); - exporterSet->exporters[ZResourceType::PlayerAnimationData] = new OTRExporter_PlayerAnimationExporter(); - exporterSet->exporters[ZResourceType::Skeleton] = new OTRExporter_Skeleton(); - exporterSet->exporters[ZResourceType::Limb] = new OTRExporter_SkeletonLimb(); - exporterSet->exporters[ZResourceType::Animation] = new OTRExporter_Animation(); - exporterSet->exporters[ZResourceType::Cutscene] = new OTRExporter_Cutscene(); - exporterSet->exporters[ZResourceType::Vertex] = new OTRExporter_Vtx(); - exporterSet->exporters[ZResourceType::Array] = new OTRExporter_Array(); - exporterSet->exporters[ZResourceType::Path] = new OTRExporter_Path(); - exporterSet->exporters[ZResourceType::Text] = new OTRExporter_Text(); - exporterSet->exporters[ZResourceType::TextMM] = new OTRExporter_TextMM(); - exporterSet->exporters[ZResourceType::Blob] = new OTRExporter_Blob(); - exporterSet->exporters[ZResourceType::Mtx] = new OTRExporter_MtxExporter(); - exporterSet->exporters[ZResourceType::Audio] = new OTRExporter_Audio(); - exporterSet->exporters[ZResourceType::TextureAnimation] = new OTRExporter_TextureAnimation(); + // In this example we set up a new exporter called "EXAMPLE". + // By running ZAPD with the argument -se EXAMPLE, we tell it that we want to use this exporter for our resources. + ExporterSet* exporterSet = new ExporterSet(); + exporterSet->processFileModeFunc = ExporterProcessFileMode; + exporterSet->parseFileModeFunc = ExporterParseFileMode; + exporterSet->processCompilableFunc = ExporterProcessCompilable; + exporterSet->parseArgsFunc = ExporterParseArgs; + exporterSet->beginFileFunc = ExporterFileBegin; + exporterSet->endFileFunc = ExporterFileEnd; + exporterSet->beginXMLFunc = ExporterXMLBegin; + exporterSet->endXMLFunc = ExporterXMLEnd; + exporterSet->resSaveFunc = ExporterResourceEnd; + exporterSet->endProgramFunc = ExporterProgramEnd; + + exporterSet->exporters[ZResourceType::Background] = new OTRExporter_Background(); + exporterSet->exporters[ZResourceType::Texture] = new OTRExporter_Texture(); + exporterSet->exporters[ZResourceType::Room] = new OTRExporter_Room(); + exporterSet->exporters[ZResourceType::AltHeader] = new OTRExporter_Room(); + exporterSet->exporters[ZResourceType::Scene] = new OTRExporter_Room(); + exporterSet->exporters[ZResourceType::CollisionHeader] = new OTRExporter_Collision(); + exporterSet->exporters[ZResourceType::DisplayList] = new OTRExporter_DisplayList(); + exporterSet->exporters[ZResourceType::PlayerAnimationData] = new OTRExporter_PlayerAnimationExporter(); + exporterSet->exporters[ZResourceType::Skeleton] = new OTRExporter_Skeleton(); + exporterSet->exporters[ZResourceType::Limb] = new OTRExporter_SkeletonLimb(); + exporterSet->exporters[ZResourceType::Animation] = new OTRExporter_Animation(); + exporterSet->exporters[ZResourceType::Cutscene] = new OTRExporter_Cutscene(); + exporterSet->exporters[ZResourceType::Vertex] = new OTRExporter_Vtx(); + exporterSet->exporters[ZResourceType::Array] = new OTRExporter_Array(); + exporterSet->exporters[ZResourceType::Path] = new OTRExporter_Path(); + exporterSet->exporters[ZResourceType::Text] = new OTRExporter_Text(); + exporterSet->exporters[ZResourceType::TextMM] = new OTRExporter_TextMM(); + exporterSet->exporters[ZResourceType::Blob] = new OTRExporter_Blob(); + exporterSet->exporters[ZResourceType::Mtx] = new OTRExporter_MtxExporter(); + exporterSet->exporters[ZResourceType::Audio] = new OTRExporter_Audio(); + exporterSet->exporters[ZResourceType::TextureAnimation] = new OTRExporter_TextureAnimation(); exporterSet->exporters[ZResourceType::KeyFrameSkel] = new OTRExporter_CKeyFrameSkel(); exporterSet->exporters[ZResourceType::KeyFrameAnimation] = new OTRExporter_CKeyFrameAnim(); - Globals::AddExporter("OTR", exporterSet); + Globals::AddExporter("OTR", exporterSet); - InitVersionInfo(); + InitVersionInfo(); } diff --git a/extract_assets.py b/extract_assets.py index bc32572..2e8dc61 100755 --- a/extract_assets.py +++ b/extract_assets.py @@ -21,7 +21,7 @@ def BuildOTR(xmlPath, rom, zapd_exe=None, genHeaders=None, customAssetsPath=None else: # generate otrs, but not headers exec_cmd.extend(["-gsf", "0", "-se", "OTR", "--customAssetsPath", customAssetsPath, - "--customOtrFile", customOtrFile, "--otrfile", "mm.otr"]) + "--customOtrFile", customOtrFile, "--otrfile", "mm.zip"]) if portVer: exec_cmd.extend(["--portVer", portVer])