Updated to use latest version of ZAPD (#777)

* Updated config file

* Added missing files

* Temporarily removed asm_processor changes.

* git subrepo pull --force tools/ZAPD

subrepo:
  subdir:   "tools/ZAPD"
  merged:   "96ffc1e62"
upstream:
  origin:   "https://github.com/zeldaret/ZAPD.git"
  branch:   "master"
  commit:   "96ffc1e62"
git-subrepo:
  version:  "0.4.3"
  origin:   "???"
  commit:   "???"

* git subrepo pull --force tools/ZAPD

subrepo:
  subdir:   "tools/ZAPD"
  merged:   "179af7d11"
upstream:
  origin:   "https://github.com/zeldaret/ZAPD.git"
  branch:   "master"
  commit:   "179af7d11"
git-subrepo:
  version:  "0.4.3"
  origin:   "???"
  commit:   "???"

* Cleanup and fixes.

* git subrepo pull --force tools/ZAPD

subrepo:
  subdir:   "tools/ZAPD"
  merged:   "50ad2fe78"
upstream:
  origin:   "https://github.com/zeldaret/ZAPD.git"
  branch:   "master"
  commit:   "50ad2fe78"
git-subrepo:
  version:  "0.4.3"
  origin:   "???"
  commit:   "???"

* Makefile fix

* git subrepo pull --force tools/ZAPD

subrepo:
  subdir:   "tools/ZAPD"
  merged:   "b9120803e"
upstream:
  origin:   "https://github.com/zeldaret/ZAPD.git"
  branch:   "master"
  commit:   "b9120803e"
git-subrepo:
  version:  "0.4.3"
  origin:   "???"
  commit:   "???"

Co-authored-by: Jack Walker <[email protected]>

Makefile

@@ -167,6 +167,8 @@ build/src/overlays/actors/%.o: CC := python3 tools/asm_processor/build.py $(CC)
 build/src/overlays/effects/%.o: CC := python3 tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
 build/src/overlays/gamestates/%.o: CC := python3 tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
 
+build/assets/%.o: CC := python3 tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
+
 #### Main Targets ###
 
 all: $(ROM)

assets/.gitignore

@@ -1,4 +1,6 @@
 *.bin
 *.c
 *.h
-*.cfg
+*.cfg
+*.vtx.inc
+*.dlist.inc

extract_assets.py

@@ -6,25 +6,26 @@
 from multiprocessing import Pool
 from multiprocessing import cpu_count
 
-def Extract(xmlPath, outputPath):
-	ExtractFile(xmlPath, outputPath, 1, 0)	
+def Extract(xmlPath, outputPath, outputSourcePath):
+	ExtractFile(xmlPath, outputPath, outputSourcePath, 1, 0)	
 
-def ExtractScene(xmlPath, outputPath):
-	ExtractFile(xmlPath, outputPath, 1, 1)
+def ExtractScene(xmlPath, outputPath, outputSourcePath):
+	ExtractFile(xmlPath, outputPath, outputSourcePath, 1, 1)
 
-def ExtractFile(xmlPath, outputPath, genSrcFile, incFilePrefix):
-	execStr = "tools/ZAPD/ZAPD.out e -eh -i %s -b baserom/ -o %s -gsf %i -ifp %i -sm tools/ZAPD/SymbolMap_OoTMqDbg.txt" % (xmlPath, outputPath, genSrcFile, incFilePrefix)
+def ExtractFile(xmlPath, outputPath, outputSourcePath, genSrcFile, incFilePrefix):
+	execStr = "tools/ZAPD/ZAPD.out e -eh -i %s -b baserom/ -o %s -osf %s -gsf %i -ifp %i -rconf tools/ZAPDConfigs/MqDbg/Config.xml" % (xmlPath, outputPath, outputSourcePath, genSrcFile, incFilePrefix)
 
 	print(execStr)
 	os.system(execStr)
 
 def ExtractFunc(fullPath):
 	outPath = ("assets/" + fullPath.split("assets/xml/")[1]).split(".xml")[0]
+	outSourcePath = ("assets/" + fullPath.split("assets/xml/")[1]).split(".xml")[0]
 
 	if (fullPath.startswith("assets/xml/scenes/")):
-		ExtractScene(fullPath, outPath)
+		ExtractScene(fullPath, outPath, outSourcePath)
 	else:
-		Extract(fullPath, outPath)
+		Extract(fullPath, outPath, outSourcePath)
 
 def main():
     parser = argparse.ArgumentParser(description="baserom asset extractor")
@@ -35,7 +36,7 @@ def main():
     if asset_path is not None:
         if asset_path.endswith("/"):
             asset_path = asset_path[0:-1]
-        Extract(f"assets/xml/{asset_path}.xml", f"assets/{asset_path}/")
+        Extract(f"assets/xml/{asset_path}.xml", f"assets/{asset_path}/", f"assets/{asset_path}/")
     else:
         xmlFiles = []
         for currentPath, folders, files in os.walk("assets"):

tools/ZAPD/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/zeldaret/ZAPD.git
 	branch = master
-	commit = c4773301a17a5a9f9c964a3e7ba6dfbf0730a94b
-	parent = 97f80eeb3f79bafac383fa6dd17765ae291e05d7
+	commit = b9120803e6094a54192ed67d9585ab046101c816
+	parent = 81b43ba32e564a00c976dec7d520e6bcb0b122f8
 	method = merge
 	cmdver = 0.4.3

tools/ZAPD/Makefile

@@ -1,5 +1,16 @@
+OPTIMIZATION_ON ?= 1
+
 CC := g++
-CFLAGS := -g -std=c++17 -I ZAPD -I lib/assimp/include -I lib/elfio -I lib/json/include -I lib/stb -I lib/tinygltf -I lib/libgfxd -I lib/tinyxml2 -O2 -rdynamic
+INC := -I ZAPD -I lib/assimp/include -I lib/elfio -I lib/json/include -I lib/stb -I lib/tinygltf -I lib/libgfxd -I lib/tinyxml2
+
+CFLAGS := -g3 -fpic -Wl,-export-dynamic -std=c++17 -rdynamic -Wall
+ifeq ($(OPTIMIZATION_ON),1)
+  CFLAGS += -O2
+else
+  CFLAGS += -O0
+endif
+
+LDFLAGS := -ldl
 UNAME := $(shell uname)
 
 FS_INC =
@@ -13,22 +24,28 @@ CPP_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.cpp))
 CPP_FILES += lib/tinyxml2/tinyxml2.cpp
 O_FILES   := $(CPP_FILES:.cpp=.o)
 
-all: ZAPD.out
+all: ZAPD.out copycheck
 
 genbuildinfo:
 	python3 ZAPD/genbuildinfo.py
 
+copycheck: ZAPD.out
+	python3 copycheck.py
+
 clean:
 	rm -f $(O_FILES) ZAPD.out
 	$(MAKE) -C lib/libgfxd clean
 
 rebuild: clean all
 
-%.o: %.cpp genbuildinfo
-	$(CC) $(CFLAGS) -c $< -o $@
+%.o: %.cpp
+	$(CC) $(CFLAGS) $(INC) -c $< -o $@ $(LDFLAGS)
+
+ZAPD/Main.o: genbuildinfo ZAPD/Main.cpp
+	$(CC) $(CFLAGS) $(INC) -c ZAPD/Main.cpp -o $@ $(LDFLAGS)
 
 lib/libgfxd/libgfxd.a:
 	$(MAKE) -C lib/libgfxd -j
 
 ZAPD.out: $(O_FILES) lib/libgfxd/libgfxd.a
-	$(CC) $(CFLAGS) $(O_FILES) lib/libgfxd/libgfxd.a -o $@ $(FS_INC)
+	$(CC) $(CFLAGS) $(INC) $(O_FILES) lib/libgfxd/libgfxd.a -o $@ $(FS_INC) $(LDFLAGS)

tools/ZAPD/ZAPD/BitConverter.h

@@ -7,103 +7,103 @@
 class BitConverter
 {
 public:
-	static inline int8_t ToInt8BE(const uint8_t* data, int offset)
+	static inline int8_t ToInt8BE(const uint8_t* data, int32_t offset)
 	{
 		return (uint8_t)data[offset + 0];
 	}
 
-	static inline int8_t ToInt8BE(const std::vector<uint8_t>& data, int offset)
+	static inline int8_t ToInt8BE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		return (uint8_t)data[offset + 0];
 	}
 
-	static inline uint8_t ToUInt8BE(const uint8_t* data, int offset)
+	static inline uint8_t ToUInt8BE(const uint8_t* data, int32_t offset)
 	{
 		return (uint8_t)data[offset + 0];
 	}
 
-	static inline uint8_t ToUInt8BE(const std::vector<uint8_t>& data, int offset)
+	static inline uint8_t ToUInt8BE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		return (uint8_t)data[offset + 0];
 	}
 
-	static inline int16_t ToInt16BE(const uint8_t* data, int offset)
+	static inline int16_t ToInt16BE(const uint8_t* data, int32_t offset)
 	{
 		return ((uint16_t)data[offset + 0] << 8) + (uint16_t)data[offset + 1];
 	}
 
-	static inline int16_t ToInt16BE(const std::vector<uint8_t>& data, int offset)
+	static inline int16_t ToInt16BE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		return ((uint16_t)data[offset + 0] << 8) + (uint16_t)data[offset + 1];
 	}
 
-	static inline uint16_t ToUInt16BE(const uint8_t* data, int offset)
+	static inline uint16_t ToUInt16BE(const uint8_t* data, int32_t offset)
 	{
 		return ((uint16_t)data[offset + 0] << 8) + (uint16_t)data[offset + 1];
 	}
 
-	static inline uint16_t ToUInt16BE(const std::vector<uint8_t>& data, int offset)
+	static inline uint16_t ToUInt16BE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		return ((uint16_t)data[offset + 0] << 8) + (uint16_t)data[offset + 1];
 	}
 
-	static inline int32_t ToInt32BE(const uint8_t* data, int offset)
+	static inline int32_t ToInt32BE(const uint8_t* data, int32_t offset)
 	{
 		return ((uint32_t)data[offset + 0] << 24) + ((uint32_t)data[offset + 1] << 16) +
 			   ((uint32_t)data[offset + 2] << 8) + (uint32_t)data[offset + 3];
 	}
 
-	static inline int32_t ToInt32BE(const std::vector<uint8_t>& data, int offset)
+	static inline int32_t ToInt32BE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		return ((uint32_t)data[offset + 0] << 24) + ((uint32_t)data[offset + 1] << 16) +
 			   ((uint32_t)data[offset + 2] << 8) + (uint32_t)data[offset + 3];
 	}
 
-	static inline uint32_t ToUInt32BE(const uint8_t* data, int offset)
+	static inline uint32_t ToUInt32BE(const uint8_t* data, int32_t offset)
 	{
 		return ((uint32_t)data[offset + 0] << 24) + ((uint32_t)data[offset + 1] << 16) +
 			   ((uint32_t)data[offset + 2] << 8) + (uint32_t)data[offset + 3];
 	}
 
-	static inline uint32_t ToUInt32BE(const std::vector<uint8_t>& data, int offset)
+	static inline uint32_t ToUInt32BE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		return ((uint32_t)data[offset + 0] << 24) + ((uint32_t)data[offset + 1] << 16) +
 			   ((uint32_t)data[offset + 2] << 8) + (uint32_t)data[offset + 3];
 	}
 
-	static inline int64_t ToInt64BE(const uint8_t* data, int offset)
+	static inline int64_t ToInt64BE(const uint8_t* data, int32_t offset)
 	{
 		return ((uint64_t)data[offset + 0] << 56) + ((uint64_t)data[offset + 1] << 48) +
 			   ((uint64_t)data[offset + 2] << 40) + ((uint64_t)data[offset + 3] << 32) +
 			   ((uint64_t)data[offset + 4] << 24) + ((uint64_t)data[offset + 5] << 16) +
 			   ((uint64_t)data[offset + 6] << 8) + ((uint64_t)data[offset + 7]);
 	}
 
-	static inline int64_t ToInt64BE(const std::vector<uint8_t>& data, int offset)
+	static inline int64_t ToInt64BE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		return ((uint64_t)data[offset + 0] << 56) + ((uint64_t)data[offset + 1] << 48) +
 			   ((uint64_t)data[offset + 2] << 40) + ((uint64_t)data[offset + 3] << 32) +
 			   ((uint64_t)data[offset + 4] << 24) + ((uint64_t)data[offset + 5] << 16) +
 			   ((uint64_t)data[offset + 6] << 8) + ((uint64_t)data[offset + 7]);
 	}
 
-	static inline uint64_t ToUInt64BE(const uint8_t* data, int offset)
+	static inline uint64_t ToUInt64BE(const uint8_t* data, int32_t offset)
 	{
 		return ((uint64_t)data[offset + 0] << 56) + ((uint64_t)data[offset + 1] << 48) +
 			   ((uint64_t)data[offset + 2] << 40) + ((uint64_t)data[offset + 3] << 32) +
 			   ((uint64_t)data[offset + 4] << 24) + ((uint64_t)data[offset + 5] << 16) +
 			   ((uint64_t)data[offset + 6] << 8) + ((uint64_t)data[offset + 7]);
 	}
 
-	static inline uint64_t ToUInt64BE(const std::vector<uint8_t>& data, int offset)
+	static inline uint64_t ToUInt64BE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		return ((uint64_t)data[offset + 0] << 56) + ((uint64_t)data[offset + 1] << 48) +
 			   ((uint64_t)data[offset + 2] << 40) + ((uint64_t)data[offset + 3] << 32) +
 			   ((uint64_t)data[offset + 4] << 24) + ((uint64_t)data[offset + 5] << 16) +
 			   ((uint64_t)data[offset + 6] << 8) + ((uint64_t)data[offset + 7]);
 	}
 
-	static inline float ToFloatBE(const uint8_t* data, int offset)
+	static inline float ToFloatBE(const uint8_t* data, int32_t offset)
 	{
 		float value;
 		uint32_t floatData = ((uint32_t)data[offset + 0] << 24) +
@@ -114,7 +114,7 @@ class BitConverter
 		return value;
 	}
 
-	static inline float ToFloatBE(const std::vector<uint8_t>& data, int offset)
+	static inline float ToFloatBE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		float value;
 		uint32_t floatData = ((uint32_t)data[offset + 0] << 24) +
@@ -125,7 +125,7 @@ class BitConverter
 		return value;
 	}
 
-	static inline double ToDoubleBE(const uint8_t* data, int offset)
+	static inline double ToDoubleBE(const uint8_t* data, int32_t offset)
 	{
 		double value;
 		uint64_t floatData =
@@ -141,7 +141,7 @@ class BitConverter
 		return value;
 	}
 
-	static inline double ToDoubleBE(const std::vector<uint8_t>& data, int offset)
+	static inline double ToDoubleBE(const std::vector<uint8_t>& data, int32_t offset)
 	{
 		double value;
 		uint64_t floatData =

tools/ZAPD/ZAPD/CRC32.h

@@ -0,0 +1,23 @@
+#pragma once
+
+static uint32_t CRC32B(unsigned char* message, int32_t size)
+{
+	int32_t byte, crc;
+	int32_t mask;
+
+	crc = 0xFFFFFFFF;
+
+	for (int32_t i = 0; i < size; i++)
+	{
+		byte = message[i];
+		crc = crc ^ byte;
+
+		for (int32_t j = 7; j >= 0; j--)
+		{
+			mask = -(crc & 1);
+			crc = (crc >> 1) ^ (0xEDB88320 & mask);
+		}
+	}
+
+	return ~(uint32_t)(crc);
+}

tools/ZAPD/ZAPD/File.h

@@ -20,7 +20,7 @@ class File
 	static std::vector<uint8_t> ReadAllBytes(const std::string& filePath)
 	{
 		std::ifstream file(filePath, std::ios::in | std::ios::binary | std::ios::ate);
-		int fileSize = (int)file.tellg();
+		int32_t fileSize = (int32_t)file.tellg();
 		file.seekg(0);
 		char* data = new char[fileSize];
 		file.read(data, fileSize);
@@ -33,7 +33,7 @@ class File
 	static std::string ReadAllText(const std::string& filePath)
 	{
 		std::ifstream file(filePath, std::ios::in | std::ios::binary | std::ios::ate);
-		int fileSize = (int)file.tellg();
+		int32_t fileSize = (int32_t)file.tellg();
 		file.seekg(0);
 		char* data = new char[fileSize + 1];
 		memset(data, 0, fileSize + 1);