From ffe86baf05116a30a37d92d23196ae01283bad84 Mon Sep 17 00:00:00 2001 From: Malkierian Date: Fri, 13 Dec 2024 01:00:59 -0700 Subject: [PATCH 1/8] Adds version of `ListFiles`, `ListFilesWithExclude`, allowing for multiple include and exclude filters in one pass. Also adds associated Load, LoadAsync, and Unload directory functions. --- src/resource/ResourceManager.cpp | 41 +++++++++++++++++++++++++ src/resource/ResourceManager.h | 9 ++++++ src/resource/archive/ArchiveManager.cpp | 33 ++++++++++++++++++++ src/resource/archive/ArchiveManager.h | 2 ++ 4 files changed, 85 insertions(+) diff --git a/src/resource/ResourceManager.cpp b/src/resource/ResourceManager.cpp index 9326a0de6..52e669980 100644 --- a/src/resource/ResourceManager.cpp +++ b/src/resource/ResourceManager.cpp @@ -284,6 +284,23 @@ ResourceManager::GetCachedResource(std::variant>>> +ResourceManager::LoadDirectoryAsyncWithExclude(const std::vector& includeMasks, + const std::vector& excludeMasks, uintptr_t owner, + BS::priority_t priority) { + auto loadedList = std::make_shared>>>(); + auto fileList = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks); + loadedList->reserve(fileList->size()); + + for (size_t i = 0; i < fileList->size(); i++) { + auto fileName = std::string(fileList->operator[](i)); + auto future = LoadResourceAsync(fileName, owner, false, priority); + loadedList->push_back(future); + } + + return loadedList; +} + std::shared_ptr>>> ResourceManager::LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority) { auto loadedList = std::make_shared>>>(); @@ -299,6 +316,21 @@ ResourceManager::LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::pr return loadedList; } +std::shared_ptr>> +ResourceManager::LoadDirectoryWithExclude(const std::vector& includeMasks, + const std::vector& excludeMasks, uintptr_t owner) { + auto futureList = LoadDirectoryAsyncWithExclude(includeMasks, excludeMasks, owner, true); + auto loadedList = std::make_shared>>(); + + for (size_t i = 0; i < futureList->size(); i++) { + const auto future = futureList->at(i); + const auto resource = future.get(); + loadedList->push_back(resource); + } + + return loadedList; +} + std::shared_ptr>>> ResourceManager::LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority) { return LoadDirectoryAsync({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }, priority); @@ -336,6 +368,15 @@ void ResourceManager::DirtyDirectory(const ResourceIdentifier& identifier) { } } +void ResourceManager::UnloadDirectoryWithExclude(const std::vector& includeMasks, + const std::vector& excludeMasks, uintptr_t owner) { + auto list = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks); + + for (const auto& key : *list.get()) { + UnloadResource(key, owner); + } +} + void ResourceManager::DirtyDirectory(const std::string& searchMask) { DirtyDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }); } diff --git a/src/resource/ResourceManager.h b/src/resource/ResourceManager.h index 003c153a0..13727f548 100644 --- a/src/resource/ResourceManager.h +++ b/src/resource/ResourceManager.h @@ -62,6 +62,9 @@ class ResourceManager { std::shared_future> LoadResourceAsync(const ResourceIdentifier& identifier, bool loadExact = false, BS::priority_t priority = BS::pr::normal, std::shared_ptr initData = nullptr); + std::shared_ptr>> + LoadDirectoryWithExclude(const std::vector& includeMasks, const std::vector& excludeMasks, + uintptr_t owner); std::shared_ptr>> LoadDirectory(const ResourceIdentifier& identifier); std::shared_ptr>>> LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority = BS::pr::normal); @@ -80,7 +83,13 @@ class ResourceManager { std::shared_ptr>> LoadDirectory(const std::string& searchMask); std::shared_ptr>>> LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal); + std::shared_ptr>>> + LoadDirectoryAsyncWithExclude(const std::vector& includeMasks, + const std::vector& excludeMasks, uintptr_t owner = 0, + BS::priority_t priority = BS::pr::normal); void DirtyDirectory(const std::string& searchMask); + void UnloadDirectoryWithExclude(const std::vector& includeMasks, + const std::vector& excludeMasks, uintptr_t owner); void UnloadDirectory(const std::string& searchMask); bool OtrSignatureCheck(const char* fileName); diff --git a/src/resource/archive/ArchiveManager.cpp b/src/resource/archive/ArchiveManager.cpp index 5f15fe761..7882f98b9 100644 --- a/src/resource/archive/ArchiveManager.cpp +++ b/src/resource/archive/ArchiveManager.cpp @@ -64,6 +64,39 @@ bool ArchiveManager::HasFile(uint64_t hash) { return mFileToArchive.count(hash) > 0; } +std::shared_ptr> +ArchiveManager::ListFilesWithExclude(const std::vector& include, const std::vector& exclude) { + auto list = std::make_shared>(); + for (const auto& [hash, path] : mHashes) { + if (include.empty() && exclude.empty()) { + list->push_back(path); + continue; + } + bool includeMatch = include.empty(); + if (!include.empty()) { + for (std::string filter : include) { + if (glob_match(filter.c_str(), path.c_str())) { + includeMatch = true; + break; + } + } + } + bool excludeMatch = false; + if (!include.empty()) { + for (std::string filter : exclude) { + if (glob_match(filter.c_str(), path.c_str())) { + excludeMatch = true; + break; + } + } + } + if (includeMatch && !excludeMatch) { + list->push_back(path); + } + } + return list; +} + std::shared_ptr> ArchiveManager::ListFiles(const std::string& filter) { auto list = std::make_shared>(); for (const auto& [hash, path] : mHashes) { diff --git a/src/resource/archive/ArchiveManager.h b/src/resource/archive/ArchiveManager.h index 42e9c956a..62b5607ff 100644 --- a/src/resource/archive/ArchiveManager.h +++ b/src/resource/archive/ArchiveManager.h @@ -31,6 +31,8 @@ class ArchiveManager { std::shared_ptr LoadFile(uint64_t hash, std::shared_ptr initData = nullptr); bool HasFile(const std::string& filePath); bool HasFile(uint64_t hash); + std::shared_ptr> ListFilesWithExclude(const std::vector& include = {}, + const std::vector& exclude = {}); std::shared_ptr> ListFiles(const std::string& filter = ""); std::vector GetGameVersions(); const std::string* HashToString(uint64_t hash) const; From b81ea71787bce0c7f0a0f9d77213587c683daa3b Mon Sep 17 00:00:00 2001 From: Malkierian Date: Fri, 13 Dec 2024 11:13:15 -0700 Subject: [PATCH 2/8] Update for latest LUS main. --- src/resource/ResourceManager.cpp | 10 +++++----- src/resource/ResourceManager.h | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/resource/ResourceManager.cpp b/src/resource/ResourceManager.cpp index 52e669980..34df9329d 100644 --- a/src/resource/ResourceManager.cpp +++ b/src/resource/ResourceManager.cpp @@ -286,7 +286,7 @@ ResourceManager::GetCachedResource(std::variant>>> ResourceManager::LoadDirectoryAsyncWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks, uintptr_t owner, + const std::vector& excludeMasks, BS::priority_t priority) { auto loadedList = std::make_shared>>>(); auto fileList = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks); @@ -294,7 +294,7 @@ ResourceManager::LoadDirectoryAsyncWithExclude(const std::vector& i for (size_t i = 0; i < fileList->size(); i++) { auto fileName = std::string(fileList->operator[](i)); - auto future = LoadResourceAsync(fileName, owner, false, priority); + auto future = LoadResourceAsync(fileName, priority); loadedList->push_back(future); } @@ -319,7 +319,7 @@ ResourceManager::LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::pr std::shared_ptr>> ResourceManager::LoadDirectoryWithExclude(const std::vector& includeMasks, const std::vector& excludeMasks, uintptr_t owner) { - auto futureList = LoadDirectoryAsyncWithExclude(includeMasks, excludeMasks, owner, true); + auto futureList = LoadDirectoryAsyncWithExclude(includeMasks, excludeMasks); auto loadedList = std::make_shared>>(); for (size_t i = 0; i < futureList->size(); i++) { @@ -369,11 +369,11 @@ void ResourceManager::DirtyDirectory(const ResourceIdentifier& identifier) { } void ResourceManager::UnloadDirectoryWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks, uintptr_t owner) { + const std::vector& excludeMasks) { auto list = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks); for (const auto& key : *list.get()) { - UnloadResource(key, owner); + UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive }); } } diff --git a/src/resource/ResourceManager.h b/src/resource/ResourceManager.h index 13727f548..65157c030 100644 --- a/src/resource/ResourceManager.h +++ b/src/resource/ResourceManager.h @@ -85,11 +85,11 @@ class ResourceManager { LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal); std::shared_ptr>>> LoadDirectoryAsyncWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks, uintptr_t owner = 0, + const std::vector& excludeMasks, BS::priority_t priority = BS::pr::normal); void DirtyDirectory(const std::string& searchMask); void UnloadDirectoryWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks, uintptr_t owner); + const std::vector& excludeMasks); void UnloadDirectory(const std::string& searchMask); bool OtrSignatureCheck(const char* fileName); From 90153c16ead2e703b998ce7fb958aa3caaa381e4 Mon Sep 17 00:00:00 2001 From: Malkierian Date: Fri, 13 Dec 2024 11:20:07 -0700 Subject: [PATCH 3/8] Clang --- src/resource/ResourceManager.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/resource/ResourceManager.cpp b/src/resource/ResourceManager.cpp index 34df9329d..92d191e2c 100644 --- a/src/resource/ResourceManager.cpp +++ b/src/resource/ResourceManager.cpp @@ -286,8 +286,7 @@ ResourceManager::GetCachedResource(std::variant>>> ResourceManager::LoadDirectoryAsyncWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks, - BS::priority_t priority) { + const std::vector& excludeMasks, BS::priority_t priority) { auto loadedList = std::make_shared>>>(); auto fileList = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks); loadedList->reserve(fileList->size()); From cbb41c4d9a95b19ca34f35c4d70b2cf443d99ba7 Mon Sep 17 00:00:00 2001 From: Malkierian Date: Tue, 17 Dec 2024 01:18:54 -0700 Subject: [PATCH 4/8] Add `ResourceFilter`, used to handle wildcards in paths. Change `LoadDirectory`, `UnloadDirectory`, and `DirtyDirectory` to `LoadResources`, etc. Move all Directory processing to the thread pool, encapsulating `ListFiles` as well. Unify `ListFiles` flows to eventually pass `ResourceFilter` to the base function. --- src/public/bridge/resourcebridge.cpp | 8 +- src/resource/ResourceManager.cpp | 131 +++++++++--------------- src/resource/ResourceManager.h | 65 ++++++------ src/resource/archive/ArchiveManager.cpp | 35 ++++--- src/resource/archive/ArchiveManager.h | 7 +- 5 files changed, 111 insertions(+), 135 deletions(-) diff --git a/src/public/bridge/resourcebridge.cpp b/src/public/bridge/resourcebridge.cpp index 38411cea0..da28c0b1c 100644 --- a/src/public/bridge/resourcebridge.cpp +++ b/src/public/bridge/resourcebridge.cpp @@ -153,7 +153,7 @@ void ResourceGetGameVersions(uint32_t* versions, size_t versionsSize, size_t* ve } void ResourceLoadDirectoryAsync(const char* name) { - Ship::Context::GetInstance()->GetResourceManager()->LoadDirectoryAsync(name); + Ship::Context::GetInstance()->GetResourceManager()->LoadResourcesAsync(name); } uint32_t ResourceHasGameVersion(uint32_t hash) { @@ -162,11 +162,11 @@ uint32_t ResourceHasGameVersion(uint32_t hash) { } void ResourceLoadDirectory(const char* name) { - Ship::Context::GetInstance()->GetResourceManager()->LoadDirectory(name); + Ship::Context::GetInstance()->GetResourceManager()->LoadResources(name); } void ResourceDirtyDirectory(const char* name) { - Ship::Context::GetInstance()->GetResourceManager()->DirtyDirectory(name); + Ship::Context::GetInstance()->GetResourceManager()->DirtyResources(name); } void ResourceDirtyByName(const char* name) { @@ -194,7 +194,7 @@ void ResourceUnloadByCrc(uint64_t crc) { } void ResourceUnloadDirectory(const char* name) { - Ship::Context::GetInstance()->GetResourceManager()->UnloadDirectory(name); + Ship::Context::GetInstance()->GetResourceManager()->UnloadResources(name); } uint32_t IsResourceManagerLoaded() { diff --git a/src/resource/ResourceManager.cpp b/src/resource/ResourceManager.cpp index 92d191e2c..d2d77a5ab 100644 --- a/src/resource/ResourceManager.cpp +++ b/src/resource/ResourceManager.cpp @@ -11,6 +11,10 @@ namespace Ship { +ResourceFilter::ResourceFilter(const std::list includeMasks, const std::list excludeMasks, + const uintptr_t owner, const std::shared_ptr parent) + : IncludeMasks(includeMasks), ExcludeMasks(excludeMasks), Owner(owner), Parent(parent) {} + size_t ResourceIdentifier::GetHash() const { return mHash; } @@ -284,112 +288,77 @@ ResourceManager::GetCachedResource(std::variant>>> -ResourceManager::LoadDirectoryAsyncWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks, BS::priority_t priority) { - auto loadedList = std::make_shared>>>(); - auto fileList = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks); - loadedList->reserve(fileList->size()); - - for (size_t i = 0; i < fileList->size(); i++) { - auto fileName = std::string(fileList->operator[](i)); - auto future = LoadResourceAsync(fileName, priority); - loadedList->push_back(future); - } - - return loadedList; -} - -std::shared_ptr>>> -ResourceManager::LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority) { - auto loadedList = std::make_shared>>>(); - auto fileList = GetArchiveManager()->ListFiles(identifier.Path); +std::shared_ptr>> +ResourceManager::LoadResourcesProcess(const ResourceFilter& filter) { + auto loadedList = std::make_shared>>(); + auto fileList = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); loadedList->reserve(fileList->size()); for (size_t i = 0; i < fileList->size(); i++) { auto fileName = std::string(fileList->operator[](i)); - auto future = LoadResourceAsync({ fileName, identifier.Owner, identifier.Parent }, false, priority); - loadedList->push_back(future); - } - - return loadedList; -} - -std::shared_ptr>> -ResourceManager::LoadDirectoryWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks, uintptr_t owner) { - auto futureList = LoadDirectoryAsyncWithExclude(includeMasks, excludeMasks); - auto loadedList = std::make_shared>>(); - - for (size_t i = 0; i < futureList->size(); i++) { - const auto future = futureList->at(i); - const auto resource = future.get(); + auto resource = LoadResource({fileName, filter.Owner, filter.Parent}); loadedList->push_back(resource); } return loadedList; } -std::shared_ptr>>> -ResourceManager::LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority) { - return LoadDirectoryAsync({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }, priority); +std::shared_future>>> +ResourceManager::LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t priority) { + return mThreadPool->submit_task( + [this, filter]() -> std::shared_ptr>> { + return LoadResourcesProcess(filter); + }, + priority); } -std::shared_ptr>> -ResourceManager::LoadDirectory(const ResourceIdentifier& identifier) { - auto futureList = LoadDirectoryAsync(identifier, true); - auto loadedList = std::make_shared>>(); - - for (size_t i = 0; i < futureList->size(); i++) { - const auto future = futureList->at(i); - const auto resource = future.get(); - loadedList->push_back(resource); - } - - return loadedList; +std::shared_future>>> +ResourceManager::LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority) { + return LoadResourcesAsync({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}, priority); } -std::shared_ptr>> ResourceManager::LoadDirectory(const std::string& searchMask) { - return LoadDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }); +std::shared_ptr>> ResourceManager::LoadResources(const std::string& searchMask) { + return LoadResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}); } -void ResourceManager::DirtyDirectory(const ResourceIdentifier& identifier) { - auto list = GetArchiveManager()->ListFiles(identifier.Path); - - for (const auto& key : *list.get()) { - auto resource = GetCachedResource({ key, identifier.Owner, identifier.Parent }); - // If it's a resource, we will set the dirty flag, else we will just unload it. - if (resource != nullptr) { - resource->Dirty(); - } else { - UnloadResource(identifier); - } - } +std::shared_ptr>> ResourceManager::LoadResources(const ResourceFilter& filter) { + return LoadResourcesAsync(filter, BS::pr::highest).get(); } -void ResourceManager::UnloadDirectoryWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks) { - auto list = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks); +void ResourceManager::DirtyResources(const ResourceFilter& filter) { + mThreadPool->submit_task( + [this, filter]() -> void { + auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); - for (const auto& key : *list.get()) { - UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive }); - } + for (const auto& key : *list.get()) { + auto resource = GetCachedResource({ key, filter.Owner, filter.Parent }); + // If it's a resource, we will set the dirty flag, else we will just unload it. + if (resource != nullptr) { + resource->Dirty(); + } else { + UnloadResource({ key, filter.Owner, filter.Parent }); + } + } + }); } -void ResourceManager::DirtyDirectory(const std::string& searchMask) { - DirtyDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }); -} +void ResourceManager::UnloadResources(const ResourceFilter& filter) { + mThreadPool->submit_task( + [this, filter]() -> void { + auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); -void ResourceManager::UnloadDirectory(const ResourceIdentifier& identifier) { - auto list = GetArchiveManager()->ListFiles(identifier.Path); + for (const auto& key : *list.get()) { + UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive }); + } + }); +} - for (const auto& key : *list.get()) { - UnloadResource({ key, identifier.Owner, identifier.Parent }); - } +void ResourceManager::DirtyResources(const std::string& searchMask) { + DirtyResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}); } -void ResourceManager::UnloadDirectory(const std::string& searchMask) { - UnloadDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }); +void ResourceManager::UnloadResources(const std::string& searchMask) { + UnloadResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}); } std::shared_ptr ResourceManager::GetArchiveManager() { diff --git a/src/resource/ResourceManager.h b/src/resource/ResourceManager.h index 65157c030..825e27a00 100644 --- a/src/resource/ResourceManager.h +++ b/src/resource/ResourceManager.h @@ -18,13 +18,23 @@ namespace Ship { struct File; +struct ResourceFilter { + ResourceFilter(const std::list includeMasks, const std::list excludeMasks, + const uintptr_t owner, const std::shared_ptr parent); + + const std::list IncludeMasks; + const std::list ExcludeMasks; + const uintptr_t Owner = 0; + const std::shared_ptr Parent = nullptr; +}; + struct ResourceIdentifier { - friend class ResourceIdentifierHash; + friend struct ResourceIdentifierHash; ResourceIdentifier(const std::string& path, const uintptr_t owner, const std::shared_ptr parent); bool operator==(const ResourceIdentifier& rhs) const; - // Path can either be a Path or a Search Mask including globs depending on usage. + // Must be an exact path. Passing a path with a wildcard will return a fail state const std::string Path = ""; const uintptr_t Owner = 0; const std::shared_ptr Parent = nullptr; @@ -53,50 +63,45 @@ class ResourceManager { std::shared_ptr GetArchiveManager(); std::shared_ptr GetResourceLoader(); + std::shared_ptr GetCachedResource(const std::string& filePath, bool loadExact = false); std::shared_ptr GetCachedResource(const ResourceIdentifier& identifier, bool loadExact = false); + std::shared_ptr LoadResource(const std::string& filePath, bool loadExact = false, + std::shared_ptr initData = nullptr); std::shared_ptr LoadResource(const ResourceIdentifier& identifier, bool loadExact = false, std::shared_ptr initData = nullptr); + std::shared_ptr LoadResourceProcess(const std::string& filePath, bool loadExact = false, + std::shared_ptr initData = nullptr); std::shared_ptr LoadResourceProcess(const ResourceIdentifier& identifier, bool loadExact = false, std::shared_ptr initData = nullptr); - size_t UnloadResource(const ResourceIdentifier& identifier); + std::shared_future> + LoadResourceAsync(const std::string& filePath, bool loadExact = false, BS::priority_t priority = BS::pr::normal, + std::shared_ptr initData = nullptr); std::shared_future> LoadResourceAsync(const ResourceIdentifier& identifier, bool loadExact = false, BS::priority_t priority = BS::pr::normal, std::shared_ptr initData = nullptr); + size_t UnloadResource(const ResourceIdentifier& identifier); + size_t UnloadResource(const std::string& filePath); + + std::shared_ptr>> LoadResources(const std::string& searchMask); std::shared_ptr>> - LoadDirectoryWithExclude(const std::vector& includeMasks, const std::vector& excludeMasks, - uintptr_t owner); - std::shared_ptr>> LoadDirectory(const ResourceIdentifier& identifier); - std::shared_ptr>>> - LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority = BS::pr::normal); - void DirtyDirectory(const ResourceIdentifier& identifier); - void UnloadDirectory(const ResourceIdentifier& identifier); + LoadResources(const ResourceFilter& filter); + std::shared_future>>> + LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal); + std::shared_future>>> + LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t priority = BS::pr::normal); - std::shared_ptr GetCachedResource(const std::string& filePath, bool loadExact = false); - std::shared_ptr LoadResource(const std::string& filePath, bool loadExact = false, - std::shared_ptr initData = nullptr); - std::shared_ptr LoadResourceProcess(const std::string& filePath, bool loadExact = false, - std::shared_ptr initData = nullptr); - size_t UnloadResource(const std::string& filePath); - std::shared_future> - LoadResourceAsync(const std::string& filePath, bool loadExact = false, BS::priority_t priority = BS::pr::normal, - std::shared_ptr initData = nullptr); - std::shared_ptr>> LoadDirectory(const std::string& searchMask); - std::shared_ptr>>> - LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal); - std::shared_ptr>>> - LoadDirectoryAsyncWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks, - BS::priority_t priority = BS::pr::normal); - void DirtyDirectory(const std::string& searchMask); - void UnloadDirectoryWithExclude(const std::vector& includeMasks, - const std::vector& excludeMasks); - void UnloadDirectory(const std::string& searchMask); + void DirtyResources(const std::string& searchMask); + void DirtyResources(const ResourceFilter& filter); + void UnloadResources(const std::string& searchMask); + void UnloadResources(const ResourceFilter& filter); bool OtrSignatureCheck(const char* fileName); bool IsAltAssetsEnabled(); void SetAltAssetsEnabled(bool isEnabled); protected: + std::shared_ptr>> + LoadResourcesProcess(const ResourceFilter& filter); std::variant> CheckCache(const ResourceIdentifier& identifier, bool loadExact = false); std::shared_ptr LoadFileProcess(const ResourceIdentifier& identifier, diff --git a/src/resource/archive/ArchiveManager.cpp b/src/resource/archive/ArchiveManager.cpp index 7882f98b9..a67d12b8f 100644 --- a/src/resource/archive/ArchiveManager.cpp +++ b/src/resource/archive/ArchiveManager.cpp @@ -65,16 +65,27 @@ bool ArchiveManager::HasFile(uint64_t hash) { } std::shared_ptr> -ArchiveManager::ListFilesWithExclude(const std::vector& include, const std::vector& exclude) { +ArchiveManager::ListFiles() { + return ListFiles({ "*" }, {}); +} + +std::shared_ptr> +ArchiveManager::ListFiles(const std::string& searchMask) { + return ListFiles({ searchMask }, {}); +} + +std::shared_ptr> +ArchiveManager::ListFiles(const std::list& includes, + const std::list& excludes) { auto list = std::make_shared>(); for (const auto& [hash, path] : mHashes) { - if (include.empty() && exclude.empty()) { + if (includes.empty() && excludes.empty()) { list->push_back(path); continue; } - bool includeMatch = include.empty(); - if (!include.empty()) { - for (std::string filter : include) { + bool includeMatch = includes.empty(); + if (!includes.empty()) { + for (std::string filter : includes) { if (glob_match(filter.c_str(), path.c_str())) { includeMatch = true; break; @@ -82,8 +93,8 @@ ArchiveManager::ListFilesWithExclude(const std::vector& include, co } } bool excludeMatch = false; - if (!include.empty()) { - for (std::string filter : exclude) { + if (!excludes.empty()) { + for (std::string filter : excludes) { if (glob_match(filter.c_str(), path.c_str())) { excludeMatch = true; break; @@ -97,16 +108,6 @@ ArchiveManager::ListFilesWithExclude(const std::vector& include, co return list; } -std::shared_ptr> ArchiveManager::ListFiles(const std::string& filter) { - auto list = std::make_shared>(); - for (const auto& [hash, path] : mHashes) { - if (filter.empty() || glob_match(filter.c_str(), path.c_str())) { - list->push_back(path); - } - } - return list; -} - std::vector ArchiveManager::GetGameVersions() { return mGameVersions; } diff --git a/src/resource/archive/ArchiveManager.h b/src/resource/archive/ArchiveManager.h index 62b5607ff..f4053874d 100644 --- a/src/resource/archive/ArchiveManager.h +++ b/src/resource/archive/ArchiveManager.h @@ -31,9 +31,10 @@ class ArchiveManager { std::shared_ptr LoadFile(uint64_t hash, std::shared_ptr initData = nullptr); bool HasFile(const std::string& filePath); bool HasFile(uint64_t hash); - std::shared_ptr> ListFilesWithExclude(const std::vector& include = {}, - const std::vector& exclude = {}); - std::shared_ptr> ListFiles(const std::string& filter = ""); + std::shared_ptr> ListFiles(); + std::shared_ptr> ListFiles(const std::string& searchMask); + std::shared_ptr> ListFiles(const std::list& includes, + const std::list& excludes); std::vector GetGameVersions(); const std::string* HashToString(uint64_t hash) const; bool IsGameVersionValid(uint32_t gameVersion); From 81c7a829996c870c25e588438ab84b40c59c487c Mon Sep 17 00:00:00 2001 From: Malkierian Date: Tue, 17 Dec 2024 01:20:15 -0700 Subject: [PATCH 5/8] clang --- src/resource/ResourceManager.cpp | 51 ++++++++++++------------- src/resource/ResourceManager.h | 8 ++-- src/resource/archive/ArchiveManager.cpp | 11 ++---- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/src/resource/ResourceManager.cpp b/src/resource/ResourceManager.cpp index d2d77a5ab..bf48f1f0e 100644 --- a/src/resource/ResourceManager.cpp +++ b/src/resource/ResourceManager.cpp @@ -13,7 +13,8 @@ namespace Ship { ResourceFilter::ResourceFilter(const std::list includeMasks, const std::list excludeMasks, const uintptr_t owner, const std::shared_ptr parent) - : IncludeMasks(includeMasks), ExcludeMasks(excludeMasks), Owner(owner), Parent(parent) {} + : IncludeMasks(includeMasks), ExcludeMasks(excludeMasks), Owner(owner), Parent(parent) { +} size_t ResourceIdentifier::GetHash() const { return mHash; @@ -296,7 +297,7 @@ ResourceManager::LoadResourcesProcess(const ResourceFilter& filter) { for (size_t i = 0; i < fileList->size(); i++) { auto fileName = std::string(fileList->operator[](i)); - auto resource = LoadResource({fileName, filter.Owner, filter.Parent}); + auto resource = LoadResource({ fileName, filter.Owner, filter.Parent }); loadedList->push_back(resource); } @@ -314,11 +315,11 @@ ResourceManager::LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t std::shared_future>>> ResourceManager::LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority) { - return LoadResourcesAsync({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}, priority); + return LoadResourcesAsync({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }, priority); } std::shared_ptr>> ResourceManager::LoadResources(const std::string& searchMask) { - return LoadResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}); + return LoadResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }); } std::shared_ptr>> ResourceManager::LoadResources(const ResourceFilter& filter) { @@ -326,39 +327,37 @@ std::shared_ptr>> ResourceManager::LoadRe } void ResourceManager::DirtyResources(const ResourceFilter& filter) { - mThreadPool->submit_task( - [this, filter]() -> void { - auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); - - for (const auto& key : *list.get()) { - auto resource = GetCachedResource({ key, filter.Owner, filter.Parent }); - // If it's a resource, we will set the dirty flag, else we will just unload it. - if (resource != nullptr) { - resource->Dirty(); - } else { - UnloadResource({ key, filter.Owner, filter.Parent }); - } + mThreadPool->submit_task([this, filter]() -> void { + auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); + + for (const auto& key : *list.get()) { + auto resource = GetCachedResource({ key, filter.Owner, filter.Parent }); + // If it's a resource, we will set the dirty flag, else we will just unload it. + if (resource != nullptr) { + resource->Dirty(); + } else { + UnloadResource({ key, filter.Owner, filter.Parent }); } - }); + } + }); } void ResourceManager::UnloadResources(const ResourceFilter& filter) { - mThreadPool->submit_task( - [this, filter]() -> void { - auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); + mThreadPool->submit_task([this, filter]() -> void { + auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); - for (const auto& key : *list.get()) { - UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive }); - } - }); + for (const auto& key : *list.get()) { + UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive }); + } + }); } void ResourceManager::DirtyResources(const std::string& searchMask) { - DirtyResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}); + DirtyResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }); } void ResourceManager::UnloadResources(const std::string& searchMask) { - UnloadResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}); + UnloadResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }); } std::shared_ptr ResourceManager::GetArchiveManager() { diff --git a/src/resource/ResourceManager.h b/src/resource/ResourceManager.h index 825e27a00..5e578e20a 100644 --- a/src/resource/ResourceManager.h +++ b/src/resource/ResourceManager.h @@ -20,7 +20,7 @@ struct File; struct ResourceFilter { ResourceFilter(const std::list includeMasks, const std::list excludeMasks, - const uintptr_t owner, const std::shared_ptr parent); + const uintptr_t owner, const std::shared_ptr parent); const std::list IncludeMasks; const std::list ExcludeMasks; @@ -83,8 +83,7 @@ class ResourceManager { size_t UnloadResource(const std::string& filePath); std::shared_ptr>> LoadResources(const std::string& searchMask); - std::shared_ptr>> - LoadResources(const ResourceFilter& filter); + std::shared_ptr>> LoadResources(const ResourceFilter& filter); std::shared_future>>> LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal); std::shared_future>>> @@ -100,8 +99,7 @@ class ResourceManager { void SetAltAssetsEnabled(bool isEnabled); protected: - std::shared_ptr>> - LoadResourcesProcess(const ResourceFilter& filter); + std::shared_ptr>> LoadResourcesProcess(const ResourceFilter& filter); std::variant> CheckCache(const ResourceIdentifier& identifier, bool loadExact = false); std::shared_ptr LoadFileProcess(const ResourceIdentifier& identifier, diff --git a/src/resource/archive/ArchiveManager.cpp b/src/resource/archive/ArchiveManager.cpp index a67d12b8f..fb96e98d3 100644 --- a/src/resource/archive/ArchiveManager.cpp +++ b/src/resource/archive/ArchiveManager.cpp @@ -64,19 +64,16 @@ bool ArchiveManager::HasFile(uint64_t hash) { return mFileToArchive.count(hash) > 0; } -std::shared_ptr> -ArchiveManager::ListFiles() { +std::shared_ptr> ArchiveManager::ListFiles() { return ListFiles({ "*" }, {}); } -std::shared_ptr> -ArchiveManager::ListFiles(const std::string& searchMask) { +std::shared_ptr> ArchiveManager::ListFiles(const std::string& searchMask) { return ListFiles({ searchMask }, {}); } -std::shared_ptr> -ArchiveManager::ListFiles(const std::list& includes, - const std::list& excludes) { +std::shared_ptr> ArchiveManager::ListFiles(const std::list& includes, + const std::list& excludes) { auto list = std::make_shared>(); for (const auto& [hash, path] : mHashes) { if (includes.empty() && excludes.empty()) { From a9998c077298e7a2d7b6c1ac3d772da03ab1b7ff Mon Sep 17 00:00:00 2001 From: Malkierian Date: Tue, 17 Dec 2024 01:29:41 -0700 Subject: [PATCH 6/8] Try to address build job fails. --- src/resource/ResourceManager.h | 2 ++ src/resource/archive/ArchiveManager.h | 1 + 2 files changed, 3 insertions(+) diff --git a/src/resource/ResourceManager.h b/src/resource/ResourceManager.h index 5e578e20a..0ae5a3d1a 100644 --- a/src/resource/ResourceManager.h +++ b/src/resource/ResourceManager.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/resource/archive/ArchiveManager.h b/src/resource/archive/ArchiveManager.h index f4053874d..fb23829f8 100644 --- a/src/resource/archive/ArchiveManager.h +++ b/src/resource/archive/ArchiveManager.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include From 0699337d57d51b3f0bd81cb3f79323c2df159c06 Mon Sep 17 00:00:00 2001 From: Malkierian Date: Tue, 17 Dec 2024 12:00:39 -0700 Subject: [PATCH 7/8] Fix reference passing and internal storage for `ResourceFilter` flows. Change the blank `ListFiles()` function to pass empty lists instead of "*" to allow it to trip the early list insertion and return short circuit based on them being empty. --- src/resource/ResourceManager.cpp | 2 +- src/resource/ResourceManager.h | 2 +- src/resource/archive/ArchiveManager.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/resource/ResourceManager.cpp b/src/resource/ResourceManager.cpp index bf48f1f0e..b1f0b6f19 100644 --- a/src/resource/ResourceManager.cpp +++ b/src/resource/ResourceManager.cpp @@ -11,7 +11,7 @@ namespace Ship { -ResourceFilter::ResourceFilter(const std::list includeMasks, const std::list excludeMasks, +ResourceFilter::ResourceFilter(const std::list& includeMasks, const std::list& excludeMasks, const uintptr_t owner, const std::shared_ptr parent) : IncludeMasks(includeMasks), ExcludeMasks(excludeMasks), Owner(owner), Parent(parent) { } diff --git a/src/resource/ResourceManager.h b/src/resource/ResourceManager.h index 0ae5a3d1a..e3a2e0cce 100644 --- a/src/resource/ResourceManager.h +++ b/src/resource/ResourceManager.h @@ -21,7 +21,7 @@ namespace Ship { struct File; struct ResourceFilter { - ResourceFilter(const std::list includeMasks, const std::list excludeMasks, + ResourceFilter(const std::list& includeMasks, const std::list& excludeMasks, const uintptr_t owner, const std::shared_ptr parent); const std::list IncludeMasks; diff --git a/src/resource/archive/ArchiveManager.cpp b/src/resource/archive/ArchiveManager.cpp index fb96e98d3..89fbc4782 100644 --- a/src/resource/archive/ArchiveManager.cpp +++ b/src/resource/archive/ArchiveManager.cpp @@ -65,7 +65,7 @@ bool ArchiveManager::HasFile(uint64_t hash) { } std::shared_ptr> ArchiveManager::ListFiles() { - return ListFiles({ "*" }, {}); + return ListFiles({}, {}); } std::shared_ptr> ArchiveManager::ListFiles(const std::string& searchMask) { From 640e80688291439e83ec18af0ab16b3cad98f288 Mon Sep 17 00:00:00 2001 From: Malkierian Date: Tue, 17 Dec 2024 15:30:49 -0700 Subject: [PATCH 8/8] Remove blank `ListFiles` in favor of default of `""` searchMask for the other one. Add handling to not add `""` to the list before passing to the base function. --- src/resource/archive/ArchiveManager.cpp | 8 ++++---- src/resource/archive/ArchiveManager.h | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/resource/archive/ArchiveManager.cpp b/src/resource/archive/ArchiveManager.cpp index 89fbc4782..463024e34 100644 --- a/src/resource/archive/ArchiveManager.cpp +++ b/src/resource/archive/ArchiveManager.cpp @@ -64,11 +64,11 @@ bool ArchiveManager::HasFile(uint64_t hash) { return mFileToArchive.count(hash) > 0; } -std::shared_ptr> ArchiveManager::ListFiles() { - return ListFiles({}, {}); -} - std::shared_ptr> ArchiveManager::ListFiles(const std::string& searchMask) { + std::list includes = {}; + if (searchMask.size() > 0) { + includes.push_back(searchMask); + } return ListFiles({ searchMask }, {}); } diff --git a/src/resource/archive/ArchiveManager.h b/src/resource/archive/ArchiveManager.h index fb23829f8..2c87b3cc2 100644 --- a/src/resource/archive/ArchiveManager.h +++ b/src/resource/archive/ArchiveManager.h @@ -32,8 +32,7 @@ class ArchiveManager { std::shared_ptr LoadFile(uint64_t hash, std::shared_ptr initData = nullptr); bool HasFile(const std::string& filePath); bool HasFile(uint64_t hash); - std::shared_ptr> ListFiles(); - std::shared_ptr> ListFiles(const std::string& searchMask); + std::shared_ptr> ListFiles(const std::string& searchMask = ""); std::shared_ptr> ListFiles(const std::list& includes, const std::list& excludes); std::vector GetGameVersions();