From b7e715e60a2773a8b3b40698b4cc10a12bffdcfe Mon Sep 17 00:00:00 2001 From: Malkierian Date: Tue, 17 Dec 2024 16:41:01 -0700 Subject: [PATCH] ResourceFilter, Directory Function Renames, Multifilter Functionality (#752) * Adds version of `ListFiles`, `ListFilesWithExclude`, allowing for multiple include and exclude filters in one pass. Also adds associated Load, LoadAsync, and Unload directory functions. * Update for latest LUS main. * Clang * Add `ResourceFilter`, used to handle wildcards in paths. Change `LoadDirectory`, `UnloadDirectory`, and `DirtyDirectory` to `LoadResources`, etc. Move all Directory processing to the thread pool, encapsulating `ListFiles` as well. Unify `ListFiles` flows to eventually pass `ResourceFilter` to the base function. * clang * Try to address build job fails. * Fix reference passing and internal storage for `ResourceFilter` flows. Change the blank `ListFiles()` function to pass empty lists instead of "*" to allow it to trip the early list insertion and return short circuit based on them being empty. * Remove blank `ListFiles` in favor of default of `""` searchMask for the other one. Add handling to not add `""` to the list before passing to the base function. --- src/public/bridge/resourcebridge.cpp | 8 +-- src/resource/ResourceManager.cpp | 92 ++++++++++++++----------- src/resource/ResourceManager.h | 58 ++++++++++------ src/resource/archive/ArchiveManager.cpp | 35 +++++++++- src/resource/archive/ArchiveManager.h | 5 +- 5 files changed, 127 insertions(+), 71 deletions(-) diff --git a/src/public/bridge/resourcebridge.cpp b/src/public/bridge/resourcebridge.cpp index eb5790dd2..8a37b0584 100644 --- a/src/public/bridge/resourcebridge.cpp +++ b/src/public/bridge/resourcebridge.cpp @@ -153,7 +153,7 @@ void ResourceGetGameVersions(uint32_t* versions, size_t versionsSize, size_t* ve } void ResourceLoadDirectoryAsync(const char* name) { - Ship::Context::GetInstance()->GetResourceManager()->LoadDirectoryAsync(name); + Ship::Context::GetInstance()->GetResourceManager()->LoadResourcesAsync(name); } uint32_t ResourceHasGameVersion(uint32_t hash) { @@ -162,11 +162,11 @@ uint32_t ResourceHasGameVersion(uint32_t hash) { } void ResourceLoadDirectory(const char* name) { - Ship::Context::GetInstance()->GetResourceManager()->LoadDirectory(name); + Ship::Context::GetInstance()->GetResourceManager()->LoadResources(name); } void ResourceDirtyDirectory(const char* name) { - Ship::Context::GetInstance()->GetResourceManager()->DirtyDirectory(name); + Ship::Context::GetInstance()->GetResourceManager()->DirtyResources(name); } void ResourceDirtyByName(const char* name) { @@ -194,7 +194,7 @@ void ResourceUnloadByCrc(uint64_t crc) { } void ResourceUnloadDirectory(const char* name) { - Ship::Context::GetInstance()->GetResourceManager()->UnloadDirectory(name); + Ship::Context::GetInstance()->GetResourceManager()->UnloadResources(name); } uint32_t IsResourceManagerLoaded() { diff --git a/src/resource/ResourceManager.cpp b/src/resource/ResourceManager.cpp index 9326a0de6..b1f0b6f19 100644 --- a/src/resource/ResourceManager.cpp +++ b/src/resource/ResourceManager.cpp @@ -11,6 +11,11 @@ namespace Ship { +ResourceFilter::ResourceFilter(const std::list& includeMasks, const std::list& excludeMasks, + const uintptr_t owner, const std::shared_ptr parent) + : IncludeMasks(includeMasks), ExcludeMasks(excludeMasks), Owner(owner), Parent(parent) { +} + size_t ResourceIdentifier::GetHash() const { return mHash; } @@ -284,72 +289,75 @@ ResourceManager::GetCachedResource(std::variant>>> -ResourceManager::LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority) { - auto loadedList = std::make_shared>>>(); - auto fileList = GetArchiveManager()->ListFiles(identifier.Path); +std::shared_ptr>> +ResourceManager::LoadResourcesProcess(const ResourceFilter& filter) { + auto loadedList = std::make_shared>>(); + auto fileList = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); loadedList->reserve(fileList->size()); for (size_t i = 0; i < fileList->size(); i++) { auto fileName = std::string(fileList->operator[](i)); - auto future = LoadResourceAsync({ fileName, identifier.Owner, identifier.Parent }, false, priority); - loadedList->push_back(future); + auto resource = LoadResource({ fileName, filter.Owner, filter.Parent }); + loadedList->push_back(resource); } return loadedList; } -std::shared_ptr>>> -ResourceManager::LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority) { - return LoadDirectoryAsync({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }, priority); +std::shared_future>>> +ResourceManager::LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t priority) { + return mThreadPool->submit_task( + [this, filter]() -> std::shared_ptr>> { + return LoadResourcesProcess(filter); + }, + priority); } -std::shared_ptr>> -ResourceManager::LoadDirectory(const ResourceIdentifier& identifier) { - auto futureList = LoadDirectoryAsync(identifier, true); - auto loadedList = std::make_shared>>(); - - for (size_t i = 0; i < futureList->size(); i++) { - const auto future = futureList->at(i); - const auto resource = future.get(); - loadedList->push_back(resource); - } +std::shared_future>>> +ResourceManager::LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority) { + return LoadResourcesAsync({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }, priority); +} - return loadedList; +std::shared_ptr>> ResourceManager::LoadResources(const std::string& searchMask) { + return LoadResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }); } -std::shared_ptr>> ResourceManager::LoadDirectory(const std::string& searchMask) { - return LoadDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }); +std::shared_ptr>> ResourceManager::LoadResources(const ResourceFilter& filter) { + return LoadResourcesAsync(filter, BS::pr::highest).get(); } -void ResourceManager::DirtyDirectory(const ResourceIdentifier& identifier) { - auto list = GetArchiveManager()->ListFiles(identifier.Path); +void ResourceManager::DirtyResources(const ResourceFilter& filter) { + mThreadPool->submit_task([this, filter]() -> void { + auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); - for (const auto& key : *list.get()) { - auto resource = GetCachedResource({ key, identifier.Owner, identifier.Parent }); - // If it's a resource, we will set the dirty flag, else we will just unload it. - if (resource != nullptr) { - resource->Dirty(); - } else { - UnloadResource(identifier); + for (const auto& key : *list.get()) { + auto resource = GetCachedResource({ key, filter.Owner, filter.Parent }); + // If it's a resource, we will set the dirty flag, else we will just unload it. + if (resource != nullptr) { + resource->Dirty(); + } else { + UnloadResource({ key, filter.Owner, filter.Parent }); + } } - } + }); } -void ResourceManager::DirtyDirectory(const std::string& searchMask) { - DirtyDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }); -} +void ResourceManager::UnloadResources(const ResourceFilter& filter) { + mThreadPool->submit_task([this, filter]() -> void { + auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks); -void ResourceManager::UnloadDirectory(const ResourceIdentifier& identifier) { - auto list = GetArchiveManager()->ListFiles(identifier.Path); + for (const auto& key : *list.get()) { + UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive }); + } + }); +} - for (const auto& key : *list.get()) { - UnloadResource({ key, identifier.Owner, identifier.Parent }); - } +void ResourceManager::DirtyResources(const std::string& searchMask) { + DirtyResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }); } -void ResourceManager::UnloadDirectory(const std::string& searchMask) { - UnloadDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }); +void ResourceManager::UnloadResources(const std::string& searchMask) { + UnloadResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }); } std::shared_ptr ResourceManager::GetArchiveManager() { diff --git a/src/resource/ResourceManager.h b/src/resource/ResourceManager.h index 003c153a0..e3a2e0cce 100644 --- a/src/resource/ResourceManager.h +++ b/src/resource/ResourceManager.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include #include @@ -18,13 +20,23 @@ namespace Ship { struct File; +struct ResourceFilter { + ResourceFilter(const std::list& includeMasks, const std::list& excludeMasks, + const uintptr_t owner, const std::shared_ptr parent); + + const std::list IncludeMasks; + const std::list ExcludeMasks; + const uintptr_t Owner = 0; + const std::shared_ptr Parent = nullptr; +}; + struct ResourceIdentifier { - friend class ResourceIdentifierHash; + friend struct ResourceIdentifierHash; ResourceIdentifier(const std::string& path, const uintptr_t owner, const std::shared_ptr parent); bool operator==(const ResourceIdentifier& rhs) const; - // Path can either be a Path or a Search Mask including globs depending on usage. + // Must be an exact path. Passing a path with a wildcard will return a fail state const std::string Path = ""; const uintptr_t Owner = 0; const std::shared_ptr Parent = nullptr; @@ -53,41 +65,43 @@ class ResourceManager { std::shared_ptr GetArchiveManager(); std::shared_ptr GetResourceLoader(); + std::shared_ptr GetCachedResource(const std::string& filePath, bool loadExact = false); std::shared_ptr GetCachedResource(const ResourceIdentifier& identifier, bool loadExact = false); + std::shared_ptr LoadResource(const std::string& filePath, bool loadExact = false, + std::shared_ptr initData = nullptr); std::shared_ptr LoadResource(const ResourceIdentifier& identifier, bool loadExact = false, std::shared_ptr initData = nullptr); + std::shared_ptr LoadResourceProcess(const std::string& filePath, bool loadExact = false, + std::shared_ptr initData = nullptr); std::shared_ptr LoadResourceProcess(const ResourceIdentifier& identifier, bool loadExact = false, std::shared_ptr initData = nullptr); - size_t UnloadResource(const ResourceIdentifier& identifier); + std::shared_future> + LoadResourceAsync(const std::string& filePath, bool loadExact = false, BS::priority_t priority = BS::pr::normal, + std::shared_ptr initData = nullptr); std::shared_future> LoadResourceAsync(const ResourceIdentifier& identifier, bool loadExact = false, BS::priority_t priority = BS::pr::normal, std::shared_ptr initData = nullptr); - std::shared_ptr>> LoadDirectory(const ResourceIdentifier& identifier); - std::shared_ptr>>> - LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority = BS::pr::normal); - void DirtyDirectory(const ResourceIdentifier& identifier); - void UnloadDirectory(const ResourceIdentifier& identifier); - - std::shared_ptr GetCachedResource(const std::string& filePath, bool loadExact = false); - std::shared_ptr LoadResource(const std::string& filePath, bool loadExact = false, - std::shared_ptr initData = nullptr); - std::shared_ptr LoadResourceProcess(const std::string& filePath, bool loadExact = false, - std::shared_ptr initData = nullptr); + size_t UnloadResource(const ResourceIdentifier& identifier); size_t UnloadResource(const std::string& filePath); - std::shared_future> - LoadResourceAsync(const std::string& filePath, bool loadExact = false, BS::priority_t priority = BS::pr::normal, - std::shared_ptr initData = nullptr); - std::shared_ptr>> LoadDirectory(const std::string& searchMask); - std::shared_ptr>>> - LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal); - void DirtyDirectory(const std::string& searchMask); - void UnloadDirectory(const std::string& searchMask); + + std::shared_ptr>> LoadResources(const std::string& searchMask); + std::shared_ptr>> LoadResources(const ResourceFilter& filter); + std::shared_future>>> + LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal); + std::shared_future>>> + LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t priority = BS::pr::normal); + + void DirtyResources(const std::string& searchMask); + void DirtyResources(const ResourceFilter& filter); + void UnloadResources(const std::string& searchMask); + void UnloadResources(const ResourceFilter& filter); bool OtrSignatureCheck(const char* fileName); bool IsAltAssetsEnabled(); void SetAltAssetsEnabled(bool isEnabled); protected: + std::shared_ptr>> LoadResourcesProcess(const ResourceFilter& filter); std::variant> CheckCache(const ResourceIdentifier& identifier, bool loadExact = false); std::shared_ptr LoadFileProcess(const ResourceIdentifier& identifier, diff --git a/src/resource/archive/ArchiveManager.cpp b/src/resource/archive/ArchiveManager.cpp index 7f160b048..53f24bc9a 100644 --- a/src/resource/archive/ArchiveManager.cpp +++ b/src/resource/archive/ArchiveManager.cpp @@ -64,10 +64,41 @@ bool ArchiveManager::HasFile(uint64_t hash) { return mFileToArchive.count(hash) > 0; } -std::shared_ptr> ArchiveManager::ListFiles(const std::string& filter) { +std::shared_ptr> ArchiveManager::ListFiles(const std::string& searchMask) { + std::list includes = {}; + if (searchMask.size() > 0) { + includes.push_back(searchMask); + } + return ListFiles({ searchMask }, {}); +} + +std::shared_ptr> ArchiveManager::ListFiles(const std::list& includes, + const std::list& excludes) { auto list = std::make_shared>(); for (const auto& [hash, path] : mHashes) { - if (filter.empty() || glob_match(filter.c_str(), path.c_str())) { + if (includes.empty() && excludes.empty()) { + list->push_back(path); + continue; + } + bool includeMatch = includes.empty(); + if (!includes.empty()) { + for (std::string filter : includes) { + if (glob_match(filter.c_str(), path.c_str())) { + includeMatch = true; + break; + } + } + } + bool excludeMatch = false; + if (!excludes.empty()) { + for (std::string filter : excludes) { + if (glob_match(filter.c_str(), path.c_str())) { + excludeMatch = true; + break; + } + } + } + if (includeMatch && !excludeMatch) { list->push_back(path); } } diff --git a/src/resource/archive/ArchiveManager.h b/src/resource/archive/ArchiveManager.h index 911512dde..b1cc0c540 100644 --- a/src/resource/archive/ArchiveManager.h +++ b/src/resource/archive/ArchiveManager.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,9 @@ class ArchiveManager { std::shared_ptr LoadFile(uint64_t hash, std::shared_ptr initData = nullptr); bool HasFile(const std::string& filePath); bool HasFile(uint64_t hash); - std::shared_ptr> ListFiles(const std::string& filter = ""); + std::shared_ptr> ListFiles(const std::string& searchMask = ""); + std::shared_ptr> ListFiles(const std::list& includes, + const std::list& excludes); std::vector GetGameVersions(); const std::string* HashToString(uint64_t hash) const; bool IsGameVersionValid(uint32_t gameVersion);