ResourceFilter, Directory Function Renames, Multifilter Functionality (…

…#752)

* Adds version of `ListFiles`, `ListFilesWithExclude`, allowing for multiple include and exclude filters in one pass. Also adds associated Load, LoadAsync, and Unload directory functions.

* Update for latest LUS main.

* Clang

* Add `ResourceFilter`, used to handle wildcards in paths.
Change `LoadDirectory`, `UnloadDirectory`, and `DirtyDirectory` to `LoadResources`, etc.
Move all Directory processing to the thread pool, encapsulating `ListFiles` as well.
Unify `ListFiles` flows to eventually pass `ResourceFilter` to the base function.

* clang

* Try to address build job fails.

* Fix reference passing and internal storage for `ResourceFilter` flows.
Change the blank `ListFiles()` function to pass empty lists instead of "*" to allow it to trip the early list insertion and return short circuit based on them being empty.

* Remove blank `ListFiles` in favor of default of `""` searchMask for the other one.
Add handling to not add `""` to the list before passing to the base function.

src/public/bridge/resourcebridge.cpp

@@ -153,7 +153,7 @@ void ResourceGetGameVersions(uint32_t* versions, size_t versionsSize, size_t* ve
 }
 
 void ResourceLoadDirectoryAsync(const char* name) {
-    Ship::Context::GetInstance()->GetResourceManager()->LoadDirectoryAsync(name);
+    Ship::Context::GetInstance()->GetResourceManager()->LoadResourcesAsync(name);
 }
 
 uint32_t ResourceHasGameVersion(uint32_t hash) {
@@ -162,11 +162,11 @@ uint32_t ResourceHasGameVersion(uint32_t hash) {
 }
 
 void ResourceLoadDirectory(const char* name) {
-    Ship::Context::GetInstance()->GetResourceManager()->LoadDirectory(name);
+    Ship::Context::GetInstance()->GetResourceManager()->LoadResources(name);
 }
 
 void ResourceDirtyDirectory(const char* name) {
-    Ship::Context::GetInstance()->GetResourceManager()->DirtyDirectory(name);
+    Ship::Context::GetInstance()->GetResourceManager()->DirtyResources(name);
 }
 
 void ResourceDirtyByName(const char* name) {
@@ -194,7 +194,7 @@ void ResourceUnloadByCrc(uint64_t crc) {
 }
 
 void ResourceUnloadDirectory(const char* name) {
-    Ship::Context::GetInstance()->GetResourceManager()->UnloadDirectory(name);
+    Ship::Context::GetInstance()->GetResourceManager()->UnloadResources(name);
 }
 
 uint32_t IsResourceManagerLoaded() {

src/resource/ResourceManager.cpp

@@ -11,6 +11,11 @@
 
 namespace Ship {
 
+ResourceFilter::ResourceFilter(const std::list<std::string>& includeMasks, const std::list<std::string>& excludeMasks,
+                               const uintptr_t owner, const std::shared_ptr<Archive> parent)
+    : IncludeMasks(includeMasks), ExcludeMasks(excludeMasks), Owner(owner), Parent(parent) {
+}
+
 size_t ResourceIdentifier::GetHash() const {
     return mHash;
 }
@@ -284,72 +289,75 @@ ResourceManager::GetCachedResource(std::variant<ResourceLoadError, std::shared_p
     return nullptr;
 }
 
-std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
-ResourceManager::LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority) {
-    auto loadedList = std::make_shared<std::vector<std::shared_future<std::shared_ptr<IResource>>>>();
-    auto fileList = GetArchiveManager()->ListFiles(identifier.Path);
+std::shared_ptr<std::vector<std::shared_ptr<IResource>>>
+ResourceManager::LoadResourcesProcess(const ResourceFilter& filter) {
+    auto loadedList = std::make_shared<std::vector<std::shared_ptr<IResource>>>();
+    auto fileList = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks);
     loadedList->reserve(fileList->size());
 
     for (size_t i = 0; i < fileList->size(); i++) {
         auto fileName = std::string(fileList->operator[](i));
-        auto future = LoadResourceAsync({ fileName, identifier.Owner, identifier.Parent }, false, priority);
-        loadedList->push_back(future);
+        auto resource = LoadResource({ fileName, filter.Owner, filter.Parent });
+        loadedList->push_back(resource);
     }
 
     return loadedList;
 }
 
-std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
-ResourceManager::LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority) {
-    return LoadDirectoryAsync({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }, priority);
+std::shared_future<std::shared_ptr<std::vector<std::shared_ptr<IResource>>>>
+ResourceManager::LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t priority) {
+    return mThreadPool->submit_task(
+        [this, filter]() -> std::shared_ptr<std::vector<std::shared_ptr<IResource>>> {
+            return LoadResourcesProcess(filter);
+        },
+        priority);
 }
 
-std::shared_ptr<std::vector<std::shared_ptr<IResource>>>
-ResourceManager::LoadDirectory(const ResourceIdentifier& identifier) {
-    auto futureList = LoadDirectoryAsync(identifier, true);
-    auto loadedList = std::make_shared<std::vector<std::shared_ptr<IResource>>>();
-
-    for (size_t i = 0; i < futureList->size(); i++) {
-        const auto future = futureList->at(i);
-        const auto resource = future.get();
-        loadedList->push_back(resource);
-    }
+std::shared_future<std::shared_ptr<std::vector<std::shared_ptr<IResource>>>>
+ResourceManager::LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority) {
+    return LoadResourcesAsync({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive }, priority);
+}
 
-    return loadedList;
+std::shared_ptr<std::vector<std::shared_ptr<IResource>>> ResourceManager::LoadResources(const std::string& searchMask) {
+    return LoadResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive });
 }
 
-std::shared_ptr<std::vector<std::shared_ptr<IResource>>> ResourceManager::LoadDirectory(const std::string& searchMask) {
-    return LoadDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive });
+std::shared_ptr<std::vector<std::shared_ptr<IResource>>> ResourceManager::LoadResources(const ResourceFilter& filter) {
+    return LoadResourcesAsync(filter, BS::pr::highest).get();
 }
 
-void ResourceManager::DirtyDirectory(const ResourceIdentifier& identifier) {
-    auto list = GetArchiveManager()->ListFiles(identifier.Path);
+void ResourceManager::DirtyResources(const ResourceFilter& filter) {
+    mThreadPool->submit_task([this, filter]() -> void {
+        auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks);
 
-    for (const auto& key : *list.get()) {
-        auto resource = GetCachedResource({ key, identifier.Owner, identifier.Parent });
-        // If it's a resource, we will set the dirty flag, else we will just unload it.
-        if (resource != nullptr) {
-            resource->Dirty();
-        } else {
-            UnloadResource(identifier);
+        for (const auto& key : *list.get()) {
+            auto resource = GetCachedResource({ key, filter.Owner, filter.Parent });
+            // If it's a resource, we will set the dirty flag, else we will just unload it.
+            if (resource != nullptr) {
+                resource->Dirty();
+            } else {
+                UnloadResource({ key, filter.Owner, filter.Parent });
+            }
         }
-    }
+    });
 }
 
-void ResourceManager::DirtyDirectory(const std::string& searchMask) {
-    DirtyDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive });
-}
+void ResourceManager::UnloadResources(const ResourceFilter& filter) {
+    mThreadPool->submit_task([this, filter]() -> void {
+        auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks);
 
-void ResourceManager::UnloadDirectory(const ResourceIdentifier& identifier) {
-    auto list = GetArchiveManager()->ListFiles(identifier.Path);
+        for (const auto& key : *list.get()) {
+            UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive });
+        }
+    });
+}
 
-    for (const auto& key : *list.get()) {
-        UnloadResource({ key, identifier.Owner, identifier.Parent });
-    }
+void ResourceManager::DirtyResources(const std::string& searchMask) {
+    DirtyResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive });
 }
 
-void ResourceManager::UnloadDirectory(const std::string& searchMask) {
-    UnloadDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive });
+void ResourceManager::UnloadResources(const std::string& searchMask) {
+    UnloadResources({ { searchMask }, {}, mDefaultCacheOwner, mDefaultCacheArchive });
 }
 
 std::shared_ptr<ArchiveManager> ResourceManager::GetArchiveManager() {

src/resource/ResourceManager.h

@@ -3,6 +3,8 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <string>
+#include <list>
+#include <vector>
 #include <mutex>
 #include <queue>
 #include <variant>
@@ -18,13 +20,23 @@
 namespace Ship {
 struct File;
 
+struct ResourceFilter {
+    ResourceFilter(const std::list<std::string>& includeMasks, const std::list<std::string>& excludeMasks,
+                   const uintptr_t owner, const std::shared_ptr<Archive> parent);
+
+    const std::list<std::string> IncludeMasks;
+    const std::list<std::string> ExcludeMasks;
+    const uintptr_t Owner = 0;
+    const std::shared_ptr<Archive> Parent = nullptr;
+};
+
 struct ResourceIdentifier {
-    friend class ResourceIdentifierHash;
+    friend struct ResourceIdentifierHash;
 
     ResourceIdentifier(const std::string& path, const uintptr_t owner, const std::shared_ptr<Archive> parent);
     bool operator==(const ResourceIdentifier& rhs) const;
 
-    // Path can either be a Path or a Search Mask including globs depending on usage.
+    // Must be an exact path. Passing a path with a wildcard will return a fail state
     const std::string Path = "";
     const uintptr_t Owner = 0;
     const std::shared_ptr<Archive> Parent = nullptr;
@@ -53,41 +65,43 @@ class ResourceManager {
     std::shared_ptr<ArchiveManager> GetArchiveManager();
     std::shared_ptr<ResourceLoader> GetResourceLoader();
 
+    std::shared_ptr<IResource> GetCachedResource(const std::string& filePath, bool loadExact = false);
     std::shared_ptr<IResource> GetCachedResource(const ResourceIdentifier& identifier, bool loadExact = false);
+    std::shared_ptr<IResource> LoadResource(const std::string& filePath, bool loadExact = false,
+                                            std::shared_ptr<ResourceInitData> initData = nullptr);
     std::shared_ptr<IResource> LoadResource(const ResourceIdentifier& identifier, bool loadExact = false,
                                             std::shared_ptr<ResourceInitData> initData = nullptr);
+    std::shared_ptr<IResource> LoadResourceProcess(const std::string& filePath, bool loadExact = false,
+                                                   std::shared_ptr<ResourceInitData> initData = nullptr);
     std::shared_ptr<IResource> LoadResourceProcess(const ResourceIdentifier& identifier, bool loadExact = false,
                                                    std::shared_ptr<ResourceInitData> initData = nullptr);
-    size_t UnloadResource(const ResourceIdentifier& identifier);
+    std::shared_future<std::shared_ptr<IResource>>
+    LoadResourceAsync(const std::string& filePath, bool loadExact = false, BS::priority_t priority = BS::pr::normal,
+                      std::shared_ptr<ResourceInitData> initData = nullptr);
     std::shared_future<std::shared_ptr<IResource>>
     LoadResourceAsync(const ResourceIdentifier& identifier, bool loadExact = false,
                       BS::priority_t priority = BS::pr::normal, std::shared_ptr<ResourceInitData> initData = nullptr);
-    std::shared_ptr<std::vector<std::shared_ptr<IResource>>> LoadDirectory(const ResourceIdentifier& identifier);
-    std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
-    LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority = BS::pr::normal);
-    void DirtyDirectory(const ResourceIdentifier& identifier);
-    void UnloadDirectory(const ResourceIdentifier& identifier);
-
-    std::shared_ptr<IResource> GetCachedResource(const std::string& filePath, bool loadExact = false);
-    std::shared_ptr<IResource> LoadResource(const std::string& filePath, bool loadExact = false,
-                                            std::shared_ptr<ResourceInitData> initData = nullptr);
-    std::shared_ptr<IResource> LoadResourceProcess(const std::string& filePath, bool loadExact = false,
-                                                   std::shared_ptr<ResourceInitData> initData = nullptr);
+    size_t UnloadResource(const ResourceIdentifier& identifier);
     size_t UnloadResource(const std::string& filePath);
-    std::shared_future<std::shared_ptr<IResource>>
-    LoadResourceAsync(const std::string& filePath, bool loadExact = false, BS::priority_t priority = BS::pr::normal,
-                      std::shared_ptr<ResourceInitData> initData = nullptr);
-    std::shared_ptr<std::vector<std::shared_ptr<IResource>>> LoadDirectory(const std::string& searchMask);
-    std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
-    LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal);
-    void DirtyDirectory(const std::string& searchMask);
-    void UnloadDirectory(const std::string& searchMask);
+
+    std::shared_ptr<std::vector<std::shared_ptr<IResource>>> LoadResources(const std::string& searchMask);
+    std::shared_ptr<std::vector<std::shared_ptr<IResource>>> LoadResources(const ResourceFilter& filter);
+    std::shared_future<std::shared_ptr<std::vector<std::shared_ptr<IResource>>>>
+    LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal);
+    std::shared_future<std::shared_ptr<std::vector<std::shared_ptr<IResource>>>>
+    LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t priority = BS::pr::normal);
+
+    void DirtyResources(const std::string& searchMask);
+    void DirtyResources(const ResourceFilter& filter);
+    void UnloadResources(const std::string& searchMask);
+    void UnloadResources(const ResourceFilter& filter);
 
     bool OtrSignatureCheck(const char* fileName);
     bool IsAltAssetsEnabled();
     void SetAltAssetsEnabled(bool isEnabled);
 
   protected:
+    std::shared_ptr<std::vector<std::shared_ptr<IResource>>> LoadResourcesProcess(const ResourceFilter& filter);
     std::variant<ResourceLoadError, std::shared_ptr<IResource>> CheckCache(const ResourceIdentifier& identifier,
                                                                            bool loadExact = false);
     std::shared_ptr<File> LoadFileProcess(const ResourceIdentifier& identifier,

src/resource/archive/ArchiveManager.cpp

@@ -64,10 +64,41 @@ bool ArchiveManager::HasFile(uint64_t hash) {
     return mFileToArchive.count(hash) > 0;
 }
 
-std::shared_ptr<std::vector<std::string>> ArchiveManager::ListFiles(const std::string& filter) {
+std::shared_ptr<std::vector<std::string>> ArchiveManager::ListFiles(const std::string& searchMask) {
+    std::list<std::string> includes = {};
+    if (searchMask.size() > 0) {
+        includes.push_back(searchMask);
+    }
+    return ListFiles({ searchMask }, {});
+}
+
+std::shared_ptr<std::vector<std::string>> ArchiveManager::ListFiles(const std::list<std::string>& includes,
+                                                                    const std::list<std::string>& excludes) {
     auto list = std::make_shared<std::vector<std::string>>();
     for (const auto& [hash, path] : mHashes) {
-        if (filter.empty() || glob_match(filter.c_str(), path.c_str())) {
+        if (includes.empty() && excludes.empty()) {
+            list->push_back(path);
+            continue;
+        }
+        bool includeMatch = includes.empty();
+        if (!includes.empty()) {
+            for (std::string filter : includes) {
+                if (glob_match(filter.c_str(), path.c_str())) {
+                    includeMatch = true;
+                    break;
+                }
+            }
+        }
+        bool excludeMatch = false;
+        if (!excludes.empty()) {
+            for (std::string filter : excludes) {
+                if (glob_match(filter.c_str(), path.c_str())) {
+                    excludeMatch = true;
+                    break;
+                }
+            }
+        }
+        if (includeMatch && !excludeMatch) {
             list->push_back(path);
         }
     }

src/resource/archive/ArchiveManager.h

@@ -3,6 +3,7 @@
 #include <string>
 #include <memory>
 #include <vector>
+#include <list>
 #include <unordered_map>
 #include <unordered_set>
 #include <stdint.h>
@@ -31,7 +32,9 @@ class ArchiveManager {
     std::shared_ptr<File> LoadFile(uint64_t hash, std::shared_ptr<ResourceInitData> initData = nullptr);
     bool HasFile(const std::string& filePath);
     bool HasFile(uint64_t hash);
-    std::shared_ptr<std::vector<std::string>> ListFiles(const std::string& filter = "");
+    std::shared_ptr<std::vector<std::string>> ListFiles(const std::string& searchMask = "");
+    std::shared_ptr<std::vector<std::string>> ListFiles(const std::list<std::string>& includes,
+                                                        const std::list<std::string>& excludes);
     std::vector<uint32_t> GetGameVersions();
     const std::string* HashToString(uint64_t hash) const;
     bool IsGameVersionValid(uint32_t gameVersion);