Skip to content

Commit

Permalink
Add ResourceFilter, used to handle wildcards in paths.
Browse files Browse the repository at this point in the history 
Change `LoadDirectory`, `UnloadDirectory`, and `DirtyDirectory` to `LoadResources`, etc.
Move all Directory processing to the thread pool, encapsulating `ListFiles` as well.
Unify `ListFiles` flows to eventually pass `ResourceFilter` to the base function.
  • Loading branch information
@Malkierian
Malkierian committed Dec 17, 2024
1 parent 90153c1 commit cbb41c4
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 135 deletions.
8 changes: 4 additions & 4 deletions src/public/bridge/resourcebridge.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ void ResourceGetGameVersions(uint32_t* versions, size_t versionsSize, size_t* ve
}

void ResourceLoadDirectoryAsync(const char* name) {
Ship::Context::GetInstance()->GetResourceManager()->LoadDirectoryAsync(name);
Ship::Context::GetInstance()->GetResourceManager()->LoadResourcesAsync(name);
}

uint32_t ResourceHasGameVersion(uint32_t hash) {
Expand All @@ -162,11 +162,11 @@ uint32_t ResourceHasGameVersion(uint32_t hash) {
}

void ResourceLoadDirectory(const char* name) {
Ship::Context::GetInstance()->GetResourceManager()->LoadDirectory(name);
Ship::Context::GetInstance()->GetResourceManager()->LoadResources(name);
}

void ResourceDirtyDirectory(const char* name) {
Ship::Context::GetInstance()->GetResourceManager()->DirtyDirectory(name);
Ship::Context::GetInstance()->GetResourceManager()->DirtyResources(name);
}

void ResourceDirtyByName(const char* name) {
Expand Down Expand Up @@ -194,7 +194,7 @@ void ResourceUnloadByCrc(uint64_t crc) {
}

void ResourceUnloadDirectory(const char* name) {
Ship::Context::GetInstance()->GetResourceManager()->UnloadDirectory(name);
Ship::Context::GetInstance()->GetResourceManager()->UnloadResources(name);
}

uint32_t IsResourceManagerLoaded() {
Expand Down
131 changes: 50 additions & 81 deletions src/resource/ResourceManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@

namespace Ship {

ResourceFilter::ResourceFilter(const std::list<std::string> includeMasks, const std::list<std::string> excludeMasks,
const uintptr_t owner, const std::shared_ptr<Archive> parent)
: IncludeMasks(includeMasks), ExcludeMasks(excludeMasks), Owner(owner), Parent(parent) {}

size_t ResourceIdentifier::GetHash() const {
return mHash;
}
Expand Down Expand Up @@ -284,112 +288,77 @@ ResourceManager::GetCachedResource(std::variant<ResourceLoadError, std::shared_p
return nullptr;
}

std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
ResourceManager::LoadDirectoryAsyncWithExclude(const std::vector<std::string>& includeMasks,
const std::vector<std::string>& excludeMasks, BS::priority_t priority) {
auto loadedList = std::make_shared<std::vector<std::shared_future<std::shared_ptr<IResource>>>>();
auto fileList = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks);
loadedList->reserve(fileList->size());

for (size_t i = 0; i < fileList->size(); i++) {
auto fileName = std::string(fileList->operator[](i));
auto future = LoadResourceAsync(fileName, priority);
loadedList->push_back(future);
}

return loadedList;
}

std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
ResourceManager::LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority) {
auto loadedList = std::make_shared<std::vector<std::shared_future<std::shared_ptr<IResource>>>>();
auto fileList = GetArchiveManager()->ListFiles(identifier.Path);
std::shared_ptr<std::vector<std::shared_ptr<IResource>>>
ResourceManager::LoadResourcesProcess(const ResourceFilter& filter) {
auto loadedList = std::make_shared<std::vector<std::shared_ptr<IResource>>>();
auto fileList = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks);
loadedList->reserve(fileList->size());

for (size_t i = 0; i < fileList->size(); i++) {
auto fileName = std::string(fileList->operator[](i));
auto future = LoadResourceAsync({ fileName, identifier.Owner, identifier.Parent }, false, priority);
loadedList->push_back(future);
}

return loadedList;
}

std::shared_ptr<std::vector<std::shared_ptr<IResource>>>
ResourceManager::LoadDirectoryWithExclude(const std::vector<std::string>& includeMasks,
const std::vector<std::string>& excludeMasks, uintptr_t owner) {
auto futureList = LoadDirectoryAsyncWithExclude(includeMasks, excludeMasks);
auto loadedList = std::make_shared<std::vector<std::shared_ptr<IResource>>>();

for (size_t i = 0; i < futureList->size(); i++) {
const auto future = futureList->at(i);
const auto resource = future.get();
auto resource = LoadResource({fileName, filter.Owner, filter.Parent});
loadedList->push_back(resource);
}

return loadedList;
}

std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
ResourceManager::LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority) {
return LoadDirectoryAsync({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive }, priority);
std::shared_future<std::shared_ptr<std::vector<std::shared_ptr<IResource>>>>
ResourceManager::LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t priority) {
return mThreadPool->submit_task(
[this, filter]() -> std::shared_ptr<std::vector<std::shared_ptr<IResource>>> {
return LoadResourcesProcess(filter);
},
priority);
}

std::shared_ptr<std::vector<std::shared_ptr<IResource>>>
ResourceManager::LoadDirectory(const ResourceIdentifier& identifier) {
auto futureList = LoadDirectoryAsync(identifier, true);
auto loadedList = std::make_shared<std::vector<std::shared_ptr<IResource>>>();

for (size_t i = 0; i < futureList->size(); i++) {
const auto future = futureList->at(i);
const auto resource = future.get();
loadedList->push_back(resource);
}

return loadedList;
std::shared_future<std::shared_ptr<std::vector<std::shared_ptr<IResource>>>>
ResourceManager::LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority) {
return LoadResourcesAsync({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive}, priority);
}

std::shared_ptr<std::vector<std::shared_ptr<IResource>>> ResourceManager::LoadDirectory(const std::string& searchMask) {
return LoadDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive });
std::shared_ptr<std::vector<std::shared_ptr<IResource>>> ResourceManager::LoadResources(const std::string& searchMask) {
return LoadResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive});
}

void ResourceManager::DirtyDirectory(const ResourceIdentifier& identifier) {
auto list = GetArchiveManager()->ListFiles(identifier.Path);

for (const auto& key : *list.get()) {
auto resource = GetCachedResource({ key, identifier.Owner, identifier.Parent });
// If it's a resource, we will set the dirty flag, else we will just unload it.
if (resource != nullptr) {
resource->Dirty();
} else {
UnloadResource(identifier);
}
}
std::shared_ptr<std::vector<std::shared_ptr<IResource>>> ResourceManager::LoadResources(const ResourceFilter& filter) {
return LoadResourcesAsync(filter, BS::pr::highest).get();
}

void ResourceManager::UnloadDirectoryWithExclude(const std::vector<std::string>& includeMasks,
const std::vector<std::string>& excludeMasks) {
auto list = GetArchiveManager()->ListFilesWithExclude(includeMasks, excludeMasks);
void ResourceManager::DirtyResources(const ResourceFilter& filter) {
mThreadPool->submit_task(
[this, filter]() -> void {
auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks);

for (const auto& key : *list.get()) {
UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive });
}
for (const auto& key : *list.get()) {
auto resource = GetCachedResource({ key, filter.Owner, filter.Parent });
// If it's a resource, we will set the dirty flag, else we will just unload it.
if (resource != nullptr) {
resource->Dirty();
} else {
UnloadResource({ key, filter.Owner, filter.Parent });
}
}
});
}

void ResourceManager::DirtyDirectory(const std::string& searchMask) {
DirtyDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive });
}
void ResourceManager::UnloadResources(const ResourceFilter& filter) {
mThreadPool->submit_task(
[this, filter]() -> void {
auto list = GetArchiveManager()->ListFiles(filter.IncludeMasks, filter.ExcludeMasks);

void ResourceManager::UnloadDirectory(const ResourceIdentifier& identifier) {
auto list = GetArchiveManager()->ListFiles(identifier.Path);
for (const auto& key : *list.get()) {
UnloadResource({ key, mDefaultCacheOwner, mDefaultCacheArchive });
}
});
}

for (const auto& key : *list.get()) {
UnloadResource({ key, identifier.Owner, identifier.Parent });
}
void ResourceManager::DirtyResources(const std::string& searchMask) {
DirtyResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive});
}

void ResourceManager::UnloadDirectory(const std::string& searchMask) {
UnloadDirectory({ searchMask, mDefaultCacheOwner, mDefaultCacheArchive });
void ResourceManager::UnloadResources(const std::string& searchMask) {
UnloadResources({ {searchMask}, {}, mDefaultCacheOwner, mDefaultCacheArchive});
}

std::shared_ptr<ArchiveManager> ResourceManager::GetArchiveManager() {
Expand Down
65 changes: 35 additions & 30 deletions src/resource/ResourceManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,23 @@
namespace Ship {
struct File;

struct ResourceFilter {
ResourceFilter(const std::list<std::string> includeMasks, const std::list<std::string> excludeMasks,
const uintptr_t owner, const std::shared_ptr<Archive> parent);

const std::list<std::string> IncludeMasks;
const std::list<std::string> ExcludeMasks;
const uintptr_t Owner = 0;
const std::shared_ptr<Archive> Parent = nullptr;
};

struct ResourceIdentifier {
friend class ResourceIdentifierHash;
friend struct ResourceIdentifierHash;

ResourceIdentifier(const std::string& path, const uintptr_t owner, const std::shared_ptr<Archive> parent);
bool operator==(const ResourceIdentifier& rhs) const;

// Path can either be a Path or a Search Mask including globs depending on usage.
// Must be an exact path. Passing a path with a wildcard will return a fail state
const std::string Path = "";
const uintptr_t Owner = 0;
const std::shared_ptr<Archive> Parent = nullptr;
Expand Down Expand Up @@ -53,50 +63,45 @@ class ResourceManager {
std::shared_ptr<ArchiveManager> GetArchiveManager();
std::shared_ptr<ResourceLoader> GetResourceLoader();

std::shared_ptr<IResource> GetCachedResource(const std::string& filePath, bool loadExact = false);
std::shared_ptr<IResource> GetCachedResource(const ResourceIdentifier& identifier, bool loadExact = false);
std::shared_ptr<IResource> LoadResource(const std::string& filePath, bool loadExact = false,
std::shared_ptr<ResourceInitData> initData = nullptr);
std::shared_ptr<IResource> LoadResource(const ResourceIdentifier& identifier, bool loadExact = false,
std::shared_ptr<ResourceInitData> initData = nullptr);
std::shared_ptr<IResource> LoadResourceProcess(const std::string& filePath, bool loadExact = false,
std::shared_ptr<ResourceInitData> initData = nullptr);
std::shared_ptr<IResource> LoadResourceProcess(const ResourceIdentifier& identifier, bool loadExact = false,
std::shared_ptr<ResourceInitData> initData = nullptr);
size_t UnloadResource(const ResourceIdentifier& identifier);
std::shared_future<std::shared_ptr<IResource>>
LoadResourceAsync(const std::string& filePath, bool loadExact = false, BS::priority_t priority = BS::pr::normal,
std::shared_ptr<ResourceInitData> initData = nullptr);
std::shared_future<std::shared_ptr<IResource>>
LoadResourceAsync(const ResourceIdentifier& identifier, bool loadExact = false,
BS::priority_t priority = BS::pr::normal, std::shared_ptr<ResourceInitData> initData = nullptr);
size_t UnloadResource(const ResourceIdentifier& identifier);
size_t UnloadResource(const std::string& filePath);

std::shared_ptr<std::vector<std::shared_ptr<IResource>>> LoadResources(const std::string& searchMask);
std::shared_ptr<std::vector<std::shared_ptr<IResource>>>
LoadDirectoryWithExclude(const std::vector<std::string>& includeMasks, const std::vector<std::string>& excludeMasks,
uintptr_t owner);
std::shared_ptr<std::vector<std::shared_ptr<IResource>>> LoadDirectory(const ResourceIdentifier& identifier);
std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
LoadDirectoryAsync(const ResourceIdentifier& identifier, BS::priority_t priority = BS::pr::normal);
void DirtyDirectory(const ResourceIdentifier& identifier);
void UnloadDirectory(const ResourceIdentifier& identifier);
LoadResources(const ResourceFilter& filter);
std::shared_future<std::shared_ptr<std::vector<std::shared_ptr<IResource>>>>
LoadResourcesAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal);
std::shared_future<std::shared_ptr<std::vector<std::shared_ptr<IResource>>>>
LoadResourcesAsync(const ResourceFilter& filter, BS::priority_t priority = BS::pr::normal);

std::shared_ptr<IResource> GetCachedResource(const std::string& filePath, bool loadExact = false);
std::shared_ptr<IResource> LoadResource(const std::string& filePath, bool loadExact = false,
std::shared_ptr<ResourceInitData> initData = nullptr);
std::shared_ptr<IResource> LoadResourceProcess(const std::string& filePath, bool loadExact = false,
std::shared_ptr<ResourceInitData> initData = nullptr);
size_t UnloadResource(const std::string& filePath);
std::shared_future<std::shared_ptr<IResource>>
LoadResourceAsync(const std::string& filePath, bool loadExact = false, BS::priority_t priority = BS::pr::normal,
std::shared_ptr<ResourceInitData> initData = nullptr);
std::shared_ptr<std::vector<std::shared_ptr<IResource>>> LoadDirectory(const std::string& searchMask);
std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
LoadDirectoryAsync(const std::string& searchMask, BS::priority_t priority = BS::pr::normal);
std::shared_ptr<std::vector<std::shared_future<std::shared_ptr<IResource>>>>
LoadDirectoryAsyncWithExclude(const std::vector<std::string>& includeMasks,
const std::vector<std::string>& excludeMasks,
BS::priority_t priority = BS::pr::normal);
void DirtyDirectory(const std::string& searchMask);
void UnloadDirectoryWithExclude(const std::vector<std::string>& includeMasks,
const std::vector<std::string>& excludeMasks);
void UnloadDirectory(const std::string& searchMask);
void DirtyResources(const std::string& searchMask);
void DirtyResources(const ResourceFilter& filter);
void UnloadResources(const std::string& searchMask);
void UnloadResources(const ResourceFilter& filter);

bool OtrSignatureCheck(const char* fileName);
bool IsAltAssetsEnabled();
void SetAltAssetsEnabled(bool isEnabled);

protected:
std::shared_ptr<std::vector<std::shared_ptr<IResource>>>
LoadResourcesProcess(const ResourceFilter& filter);
std::variant<ResourceLoadError, std::shared_ptr<IResource>> CheckCache(const ResourceIdentifier& identifier,
bool loadExact = false);
std::shared_ptr<File> LoadFileProcess(const ResourceIdentifier& identifier,
Expand Down
35 changes: 18 additions & 17 deletions src/resource/archive/ArchiveManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,25 +65,36 @@ bool ArchiveManager::HasFile(uint64_t hash) {
}

std::shared_ptr<std::vector<std::string>>
ArchiveManager::ListFilesWithExclude(const std::vector<std::string>& include, const std::vector<std::string>& exclude) {
ArchiveManager::ListFiles() {
return ListFiles({ "*" }, {});
}

std::shared_ptr<std::vector<std::string>>
ArchiveManager::ListFiles(const std::string& searchMask) {
return ListFiles({ searchMask }, {});
}

std::shared_ptr<std::vector<std::string>>
ArchiveManager::ListFiles(const std::list<std::string>& includes,
const std::list<std::string>& excludes) {
auto list = std::make_shared<std::vector<std::string>>();
for (const auto& [hash, path] : mHashes) {
if (include.empty() && exclude.empty()) {
if (includes.empty() && excludes.empty()) {
list->push_back(path);
continue;
}
bool includeMatch = include.empty();
if (!include.empty()) {
for (std::string filter : include) {
bool includeMatch = includes.empty();
if (!includes.empty()) {
for (std::string filter : includes) {
if (glob_match(filter.c_str(), path.c_str())) {
includeMatch = true;
break;
}
}
}
bool excludeMatch = false;
if (!include.empty()) {
for (std::string filter : exclude) {
if (!excludes.empty()) {
for (std::string filter : excludes) {
if (glob_match(filter.c_str(), path.c_str())) {
excludeMatch = true;
break;
Expand All @@ -97,16 +108,6 @@ ArchiveManager::ListFilesWithExclude(const std::vector<std::string>& include, co
return list;
}

std::shared_ptr<std::vector<std::string>> ArchiveManager::ListFiles(const std::string& filter) {
auto list = std::make_shared<std::vector<std::string>>();
for (const auto& [hash, path] : mHashes) {
if (filter.empty() || glob_match(filter.c_str(), path.c_str())) {
list->push_back(path);
}
}
return list;
}

std::vector<uint32_t> ArchiveManager::GetGameVersions() {
return mGameVersions;
}
Expand Down
7 changes: 4 additions & 3 deletions src/resource/archive/ArchiveManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,10 @@ class ArchiveManager {
std::shared_ptr<File> LoadFile(uint64_t hash, std::shared_ptr<ResourceInitData> initData = nullptr);
bool HasFile(const std::string& filePath);
bool HasFile(uint64_t hash);
std::shared_ptr<std::vector<std::string>> ListFilesWithExclude(const std::vector<std::string>& include = {},
const std::vector<std::string>& exclude = {});
std::shared_ptr<std::vector<std::string>> ListFiles(const std::string& filter = "");
std::shared_ptr<std::vector<std::string>> ListFiles();
std::shared_ptr<std::vector<std::string>> ListFiles(const std::string& searchMask);
std::shared_ptr<std::vector<std::string>> ListFiles(const std::list<std::string>& includes,
const std::list<std::string>& excludes);
std::vector<uint32_t> GetGameVersions();
const std::string* HashToString(uint64_t hash) const;
bool IsGameVersionValid(uint32_t gameVersion);
Expand Down

0 comments on commit cbb41c4

Please sign in to comment.