From d9408ccb2538a7e2c510c7a7f1b4a164a8e0847b Mon Sep 17 00:00:00 2001 From: Konstantin Savosteev Date: Wed, 25 Sep 2024 15:03:32 +0200 Subject: [PATCH] VCST-1736: add Module Sequence Boost (#2837) Co-authored-by: artem-dudarev --- .../Modularity/ModuleCatalog.cs | 34 +++--- .../Modularity/ModuleDependencySolver.cs | 108 ++++++++++++++---- .../Modularity/ModuleSequenceBoostOptions.cs | 6 + .../External/ExternalModuleCatalog.cs | 8 +- .../Local/LocalStorageModuleCatalog.cs | 4 +- .../Controllers/Api/ModulesController.cs | 20 ++++ src/VirtoCommerce.Platform.Web/Startup.cs | 2 + .../Modularity/ExternalModuleCatalogTests.cs | 3 +- .../Modularity/ModuleInstallerUnitTests.cs | 3 +- .../ModulePlatformCompatibilityTests.cs | 4 +- 10 files changed, 146 insertions(+), 46 deletions(-) create mode 100644 src/VirtoCommerce.Platform.Core/Modularity/ModuleSequenceBoostOptions.cs diff --git a/src/VirtoCommerce.Platform.Core/Modularity/ModuleCatalog.cs b/src/VirtoCommerce.Platform.Core/Modularity/ModuleCatalog.cs index d0b1231418e..66c63ecfbff 100644 --- a/src/VirtoCommerce.Platform.Core/Modularity/ModuleCatalog.cs +++ b/src/VirtoCommerce.Platform.Core/Modularity/ModuleCatalog.cs @@ -3,6 +3,7 @@ using System.Collections.ObjectModel; using System.Collections.Specialized; using System.Linq; +using Microsoft.Extensions.Options; using VirtoCommerce.Platform.Core.Common; using VirtoCommerce.Platform.Core.Modularity.Exceptions; @@ -27,16 +28,19 @@ namespace VirtoCommerce.Platform.Core.Modularity /// public class ModuleCatalog : IModuleCatalog { + private readonly ModuleSequenceBoostOptions _boostOptions; private readonly ModuleCatalogItemCollection items; private bool isLoaded; /// /// Initializes a new instance of the class. /// - public ModuleCatalog() + public ModuleCatalog(IOptions boostOptions) { - this.items = new ModuleCatalogItemCollection(); - this.items.CollectionChanged += this.ItemsCollectionChanged; + _boostOptions = boostOptions.Value; + + items = new ModuleCatalogItemCollection(); + items.CollectionChanged += ItemsCollectionChanged; } /// @@ -44,14 +48,15 @@ public ModuleCatalog() /// initial list of s. /// /// The initial list of modules. - public ModuleCatalog(IEnumerable modules) - : this() + /// Module boost options + public ModuleCatalog(IEnumerable modules, IOptions boostOptions) + : this(boostOptions) { - if (modules == null) - throw new System.ArgumentNullException("modules"); - foreach (ModuleInfo moduleInfo in modules) + ArgumentNullException.ThrowIfNull(modules); + + foreach (var moduleInfo in modules) { - this.Items.Add(moduleInfo); + Items.Add(moduleInfo); } } @@ -331,14 +336,11 @@ public virtual ModuleCatalog AddGroup(InitializationMode initializationMode, str /// /// the. /// - protected static string[] SolveDependencies(IEnumerable modules) + protected string[] SolveDependencies(IEnumerable modules) { - if (modules == null) - { - throw new System.ArgumentNullException("modules"); - } + ArgumentNullException.ThrowIfNull(modules); - var solver = new ModuleDependencySolver(); + var solver = new ModuleDependencySolver(_boostOptions); foreach (var data in modules.ToArray()) { @@ -363,7 +365,7 @@ protected static string[] SolveDependencies(IEnumerable modules) return solver.Solve(); } - return new string[0]; + return []; } /// diff --git a/src/VirtoCommerce.Platform.Core/Modularity/ModuleDependencySolver.cs b/src/VirtoCommerce.Platform.Core/Modularity/ModuleDependencySolver.cs index 050bdb42414..537046f530d 100644 --- a/src/VirtoCommerce.Platform.Core/Modularity/ModuleDependencySolver.cs +++ b/src/VirtoCommerce.Platform.Core/Modularity/ModuleDependencySolver.cs @@ -12,8 +12,16 @@ namespace VirtoCommerce.Platform.Core.Modularity /// public class ModuleDependencySolver { - private readonly ListDictionary dependencyMatrix = new ListDictionary(); - private readonly List knownModules = new List(); + private readonly ListDictionary _dependencyMatrix = []; + private readonly List _knownModules = []; + + private readonly List _boostedModules; + private readonly ListDictionary _boostedDependencyMatrix = []; + + public ModuleDependencySolver(ModuleSequenceBoostOptions boostOptions) + { + _boostedModules = boostOptions.ModuleSequenceBoost.ToList(); + } /// /// Adds a module to the solver. @@ -21,11 +29,18 @@ public class ModuleDependencySolver /// The name that uniquely identifies the module. public void AddModule(string name) { - if (String.IsNullOrEmpty(name)) + if (string.IsNullOrEmpty(name)) + { throw new ArgumentNullException(nameof(name)); + } AddToDependencyMatrix(name); AddToKnownModules(name); + + if (_boostedModules.Contains(name)) + { + AddToBoostedDependencyMatrix(name); + } } /// @@ -37,32 +52,54 @@ public void AddModule(string name) /// depends on. public void AddDependency(string dependingModule, string dependentModule) { - if (String.IsNullOrEmpty(dependingModule)) + if (string.IsNullOrEmpty(dependingModule)) + { throw new ArgumentNullException(nameof(dependingModule)); + } - if (String.IsNullOrEmpty(dependentModule)) + if (string.IsNullOrEmpty(dependentModule)) + { throw new ArgumentNullException(nameof(dependentModule)); + } - if (!knownModules.Contains(dependingModule)) + if (!_knownModules.Contains(dependingModule)) + { throw new ArgumentException($"Cannot add dependency for unknown module {dependingModule}"); + } AddToDependencyMatrix(dependentModule); - dependencyMatrix.Add(dependentModule, dependingModule); + _dependencyMatrix.Add(dependentModule, dependingModule); + + if (_boostedModules.Contains(dependingModule)) + { + var index = _boostedModules.IndexOf(dependingModule); + _boostedModules.Insert(index, dependentModule); + + _boostedDependencyMatrix.Add(dependentModule, dependingModule); + } } private void AddToDependencyMatrix(string module) { - if (!dependencyMatrix.ContainsKey(module)) + if (!_dependencyMatrix.ContainsKey(module)) { - dependencyMatrix.Add(module); + _dependencyMatrix.Add(module); } } private void AddToKnownModules(string module) { - if (!knownModules.Contains(module)) + if (!_knownModules.Contains(module)) { - knownModules.Add(module); + _knownModules.Add(module); + } + } + + private void AddToBoostedDependencyMatrix(string module) + { + if (!_boostedDependencyMatrix.ContainsKey(module)) + { + _boostedDependencyMatrix.Add(module); } } @@ -72,14 +109,14 @@ private void AddToKnownModules(string module) /// /// The resulting ordered list of modules. /// This exception is thrown - /// when a cycle is found in the defined depedency graph. + /// when a cycle is found in the defined dependency graph. public string[] Solve() { - List skip = new List(); - while (skip.Count < dependencyMatrix.Count) + var skip = new List(); + while (skip.Count < _dependencyMatrix.Count) { - List leaves = this.FindLeaves(skip); - if (leaves.Count == 0 && skip.Count < dependencyMatrix.Count) + var leaves = FindLeaves(skip, _dependencyMatrix); + if (leaves.Count == 0 && skip.Count < _dependencyMatrix.Count) { throw new CyclicDependencyFoundException($"At least one cyclic dependency has been found in the module catalog. Cycles in the module dependencies must be avoided."); } @@ -87,11 +124,20 @@ public string[] Solve() } skip.Reverse(); - if (skip.Count > knownModules.Count) + if (_boostedDependencyMatrix.Count > 0) { - var missedDependencies = skip.Except(knownModules).ToList(); + var boostedModules = GetBoostedSortedModules(); + + // Remove boosted modules and add them to the start of the list + skip.RemoveAll(boostedModules.Contains); + skip = boostedModules.Concat(skip).ToList(); + } + + if (skip.Count > _knownModules.Count) + { + var missedDependencies = skip.Except(_knownModules).ToList(); // Create missed module matrix (key: missed module, value: module that miss it) and reverse it (keys to values, values to keys; key: module that miss other module, value: missed module) - var missedDependenciesMatrix = missedDependencies.ToDictionary(md => md, md => dependencyMatrix[md]) + var missedDependenciesMatrix = missedDependencies.ToDictionary(md => md, md => _dependencyMatrix[md]) .SelectMany(p => p.Value.Select(m => new KeyValuePair(m, p.Key))) .GroupBy(p => p.Key) .ToDictionary(g => g.Key, g => g.Select(p => p.Value)); @@ -101,28 +147,40 @@ public string[] Solve() return skip.ToArray(); } + private List GetBoostedSortedModules() + { + var result = new List(); + while (result.Count < _boostedDependencyMatrix.Count) + { + var leaves = FindLeaves(result, _boostedDependencyMatrix); + result.AddRange(leaves); + } + result.Reverse(); + return result; + } + /// /// Gets the number of modules added to the solver. /// /// The number of modules. public int ModuleCount { - get { return dependencyMatrix.Count; } + get { return _dependencyMatrix.Count; } } - private List FindLeaves(List skip) + private static List FindLeaves(List skip, ListDictionary dependencies) { - List result = new List(); + var result = new List(); - foreach (string precedent in dependencyMatrix.Keys) + foreach (var precedent in dependencies.Keys) { if (skip.Contains(precedent)) { continue; } - int count = 0; - foreach (string dependent in dependencyMatrix[precedent]) + var count = 0; + foreach (var dependent in dependencies[precedent]) { if (skip.Contains(dependent)) { diff --git a/src/VirtoCommerce.Platform.Core/Modularity/ModuleSequenceBoostOptions.cs b/src/VirtoCommerce.Platform.Core/Modularity/ModuleSequenceBoostOptions.cs new file mode 100644 index 00000000000..13af1a9858a --- /dev/null +++ b/src/VirtoCommerce.Platform.Core/Modularity/ModuleSequenceBoostOptions.cs @@ -0,0 +1,6 @@ +namespace VirtoCommerce.Platform.Core.Modularity; + +public class ModuleSequenceBoostOptions +{ + public string[] ModuleSequenceBoost { get; set; } = []; +} diff --git a/src/VirtoCommerce.Platform.Modules/External/ExternalModuleCatalog.cs b/src/VirtoCommerce.Platform.Modules/External/ExternalModuleCatalog.cs index 5a742ad9e74..2c1d292467f 100644 --- a/src/VirtoCommerce.Platform.Modules/External/ExternalModuleCatalog.cs +++ b/src/VirtoCommerce.Platform.Modules/External/ExternalModuleCatalog.cs @@ -19,7 +19,13 @@ public class ExternalModuleCatalog : ModuleCatalog, IExternalModuleCatalog private static readonly object _lockObject = new object(); - public ExternalModuleCatalog(ILocalModuleCatalog otherCatalog, IExternalModulesClient externalClient, IOptions options, ILogger logger) + public ExternalModuleCatalog( + ILocalModuleCatalog otherCatalog, + IExternalModulesClient externalClient, + IOptions options, + ILogger logger, + IOptions boostOptions) + : base(boostOptions) { _externalClient = externalClient; _installedModules = otherCatalog.Modules; diff --git a/src/VirtoCommerce.Platform.Modules/Local/LocalStorageModuleCatalog.cs b/src/VirtoCommerce.Platform.Modules/Local/LocalStorageModuleCatalog.cs index 40d45f21af9..7b47b5997e2 100644 --- a/src/VirtoCommerce.Platform.Modules/Local/LocalStorageModuleCatalog.cs +++ b/src/VirtoCommerce.Platform.Modules/Local/LocalStorageModuleCatalog.cs @@ -27,7 +27,9 @@ public LocalStorageModuleCatalog( IOptions options, IInternalDistributedLockService distributedLockProvider, IFileCopyPolicy fileCopyPolicy, - ILogger logger) + ILogger logger, + IOptions boostOptions) + : base(boostOptions) { _options = options.Value; _probingPath = _options.ProbingPath is null ? null : Path.GetFullPath(_options.ProbingPath); diff --git a/src/VirtoCommerce.Platform.Web/Controllers/Api/ModulesController.cs b/src/VirtoCommerce.Platform.Web/Controllers/Api/ModulesController.cs index a93b0fdd32c..32317e12955 100644 --- a/src/VirtoCommerce.Platform.Web/Controllers/Api/ModulesController.cs +++ b/src/VirtoCommerce.Platform.Web/Controllers/Api/ModulesController.cs @@ -368,6 +368,26 @@ public ActionResult GetAutoInstallState() return Ok(state); } + [HttpGet] + [Route("loading-order")] + [Authorize(PlatformConstants.Security.Permissions.ModuleManage)] + public ActionResult GetModulesLoadingOrder() + { + EnsureModulesCatalogInitialized(); + + var modules = _externalModuleCatalog.Modules + .OfType() + .Where(x => x.IsInstalled) + .ToArray(); + + var loadingOrder = _externalModuleCatalog.CompleteListWithDependencies(modules) + .OfType() + .Select(x => x.Id) + .ToArray(); + + return Ok(loadingOrder); + } + [ApiExplorerSettings(IgnoreApi = true)] public void ModuleBackgroundJob(ModuleBackgroundJobOptions options, ModulePushNotification notification) { diff --git a/src/VirtoCommerce.Platform.Web/Startup.cs b/src/VirtoCommerce.Platform.Web/Startup.cs index c0ebf2a50c3..82ab2566dc7 100644 --- a/src/VirtoCommerce.Platform.Web/Startup.cs +++ b/src/VirtoCommerce.Platform.Web/Startup.cs @@ -471,6 +471,8 @@ public void ConfigureServices(IServiceCollection services) }) .ValidateDataAnnotations(); + services.AddOptions().Bind(Configuration.GetSection("VirtoCommerce")); + services.AddModules(mvcBuilder); services.AddOptions().Bind(Configuration.GetSection("ExternalModules")).ValidateDataAnnotations(); diff --git a/tests/VirtoCommerce.Platform.Tests/Modularity/ExternalModuleCatalogTests.cs b/tests/VirtoCommerce.Platform.Tests/Modularity/ExternalModuleCatalogTests.cs index c887e131455..39105bf4cc4 100644 --- a/tests/VirtoCommerce.Platform.Tests/Modularity/ExternalModuleCatalogTests.cs +++ b/tests/VirtoCommerce.Platform.Tests/Modularity/ExternalModuleCatalogTests.cs @@ -341,7 +341,8 @@ private static ExternalModuleCatalog CreateExternalModuleCatalog(ExternalModuleM var logger = new Mock>(); var options = Options.Create(new ExternalModuleCatalogOptions() { ModulesManifestUrl = new Uri("http://nowhere.mock"), IncludePrerelease = includePrerelease }); - var result = new ExternalModuleCatalog(localModulesCatalog.Object, client.Object, options, logger.Object); + var boostOptions = Options.Create(new ModuleSequenceBoostOptions()); + var result = new ExternalModuleCatalog(localModulesCatalog.Object, client.Object, options, logger.Object, boostOptions); return result; } diff --git a/tests/VirtoCommerce.Platform.Tests/Modularity/ModuleInstallerUnitTests.cs b/tests/VirtoCommerce.Platform.Tests/Modularity/ModuleInstallerUnitTests.cs index d0d08ae87db..75db019b929 100644 --- a/tests/VirtoCommerce.Platform.Tests/Modularity/ModuleInstallerUnitTests.cs +++ b/tests/VirtoCommerce.Platform.Tests/Modularity/ModuleInstallerUnitTests.cs @@ -104,8 +104,9 @@ private IExternalModuleCatalog GetExternalModuleCatalog(ModuleManifest[] install var externalModulesClientMock = new Mock(); var options = Options.Create(new Mock().Object); var loggerMock = new Mock>(); + var boostOptions = Options.Create(new ModuleSequenceBoostOptions()); - var externalModuleCatalog = new ExternalModuleCatalog(localCatalogModulesMock.Object, externalModulesClientMock.Object, options, loggerMock.Object); + var externalModuleCatalog = new ExternalModuleCatalog(localCatalogModulesMock.Object, externalModulesClientMock.Object, options, loggerMock.Object, boostOptions); foreach (var module in installedModules) { diff --git a/tests/VirtoCommerce.Platform.Tests/Modularity/ModulePlatformCompatibilityTests.cs b/tests/VirtoCommerce.Platform.Tests/Modularity/ModulePlatformCompatibilityTests.cs index 8dad7c339a7..918df5dd60c 100644 --- a/tests/VirtoCommerce.Platform.Tests/Modularity/ModulePlatformCompatibilityTests.cs +++ b/tests/VirtoCommerce.Platform.Tests/Modularity/ModulePlatformCompatibilityTests.cs @@ -22,12 +22,14 @@ public class ModulePlatformCompatibilityTests public void Module(string targetPlatformVersion, string runningPlatformVersion, bool violation) { var catalogOptionsMock = new Mock>(); + var boostOptions = Options.Create(new ModuleSequenceBoostOptions()); catalogOptionsMock.Setup(x => x.Value).Returns(new LocalStorageModuleCatalogOptions() { DiscoveryPath = string.Empty }); var catalog = new LocalStorageModuleCatalog( catalogOptionsMock.Object, new Mock().Object, new Mock().Object, - new Mock>().Object); + new Mock>().Object, + boostOptions); PlatformVersion.CurrentVersion = SemanticVersion.Parse(runningPlatformVersion); var module = new ManifestModuleInfo().LoadFromManifest(new ModuleManifest() { PlatformVersion = targetPlatformVersion, Id = "Fake", Version = "0.0.0" /*Does not matter (not used in test)*/ }); catalog.AddModule(module);