VCST-1736: add Module Sequence Boost (#2837)

Co-authored-by: artem-dudarev <[email protected]>

src/VirtoCommerce.Platform.Core/Modularity/ModuleCatalog.cs

@@ -3,6 +3,7 @@
 using System.Collections.ObjectModel;
 using System.Collections.Specialized;
 using System.Linq;
+using Microsoft.Extensions.Options;
 using VirtoCommerce.Platform.Core.Common;
 using VirtoCommerce.Platform.Core.Modularity.Exceptions;
 
@@ -27,31 +28,35 @@ namespace VirtoCommerce.Platform.Core.Modularity
     /// </summary>
     public class ModuleCatalog : IModuleCatalog
     {
+        private readonly ModuleSequenceBoostOptions _boostOptions;
         private readonly ModuleCatalogItemCollection items;
         private bool isLoaded;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ModuleCatalog"/> class.
         /// </summary>
-        public ModuleCatalog()
+        public ModuleCatalog(IOptions<ModuleSequenceBoostOptions> boostOptions)
         {
-            this.items = new ModuleCatalogItemCollection();
-            this.items.CollectionChanged += this.ItemsCollectionChanged;
+            _boostOptions = boostOptions.Value;
+
+            items = new ModuleCatalogItemCollection();
+            items.CollectionChanged += ItemsCollectionChanged;
         }
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ModuleCatalog"/> class while providing an 
         /// initial list of <see cref="ModuleInfo"/>s.
         /// </summary>
         /// <param name="modules">The initial list of modules.</param>
-        public ModuleCatalog(IEnumerable<ModuleInfo> modules)
-            : this()
+        /// <param name="boostOptions">Module boost options</param>
+        public ModuleCatalog(IEnumerable<ModuleInfo> modules, IOptions<ModuleSequenceBoostOptions> boostOptions)
+            : this(boostOptions)
         {
-            if (modules == null)
-                throw new System.ArgumentNullException("modules");
-            foreach (ModuleInfo moduleInfo in modules)
+            ArgumentNullException.ThrowIfNull(modules);
+
+            foreach (var moduleInfo in modules)
             {
-                this.Items.Add(moduleInfo);
+                Items.Add(moduleInfo);
             }
         }
 
@@ -331,14 +336,11 @@ public virtual ModuleCatalog AddGroup(InitializationMode initializationMode, str
         /// </summary>
         /// <param name="modules">the.</param>
         /// <returns></returns>
-        protected static string[] SolveDependencies(IEnumerable<ModuleInfo> modules)
+        protected string[] SolveDependencies(IEnumerable<ModuleInfo> modules)
         {
-            if (modules == null)
-            {
-                throw new System.ArgumentNullException("modules");
-            }
+            ArgumentNullException.ThrowIfNull(modules);
 
-            var solver = new ModuleDependencySolver();
+            var solver = new ModuleDependencySolver(_boostOptions);
 
             foreach (var data in modules.ToArray())
             {
@@ -363,7 +365,7 @@ protected static string[] SolveDependencies(IEnumerable<ModuleInfo> modules)
                 return solver.Solve();
             }
 
-            return new string[0];
+            return [];
         }
 
         /// <summary>

src/VirtoCommerce.Platform.Core/Modularity/ModuleDependencySolver.cs

@@ -12,20 +12,35 @@ namespace VirtoCommerce.Platform.Core.Modularity
     /// </summary>
     public class ModuleDependencySolver
     {
-        private readonly ListDictionary<string, string> dependencyMatrix = new ListDictionary<string, string>();
-        private readonly List<string> knownModules = new List<string>();
+        private readonly ListDictionary<string, string> _dependencyMatrix = [];
+        private readonly List<string> _knownModules = [];
+
+        private readonly List<string> _boostedModules;
+        private readonly ListDictionary<string, string> _boostedDependencyMatrix = [];
+
+        public ModuleDependencySolver(ModuleSequenceBoostOptions boostOptions)
+        {
+            _boostedModules = boostOptions.ModuleSequenceBoost.ToList();
+        }
 
         /// <summary>
         /// Adds a module to the solver.
         /// </summary>
         /// <param name="name">The name that uniquely identifies the module.</param>
         public void AddModule(string name)
         {
-            if (String.IsNullOrEmpty(name))
+            if (string.IsNullOrEmpty(name))
+            {
                 throw new ArgumentNullException(nameof(name));
+            }
 
             AddToDependencyMatrix(name);
             AddToKnownModules(name);
+
+            if (_boostedModules.Contains(name))
+            {
+                AddToBoostedDependencyMatrix(name);
+            }
         }
 
         /// <summary>
@@ -37,32 +52,54 @@ public void AddModule(string name)
         /// depends on.</param>
         public void AddDependency(string dependingModule, string dependentModule)
         {
-            if (String.IsNullOrEmpty(dependingModule))
+            if (string.IsNullOrEmpty(dependingModule))
+            {
                 throw new ArgumentNullException(nameof(dependingModule));
+            }
 
-            if (String.IsNullOrEmpty(dependentModule))
+            if (string.IsNullOrEmpty(dependentModule))
+            {
                 throw new ArgumentNullException(nameof(dependentModule));
+            }
 
-            if (!knownModules.Contains(dependingModule))
+            if (!_knownModules.Contains(dependingModule))
+            {
                 throw new ArgumentException($"Cannot add dependency for unknown module {dependingModule}");
+            }
 
             AddToDependencyMatrix(dependentModule);
-            dependencyMatrix.Add(dependentModule, dependingModule);
+            _dependencyMatrix.Add(dependentModule, dependingModule);
+
+            if (_boostedModules.Contains(dependingModule))
+            {
+                var index = _boostedModules.IndexOf(dependingModule);
+                _boostedModules.Insert(index, dependentModule);
+
+                _boostedDependencyMatrix.Add(dependentModule, dependingModule);
+            }
         }
 
         private void AddToDependencyMatrix(string module)
         {
-            if (!dependencyMatrix.ContainsKey(module))
+            if (!_dependencyMatrix.ContainsKey(module))
             {
-                dependencyMatrix.Add(module);
+                _dependencyMatrix.Add(module);
             }
         }
 
         private void AddToKnownModules(string module)
         {
-            if (!knownModules.Contains(module))
+            if (!_knownModules.Contains(module))
             {
-                knownModules.Add(module);
+                _knownModules.Add(module);
+            }
+        }
+
+        private void AddToBoostedDependencyMatrix(string module)
+        {
+            if (!_boostedDependencyMatrix.ContainsKey(module))
+            {
+                _boostedDependencyMatrix.Add(module);
             }
         }
 
@@ -72,26 +109,35 @@ private void AddToKnownModules(string module)
         /// </summary>
         /// <returns>The resulting ordered list of modules.</returns>
         /// <exception cref="CyclicDependencyFoundException">This exception is thrown
-        /// when a cycle is found in the defined depedency graph.</exception>
+        /// when a cycle is found in the defined dependency graph.</exception>
         public string[] Solve()
         {
-            List<string> skip = new List<string>();
-            while (skip.Count < dependencyMatrix.Count)
+            var skip = new List<string>();
+            while (skip.Count < _dependencyMatrix.Count)
             {
-                List<string> leaves = this.FindLeaves(skip);
-                if (leaves.Count == 0 && skip.Count < dependencyMatrix.Count)
+                var leaves = FindLeaves(skip, _dependencyMatrix);
+                if (leaves.Count == 0 && skip.Count < _dependencyMatrix.Count)
                 {
                     throw new CyclicDependencyFoundException($"At least one cyclic dependency has been found in the module catalog. Cycles in the module dependencies must be avoided.");
                 }
                 skip.AddRange(leaves);
             }
             skip.Reverse();
 
-            if (skip.Count > knownModules.Count)
+            if (_boostedDependencyMatrix.Count > 0)
             {
-                var missedDependencies = skip.Except(knownModules).ToList();
+                var boostedModules = GetBoostedSortedModules();
+
+                // Remove boosted modules and add them to the start of the list
+                skip.RemoveAll(boostedModules.Contains);
+                skip = boostedModules.Concat(skip).ToList();
+            }
+
+            if (skip.Count > _knownModules.Count)
+            {
+                var missedDependencies = skip.Except(_knownModules).ToList();
                 // Create missed module matrix (key: missed module, value: module that miss it) and reverse it (keys to values, values to keys; key: module that miss other module, value: missed module)
-                var missedDependenciesMatrix = missedDependencies.ToDictionary(md => md, md => dependencyMatrix[md])
+                var missedDependenciesMatrix = missedDependencies.ToDictionary(md => md, md => _dependencyMatrix[md])
                     .SelectMany(p => p.Value.Select(m => new KeyValuePair<string, string>(m, p.Key)))
                     .GroupBy(p => p.Key)
                     .ToDictionary(g => g.Key, g => g.Select(p => p.Value));
@@ -101,28 +147,40 @@ public string[] Solve()
             return skip.ToArray();
         }
 
+        private List<string> GetBoostedSortedModules()
+        {
+            var result = new List<string>();
+            while (result.Count < _boostedDependencyMatrix.Count)
+            {
+                var leaves = FindLeaves(result, _boostedDependencyMatrix);
+                result.AddRange(leaves);
+            }
+            result.Reverse();
+            return result;
+        }
+
         /// <summary>
         /// Gets the number of modules added to the solver.
         /// </summary>
         /// <value>The number of modules.</value>
         public int ModuleCount
         {
-            get { return dependencyMatrix.Count; }
+            get { return _dependencyMatrix.Count; }
         }
 
-        private List<string> FindLeaves(List<string> skip)
+        private static List<string> FindLeaves(List<string> skip, ListDictionary<string, string> dependencies)
         {
-            List<string> result = new List<string>();
+            var result = new List<string>();
 
-            foreach (string precedent in dependencyMatrix.Keys)
+            foreach (var precedent in dependencies.Keys)
             {
                 if (skip.Contains(precedent))
                 {
                     continue;
                 }
 
-                int count = 0;
-                foreach (string dependent in dependencyMatrix[precedent])
+                var count = 0;
+                foreach (var dependent in dependencies[precedent])
                 {
                     if (skip.Contains(dependent))
                     {

src/VirtoCommerce.Platform.Core/Modularity/ModuleSequenceBoostOptions.cs

@@ -0,0 +1,6 @@
+namespace VirtoCommerce.Platform.Core.Modularity;
+
+public class ModuleSequenceBoostOptions
+{
+    public string[] ModuleSequenceBoost { get; set; } = [];
+}

src/VirtoCommerce.Platform.Modules/External/ExternalModuleCatalog.cs

@@ -19,7 +19,13 @@ public class ExternalModuleCatalog : ModuleCatalog, IExternalModuleCatalog
 
         private static readonly object _lockObject = new object();
 
-        public ExternalModuleCatalog(ILocalModuleCatalog otherCatalog, IExternalModulesClient externalClient, IOptions<ExternalModuleCatalogOptions> options, ILogger<ExternalModuleCatalog> logger)
+        public ExternalModuleCatalog(
+            ILocalModuleCatalog otherCatalog,
+            IExternalModulesClient externalClient,
+            IOptions<ExternalModuleCatalogOptions> options,
+            ILogger<ExternalModuleCatalog> logger,
+            IOptions<ModuleSequenceBoostOptions> boostOptions)
+            : base(boostOptions)
         {
             _externalClient = externalClient;
             _installedModules = otherCatalog.Modules;

src/VirtoCommerce.Platform.Modules/Local/LocalStorageModuleCatalog.cs

@@ -27,7 +27,9 @@ public LocalStorageModuleCatalog(
             IOptions<LocalStorageModuleCatalogOptions> options,
             IInternalDistributedLockService distributedLockProvider,
             IFileCopyPolicy fileCopyPolicy,
-            ILogger<LocalStorageModuleCatalog> logger)
+            ILogger<LocalStorageModuleCatalog> logger,
+            IOptions<ModuleSequenceBoostOptions> boostOptions)
+            : base(boostOptions)
         {
             _options = options.Value;
             _probingPath = _options.ProbingPath is null ? null : Path.GetFullPath(_options.ProbingPath);

src/VirtoCommerce.Platform.Web/Controllers/Api/ModulesController.cs

@@ -368,6 +368,26 @@ public ActionResult<string> GetAutoInstallState()
             return Ok(state);
         }
 
+        [HttpGet]
+        [Route("loading-order")]
+        [Authorize(PlatformConstants.Security.Permissions.ModuleManage)]
+        public ActionResult<string[]> GetModulesLoadingOrder()
+        {
+            EnsureModulesCatalogInitialized();
+
+            var modules = _externalModuleCatalog.Modules
+                .OfType<ManifestModuleInfo>()
+                .Where(x => x.IsInstalled)
+                .ToArray();
+
+            var loadingOrder = _externalModuleCatalog.CompleteListWithDependencies(modules)
+                .OfType<ManifestModuleInfo>()
+                .Select(x => x.Id)
+                .ToArray();
+
+            return Ok(loadingOrder);
+        }
+
         [ApiExplorerSettings(IgnoreApi = true)]
         public void ModuleBackgroundJob(ModuleBackgroundJobOptions options, ModulePushNotification notification)
         {

src/VirtoCommerce.Platform.Web/Startup.cs

@@ -471,6 +471,8 @@ public void ConfigureServices(IServiceCollection services)
                     })
                     .ValidateDataAnnotations();
 
+            services.AddOptions<ModuleSequenceBoostOptions>().Bind(Configuration.GetSection("VirtoCommerce"));
+
             services.AddModules(mvcBuilder);
 
             services.AddOptions<ExternalModuleCatalogOptions>().Bind(Configuration.GetSection("ExternalModules")).ValidateDataAnnotations();

tests/VirtoCommerce.Platform.Tests/Modularity/ExternalModuleCatalogTests.cs

@@ -341,7 +341,8 @@ private static ExternalModuleCatalog CreateExternalModuleCatalog(ExternalModuleM
             var logger = new Mock<ILogger<ExternalModuleCatalog>>();
 
             var options = Options.Create(new ExternalModuleCatalogOptions() { ModulesManifestUrl = new Uri("http://nowhere.mock"), IncludePrerelease = includePrerelease });
-            var result = new ExternalModuleCatalog(localModulesCatalog.Object, client.Object, options, logger.Object);
+            var boostOptions = Options.Create(new ModuleSequenceBoostOptions());
+            var result = new ExternalModuleCatalog(localModulesCatalog.Object, client.Object, options, logger.Object, boostOptions);
             return result;
         }
 

tests/VirtoCommerce.Platform.Tests/Modularity/ModuleInstallerUnitTests.cs

@@ -104,8 +104,9 @@ private IExternalModuleCatalog GetExternalModuleCatalog(ModuleManifest[] install
             var externalModulesClientMock = new Mock<IExternalModulesClient>();
             var options = Options.Create(new Mock<ExternalModuleCatalogOptions>().Object);
             var loggerMock = new Mock<ILogger<ExternalModuleCatalog>>();
+            var boostOptions = Options.Create(new ModuleSequenceBoostOptions());
 
-            var externalModuleCatalog = new ExternalModuleCatalog(localCatalogModulesMock.Object, externalModulesClientMock.Object, options, loggerMock.Object);
+            var externalModuleCatalog = new ExternalModuleCatalog(localCatalogModulesMock.Object, externalModulesClientMock.Object, options, loggerMock.Object, boostOptions);
 
             foreach (var module in installedModules)
             {

tests/VirtoCommerce.Platform.Tests/Modularity/ModulePlatformCompatibilityTests.cs

@@ -22,12 +22,14 @@ public class ModulePlatformCompatibilityTests
         public void Module(string targetPlatformVersion, string runningPlatformVersion, bool violation)
         {
             var catalogOptionsMock = new Mock<IOptions<LocalStorageModuleCatalogOptions>>();
+            var boostOptions = Options.Create(new ModuleSequenceBoostOptions());
             catalogOptionsMock.Setup(x => x.Value).Returns(new LocalStorageModuleCatalogOptions() { DiscoveryPath = string.Empty });
             var catalog = new LocalStorageModuleCatalog(
                 catalogOptionsMock.Object,
                 new Mock<IInternalDistributedLockService>().Object,
                 new Mock<IFileCopyPolicy>().Object,
-                new Mock<ILogger<LocalStorageModuleCatalog>>().Object);
+                new Mock<ILogger<LocalStorageModuleCatalog>>().Object,
+                boostOptions);
             PlatformVersion.CurrentVersion = SemanticVersion.Parse(runningPlatformVersion);
             var module = new ManifestModuleInfo().LoadFromManifest(new ModuleManifest() { PlatformVersion = targetPlatformVersion, Id = "Fake", Version = "0.0.0" /*Does not matter (not used in test)*/ });
             catalog.AddModule(module);