VCI-821: Check if module in symlink on update

src/VirtoCommerce.Build.Tests/PackageManagerTests.cs

@@ -1,6 +1,6 @@
 using Extensions;
-using Nuke.Common.IO;
 using PlatformTools;
+using PlatformTools.Modules;
 using VirtoCommerce.Build.PlatformTools;
 
 namespace VirtoCommerce.Build.Tests

src/VirtoCommerce.Build/PlatformTools/Build.InitPlatform.cs

@@ -1,6 +1,7 @@
 using Nuke.Common;
 using Nuke.Common.IO;
 using PlatformTools;
+using PlatformTools.Modules;
 using VirtoCommerce.Platform.Core.Common;
 
 namespace VirtoCommerce.Build

src/VirtoCommerce.Build/PlatformTools/Build.PackageManager.cs

@@ -11,9 +11,11 @@
 using Nuke.Common.Utilities;
 using Nuke.Common.Utilities.Collections;
 using PlatformTools;
-using PlatformTools.Azure;
-using PlatformTools.Github;
-using PlatformTools.Gitlab;
+using PlatformTools.Extensions;
+using PlatformTools.Modules;
+using PlatformTools.Modules.Azure;
+using PlatformTools.Modules.Github;
+using PlatformTools.Modules.Gitlab;
 using Serilog;
 using VirtoCommerce.Build.PlatformTools;
 using VirtoCommerce.Platform.Core.Common;
@@ -203,7 +205,7 @@ private static bool PlatformVersionChanged()
                     modulesDirs = Directory.EnumerateDirectories(discoveryPath).ToList();
                 }
                 var symlinks = modulesDirs.Where(m => new DirectoryInfo(m).LinkTarget != null).ToList();
-                CompressionExtensions.TarGZipTo(RootDirectory, BackupFile, filter: f => !f.ToFileInfo().FullName.StartsWith(RootDirectory / ".nuke") && !symlinks.Any(s => f.ToFileInfo().FullName.StartsWith(s)));
+                CompressionExtensions.TarGZipTo(RootDirectory, BackupFile, filter: f => !f.ToFileInfo().FullName.StartsWith(RootDirectory / ".nuke") && !symlinks.Exists(s => f.ToFileInfo().FullName.StartsWith(s)));
             });
 
         public Target Rollback => _ => _
@@ -322,6 +324,7 @@ private static bool IsPlatformInstallationNeeded(string version)
                  {
                      SkipDependencySolving = true;
                  }
+
                  var packageManifest = PackageManager.FromFile(PackageManifestPath);
                  var moduleSources = PackageManager.GetModuleSources(packageManifest).Where(s => s is not GithubReleases).ToList();
                  var githubReleases = PackageManager.GetGithubModulesSource(packageManifest);
@@ -343,7 +346,7 @@ private static bool IsPlatformInstallationNeeded(string version)
                          return;
                      }
 
-                     if (alreadyInstalledModules.Any(installedModule => installedModule.ModuleName == module.Id && installedModule.Version.ToString() == module.Version))
+                     if (alreadyInstalledModules.Exists(installedModule => installedModule.ModuleName == module.Id && installedModule.Version.ToString() == module.Version) || localModuleCatalog.IsModuleSymlinked(module.Id))
                      {
                          continue;
                      }
@@ -485,7 +488,7 @@ private static ManifestModuleInfo LoadModuleInfo(ModuleItem module, ManifestModu
              .Executes(async () =>
              {
                  SkipDependencySolving = true;
-                 var manifest = PackageManager.FromFile(PackageManifestPath);
+                 var manifest = await OpenOrCreateManifest(PackageManifestPath, Edge);
 
                  if (Edge)
                  {
@@ -607,7 +610,7 @@ private static async Task<ManifestBase> OpenOrCreateManifest(string packageManif
             else if (!File.Exists(packageManifestPath) && File.Exists(platformWebDllPath))
             {
                 var discoveryAbsolutePath = Path.GetFullPath(GetDiscoveryPath());
-                return CreateManifestFromEnvironment(RootDirectory, discoveryAbsolutePath.ToAbsolutePath());
+                return await CreateManifestFromEnvironment(RootDirectory, discoveryAbsolutePath.ToAbsolutePath());
             }
             else if (!File.Exists(packageManifestPath))
             {
@@ -638,10 +641,10 @@ private static async Task DownloadBundleManifest(string bundleName, string outFi
             }
 
             var manifestUrl = bundle.Value;
-            await HttpTasks.HttpDownloadFileAsync(manifestUrl, outFile);
+            await HttpTasks.HttpDownloadFileAsync(manifestUrl, outFile.ToAbsolutePath());
         }
 
-        private static ManifestBase CreateManifestFromEnvironment(AbsolutePath platformPath, AbsolutePath discoveryPath)
+        private async static Task<ManifestBase> CreateManifestFromEnvironment(AbsolutePath platformPath, AbsolutePath discoveryPath)
         {
             var platformWebDllPath = platformPath / "VirtoCommerce.Platform.Web.dll";
             if (!File.Exists(platformWebDllPath))
@@ -652,13 +655,25 @@ private static ManifestBase CreateManifestFromEnvironment(AbsolutePath platformP
             var platformVersion = platformWebDllFileInfo.ProductVersion;
             var packageManifest = PackageManager.CreatePackageManifest(platformVersion);
             var githubModules = PackageManager.GetGithubModules(packageManifest);
-
+            var githubModulesSource = PackageManager.GetGithubModulesSource(packageManifest);
+            var localModuleCatalog = (LocalCatalog)LocalModuleCatalog.GetCatalog(GetDiscoveryPath(), ProbingPath);
+            var externalModuleCatalog = await ExtModuleCatalog.GetCatalog(GitHubToken, localModuleCatalog, githubModulesSource.ModuleSources);
+            var modulesInCatalog = externalModuleCatalog.Modules.Where(m => !m.Ref.StartsWith("file://")).Select(m => m.ModuleName).ToList();
             var manifests = discoveryPath.GlobFiles("*/module.manifest");
             manifests.ForEach(m =>
             {
                 var manifest = ManifestReader.Read(m);
-                githubModules.Add(new ModuleItem(manifest.Id, manifest.Version));
+                if (!modulesInCatalog.Contains(manifest.Id))
+                {
+                    Log.Warning("There is no module {0}:{1} in external catalog. You should add it in manifest manually.", manifest.Id, manifest.Version);
+
+                }
+                else
+                {
+                    githubModules.Add(new ModuleItem(manifest.Id, manifest.Version));
+                }
             });
+
             return packageManifest;
         }
     }

src/VirtoCommerce.Build/PlatformTools/Extensions/LocalModuleCatalogExtensions.cs

@@ -0,0 +1,23 @@
+using System.IO;
+using System.Linq;
+using PlatformTools.Modules;
+using VirtoCommerce.Platform.Core.Modularity;
+
+namespace PlatformTools.Extensions
+{
+    public static class LocalModuleCatalogExtensions
+    {
+        public static bool IsModuleSymlinked(this ILocalModuleCatalog moduleCatalog, string moduleId)
+        {
+            var moduleInfo = moduleCatalog.Modules.OfType<ManifestModuleInfo>().FirstOrDefault(m => m.ModuleName == moduleId);
+            if(moduleInfo == null)
+            {
+                return false;
+            }
+
+            var fileInfo = new FileInfo(moduleInfo.FullPhysicalPath);
+
+            return fileInfo.LinkTarget != null;
+        }
+    }
+}

src/VirtoCommerce.Build/PlatformTools/GithubReleases.cs

@@ -1,5 +1,5 @@
 using System.Collections.Generic;
-using PlatformTools;
+using PlatformTools.Modules;
 
 namespace VirtoCommerce.Build.PlatformTools
 {

src/VirtoCommerce.Build/PlatformTools/MixedPackageManifest.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using PlatformTools.Modules;
 
 namespace VirtoCommerce.Build.PlatformTools
 {

src/VirtoCommerce.Build/PlatformTools/Azure/AzureBlob.cs → src/VirtoCommerce.Build/PlatformTools/Modules/Azure/AzureBlob.cs

@@ -1,7 +1,7 @@
 using System.Collections.Generic;
-using VirtoCommerce.Build.PlatformTools;
+using PlatformTools.Modules;
 
-namespace PlatformTools.Azure
+namespace PlatformTools.Modules.Azure
 {
     internal class AzureBlob : ModuleSource
     {

src/VirtoCommerce.Build/PlatformTools/Azure/AzureBlobModuleInstaller.cs → src/VirtoCommerce.Build/PlatformTools/Modules/Azure/AzureBlobModuleInstaller.cs

@@ -3,11 +3,11 @@
 using System.IO.Compression;
 using System.Linq;
 using System.Threading.Tasks;
-using VirtoCommerce.Build.PlatformTools;
+using PlatformTools.Modules;
 using VirtoCommerce.Platform.Core.Modularity;
 using AzureBlobs = Azure.Storage.Blobs;
 
-namespace PlatformTools.Azure
+namespace PlatformTools.Modules.Azure
 {
     internal class AzureBlobModuleInstaller : ModulesInstallerBase
     {
@@ -31,7 +31,7 @@ protected override Task InnerInstall(ModuleSource source, IProgress<ProgressMess
             {
                 progress.ReportInfo($"Installing {moduleBlobName}");
                 var zipName = moduleBlobName;
-                if(!zipName.EndsWith(".zip"))
+                if (!zipName.EndsWith(".zip"))
                 {
                     zipName += ".zip";
                 }

src/VirtoCommerce.Build/PlatformTools/Modules/Azure/AzureBlobModuleItem.cs

@@ -0,0 +1,13 @@
+using PlatformTools.Modules;
+
+namespace PlatformTools.Modules.Azure
+{
+    internal class AzureBlobModuleItem : ModuleItem
+    {
+        public AzureBlobModuleItem(string id, string version) : base(id, version)
+        {
+
+        }
+        public string BlobName { get; set; }
+    }
+}

src/VirtoCommerce.Build/PlatformTools/Azure/AzureCliToolSettings.cs → src/VirtoCommerce.Build/PlatformTools/Modules/Azure/AzureCliToolSettings.cs

@@ -1,7 +1,7 @@
 using System;
 using Nuke.Common.Tooling;
 
-namespace PlatformTools.Azure
+namespace PlatformTools.Modules.Azure
 {
     [Serializable]
     internal class AzureCliToolSettings : ToolSettings

src/VirtoCommerce.Build/PlatformTools/Azure/AzureDevClient.cs → src/VirtoCommerce.Build/PlatformTools/Modules/Azure/AzureDevClient.cs

@@ -1,40 +1,40 @@
-using System;
-using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.TeamFoundation.Build.WebApi;
-using Microsoft.VisualStudio.Services.Common;
-using Microsoft.VisualStudio.Services.WebApi;
-using Nuke.Common;
-
-namespace PlatformTools.Azure;
-
-public class AzureDevClient
-{
-    private readonly VssConnection _connection;
-
-    public AzureDevClient(string orgName, string token)
-    {
-        _connection = new VssConnection(new Uri(orgName), new VssBasicCredential(string.Empty, token));
-    }
-
-    public async Task<Uri> GetArtifactUrl(Guid project, string branch, string definitionName)
-    {
-        var client = await _connection.GetClientAsync<BuildHttpClient>();
-        var build = await client.GetLatestBuildAsync(project, definitionName, branch);
-        var artifacts = await client.GetArtifactsAsync(project, build.Id);
-        Assert.NotEmpty(artifacts,
-            $"No builds found for the project id: {project.ToString()}{Environment.NewLine}branch: {branch}{Environment.NewLine}definition: {definitionName}");
-        var result = artifacts.FirstOrDefault()?.Resource.DownloadUrl;
-
-        return new Uri(result ?? string.Empty);
-    }
-
-    public async Task<Stream> GetArtifactStream(Guid project, string branch, string definitionName)
-    {
-        var client = await _connection.GetClientAsync<BuildHttpClient>();
-        var build = await client.GetLatestBuildAsync(project, definitionName, branch);
-        var result = await client.GetArtifactContentZipAsync(project, build.Id, "backend");
-        return result;
-    }
-}
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.TeamFoundation.Build.WebApi;
+using Microsoft.VisualStudio.Services.Common;
+using Microsoft.VisualStudio.Services.WebApi;
+using Nuke.Common;
+
+namespace PlatformTools.Modules.Azure;
+
+public class AzureDevClient
+{
+    private readonly VssConnection _connection;
+
+    public AzureDevClient(string orgName, string token)
+    {
+        _connection = new VssConnection(new Uri(orgName), new VssBasicCredential(string.Empty, token));
+    }
+
+    public async Task<Uri> GetArtifactUrl(Guid project, string branch, string definitionName)
+    {
+        var client = await _connection.GetClientAsync<BuildHttpClient>();
+        var build = await client.GetLatestBuildAsync(project, definitionName, branch);
+        var artifacts = await client.GetArtifactsAsync(project, build.Id);
+        Assert.NotEmpty(artifacts,
+            $"No builds found for the project id: {project.ToString()}{Environment.NewLine}branch: {branch}{Environment.NewLine}definition: {definitionName}");
+        var result = artifacts.FirstOrDefault()?.Resource.DownloadUrl;
+
+        return new Uri(result ?? string.Empty);
+    }
+
+    public async Task<Stream> GetArtifactStream(Guid project, string branch, string definitionName)
+    {
+        var client = await _connection.GetClientAsync<BuildHttpClient>();
+        var build = await client.GetLatestBuildAsync(project, definitionName, branch);
+        var result = await client.GetArtifactContentZipAsync(project, build.Id, "backend");
+        return result;
+    }
+}

src/VirtoCommerce.Build/PlatformTools/Azure/AzureModuleItem.cs → src/VirtoCommerce.Build/PlatformTools/Modules/Azure/AzureModuleItem.cs

@@ -1,14 +1,16 @@
-namespace PlatformTools.Azure
-{
-    internal class AzureModuleItem : ModuleItem
-    {
-        public AzureModuleItem(string id, string version) : base(id, version)
-        {
-        }
-
-        public string BuildId { get; set; }
-        public string Branch { get; set; }
-        public string Definition { get; set; }
-    }
-}
-
+using PlatformTools.Modules;
+
+namespace PlatformTools.Modules.Azure
+{
+    internal class AzureModuleItem : ModuleItem
+    {
+        public AzureModuleItem(string id, string version) : base(id, version)
+        {
+        }
+
+        public string BuildId { get; set; }
+        public string Branch { get; set; }
+        public string Definition { get; set; }
+    }
+}
+

src/VirtoCommerce.Build/PlatformTools/Azure/AzurePipelineArtifacts.cs → src/VirtoCommerce.Build/PlatformTools/Modules/Azure/AzurePipelineArtifacts.cs

@@ -1,14 +1,14 @@
-using System.Collections.Generic;
-using VirtoCommerce.Build.PlatformTools;
-
-namespace PlatformTools.Azure
-{
-    internal class AzurePipelineArtifacts : ModuleSource
-    {
-        public string Organization { get; set; }
-        public string Project { get; set; }
-        public List<AzureModuleItem> Modules { get; set; }
-        public override string Name { get; set; } = nameof(AzurePipelineArtifacts);
-    }
-}
-
+using System.Collections.Generic;
+using PlatformTools.Modules;
+
+namespace PlatformTools.Modules.Azure
+{
+    internal class AzurePipelineArtifacts : ModuleSource
+    {
+        public string Organization { get; set; }
+        public string Project { get; set; }
+        public List<AzureModuleItem> Modules { get; set; }
+        public override string Name { get; set; } = nameof(AzurePipelineArtifacts);
+    }
+}
+