Support AWS Cloud Front

JetBrains · Feb 16, 2021 · 4d3e028 · 4d3e028
1 parent f552ad7
commit 4d3e028
Show file tree

Hide file tree

Showing 10 changed files with 27 additions and 23 deletions.
diff --git a/Common.targets b/Common.targets
@@ -3,6 +3,6 @@
     <TargetFramework>net5.0</TargetFramework>
     <SignAssembly>true</SignAssembly>
     <AssemblyOriginatorKeyFile>$(MSBuildThisFileDirectory)\Subplatform.Snk</AssemblyOriginatorKeyFile>
-    <Version>1.0.1</Version>
+    <Version>1.0.2</Version>
   </PropertyGroup>
 </Project>
diff --git a/Core/src/Impl/Commands/CreateCommand.cs b/Core/src/Impl/Commands/CreateCommand.cs
@@ -74,7 +74,6 @@ public async Task<int> Execute()
             await WriteDataPacked(Path.Combine(srcDir, srcFile), dstFile, (len, stream) => myStorage.CreateForWriting(dstFile, AccessMode.Public, len, stream));
             dstFiles.Add(dstFile);
           }).Execute();
-      await myStorage.InvalidateExternalServices(dstFiles);
       myLogger.Info($"[{DateTime.Now:s}] Done with data (warnings: {statistics.Warnings}, errors: {statistics.Errors})");
       if (statistics.HasProblems)
       {
@@ -83,6 +82,7 @@ public async Task<int> Execute()
       }
 
       await WriteTag(dstFiles.Select(Path.GetDirectoryName).Distinct());
+      await myStorage.InvalidateExternalServices();
       return 0;
     }
 
@@ -103,7 +103,6 @@ private async Task WriteTag([NotNull] IEnumerable<string> dirs)
 
       var tagFile = Path.Combine(TagUtil.TagDirectory, myProduct, myProduct + '-' + myVersion + '-' + fileId.ToString("N") + TagUtil.TagExtension);
       await myStorage.CreateForWriting(tagFile, AccessMode.Private, stream.Length, stream.Rewind());
-      await myStorage.InvalidateExternalServices(new[] {tagFile});
     }
 
     private static async Task WriteData(

diff --git a/Core/src/Impl/Commands/DeleteCommand.cs b/Core/src/Impl/Commands/DeleteCommand.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using JetBrains.Annotations;
 using JetBrains.SymbolStorage.Impl.Logger;
@@ -63,6 +64,8 @@ public async Task<int> Execute()
       {
         var (_, files) = await validator.GatherDataFiles();
         var (statistics, deleted) = await validator.Validate(tagItems, files, storageFormat, Validator.ValidateMode.Delete);
+        if (deleted > 0)
+          await myStorage.InvalidateExternalServices();
         myLogger.Info($"[{DateTime.Now:s}] Done (deleted tag files: {deleteTags}, deleted data files: {deleted}, warnings: {statistics.Warnings}, errors: {statistics.Errors}, fixes: {statistics.Fixes})");
         return statistics.HasProblems ? 1 : 0;
       }

diff --git a/Core/src/Impl/Commands/UploadCommand.cs b/Core/src/Impl/Commands/UploadCommand.cs
@@ -114,7 +114,7 @@ public async Task<int> Execute()
             Interlocked.Add(ref totalSize, memoryStream.Length);
           }
 
-          await myStorage.InvalidateExternalServices(uploadFiles.Select(x => x.Item2).ToList());
+          await myStorage.InvalidateExternalServices();
           myLogger.Info($"[{DateTime.Now:s}] Done with uploading (size: {totalSize.ToKibibyte()}, files: {uploadFiles.Count})");
         }
       }

diff --git a/Core/src/Impl/Commands/ValidateCommand.cs b/Core/src/Impl/Commands/ValidateCommand.cs
@@ -34,6 +34,8 @@ public async Task<int> Execute()
       validator.DumpProperties(tagItems);
       var (totalSize, files) = await validator.GatherDataFiles();
       var (statistics, _) = await validator.Validate(tagItems, files, storageFormat, myFix ? Validator.ValidateMode.Fix : Validator.ValidateMode.Validate, myVerifyAcl);
+      if (statistics.Fixes > 0)
+        await myStorage.InvalidateExternalServices();
       myLogger.Info($"[{DateTime.Now:s}] Done (size: {totalSize.ToKibibyte()}, warnings: {statistics.Warnings}, errors: {statistics.Errors}, fixes: {statistics.Fixes})");
       return statistics.HasProblems ? 1 : 0;
     }

diff --git a/Core/src/Impl/Commands/Validator.cs b/Core/src/Impl/Commands/Validator.cs
@@ -55,7 +55,6 @@ public async Task CreateStorageMarkers(StorageFormat newStorageFormat)
 
       foreach (var file in files)
         await myStorage.CreateEmpty(file, AccessMode.Private);
-      await myStorage.InvalidateExternalServices(files);
     }
 
     public async Task<StorageFormat> ValidateStorageMarkers()
@@ -228,7 +227,6 @@ public async Task<Tuple<Statistics, long>> Validate(
             await using var stream = new MemoryStream();
             TagUtil.WriteTagScript(tag, stream);
             await myStorage.CreateForWriting(tagFile, AccessMode.Private, stream.Length, stream.Rewind());
-            await myStorage.InvalidateExternalServices(new[] {tagFile});
           }
         }
       }

diff --git a/Core/src/Impl/Storages/AwsS3Storage.cs b/Core/src/Impl/Storages/AwsS3Storage.cs
@@ -231,20 +231,21 @@ public async IAsyncEnumerable<ChildrenItem> GetChildren(ChildrenMode mode, strin
 
     public async Task InvalidateExternalServices(IEnumerable<string> keys)
     {
-      if (keys == null)
-        throw new ArgumentNullException(nameof(keys));
       if (!string.IsNullOrEmpty(myCloudFrontDistributionId))
       {
-        var items = keys.Select(x => x.NormalizeLinux().StartsWith("/") ? x : '/' + x).ToList();
-        await myCloudFrontClient.CreateInvalidationAsync(new CreateInvalidationRequest
-          {
-            DistributionId = myCloudFrontDistributionId,
-            InvalidationBatch = new InvalidationBatch(new Paths
-              {
-                Items = items,
-                Quantity = items.Count
-              }, $"symbol-storage-{DateTime.UtcNow:s}")
-          });
+        var items = keys != null 
+          ? keys.Select(x => '/' + x.NormalizeLinux()).ToList()
+          : new List<string> {"/*"};
+        if (items.Count > 0)
+          await myCloudFrontClient.CreateInvalidationAsync(new CreateInvalidationRequest
+            {
+              DistributionId = myCloudFrontDistributionId,
+              InvalidationBatch = new InvalidationBatch(new Paths
+                {
+                  Items = items,
+                  Quantity = items.Count
+                }, $"symbol-storage-{DateTime.UtcNow:s}")
+            });
       }
     }
 

diff --git a/Core/src/Impl/Storages/FileSystemStorage.cs b/Core/src/Impl/Storages/FileSystemStorage.cs
@@ -179,8 +179,6 @@ public async IAsyncEnumerable<ChildrenItem> GetChildren(ChildrenMode mode, [NotN
 
     public Task InvalidateExternalServices(IEnumerable<string> keys)
     {
-      if (keys == null)
-        throw new ArgumentNullException(nameof(keys));
       return Task.CompletedTask;
     }
 

diff --git a/Core/src/Impl/Storages/IStorage.cs b/Core/src/Impl/Storages/IStorage.cs
@@ -25,6 +25,6 @@ internal interface IStorage
 
     IAsyncEnumerable<ChildrenItem> GetChildren(ChildrenMode mode, string prefixDir = null);
 
-    Task InvalidateExternalServices([NotNull] IEnumerable<string> keys);
+    Task InvalidateExternalServices([CanBeNull] IEnumerable<string> keys = null);
   }
 }
diff --git a/Core/src/MainUtil.cs b/Core/src/MainUtil.cs
@@ -123,6 +123,9 @@ public static int Main(Assembly mainAssembly, [NotNull] string[] args, MainMode
             var sourcesOption = x.Argument("path [path [...]]", "Source directories or files with symbols, executables and shared libraries.", true);
             x.OnExecute(async () =>
               {
+                var storage = AccessUtil.GetStorage(dirOption.Value(), awsS3BucketNameOption.Value());
+                var newStorageFormat = AccessUtil.GetStorageFormat(newStorageFormatOption.Value());
+
                 var tempDir = Path.Combine(Path.GetTempPath(), "storage_" + Guid.NewGuid().ToString("D"));
                 try
                 {
@@ -143,9 +146,9 @@ public static int Main(Assembly mainAssembly, [NotNull] string[] args, MainMode
 
                   return await new UploadCommand(
                     ConsoleLogger.Instance,
-                    AccessUtil.GetStorage(dirOption.Value(), awsS3BucketNameOption.Value()),
+                    storage,
                     tempDir,
-                    AccessUtil.GetStorageFormat(newStorageFormatOption.Value())).Execute();
+                    newStorageFormat).Execute();
                 }
                 finally
                 {