Smarter cleaning rules for packages with SemVer

docs/docs/configuration.md

@@ -175,16 +175,27 @@ downloaded if you know the package's id and version. You can override this behav
 }
 ```
 
-## Enable package auto-deletion
+## Package auto-deletion
 
-If your build server generates many nuget packages, your BaGet server can quickly run out of space. To avoid this issue, `MaxVersionsPerPackage` can be configured to auto-delete packages older packages when a new one is uploaded. This will use the `HardDelete` option detailed above and will unlist and delete the files for the older packages. By default this value is not configured and no packages will be deleted automatically.
+If your build server generates many nuget packages, your BaGet server can quickly run out of space. Bagetter leverages [SemVer 2](https://semver.org/) and has logic to keep a history of packages based on the version numbering such as `<major>.<minor>.<patch>-<prerelease tag>.<prerelease build number>`.
+
+There is an optional section for `Retention` and the following parameters can be enabled to limit history for each level of the version. If none of these are set, there are no cleaning rules enforced. Each parameter is optional, e.g. if you specify only a `MaxHistoryPerPatch`, the package limit will only enforced for each major and minor version combination.
+Packages deleted are always the oldest based on version numbers.
+
+- MaxHistoryPerMajorVersion: Maximum number of major versions
+- MaxHistoryPerMinorVersion: Maximum number of minor versions for each major version
+- MaxHistoryPerPatch: Maximum number of patch versions for each major + minor version
+- MaxHistoryPerPrerelease: Maximum number of prerelease versions for each major + minor + patch version and prerelease type. if you have `beta` and `alpha` this will keep `MaxHistoryPerPrerelease` versions for both `beta` and `alpha`.
 
 ```json
 {
     ...
-
-    "MaxVersionsPerPackage ": 5,
-
+    "Retention": {
+        "MaxHistoryPerMajorVersion": 5,
+        "MaxHistoryPerMinorVersion": 5,
+        "MaxHistoryPerPatch": 5,
+        "MaxHistoryPerPrerelease": 5,
+    }
     ...
 }
 ```

src/BaGetter.Core/Configuration/BaGetterOptions.cs

@@ -6,7 +6,7 @@
 {
     /// <summary>
     /// The API Key required to authenticate package
     /// operations. If <see cref="ApiKeys"/> and  <see cref="ApiKey"/> are not set, package operations do not require authentication.
     /// </summary>
     public string ApiKey { get; set; }
 
@@ -49,11 +49,7 @@
     /// </summary>
     public uint MaxPackageSizeGiB { get; set; } = 8;
 
-    /// <summary>
-    /// If this is set to a value, it will limit the number of versions that can be pushed for a package.
-    /// the older versions will be deleted.
-    /// </summary>
-    public uint? MaxVersionsPerPackage { get; set; } = null;
+    public RetentionOptions Retention { get; set; }
 
     public DatabaseOptions Database { get; set; }
 

src/BaGetter.Core/Configuration/RetentionOptions.cs

@@ -0,0 +1,33 @@
+namespace BaGetter.Core;
+
+public class RetentionOptions
+{
+    /// <summary>
+    /// If this is set to a value, it will limit the number of versions that can be pushed for a package.
+    /// The limit is applied to each major version of the package, and if the limit is exceeded,
+    /// the older versions will be deleted.
+    /// </summary>
+    public uint? MaxHistoryPerMajorVersion { get; set; } = null;
+
+    /// <summary>
+    /// This corresponds to the maximum number of minor versions for each major version.
+    /// If this is set to a value, it will limit the number of versions that can be pushed for a package.
+    /// The limit is applied to each minor version of the package, and if the limit is exceeded,
+    /// the older versions will be deleted.
+    /// </summary>
+    public uint? MaxHistoryPerMinorVersion { get; set; }
+
+    /// <summary>
+    /// If this is set to a value, it will limit the number of versions that can be pushed for a package.
+    /// The limit is applied to each patch number of the package, and if the limit is exceeded,
+    /// the older versions will be deleted.
+    /// </summary>
+    public uint? MaxHistoryPerPatch { get; set; }
+
+    /// <summary>
+    /// If this is set to a value, it will limit the number of versions that can be pushed for a package.
+    /// The limit is applied to each pre-release of the package, and if the limit is exceeded,
+    /// the older versions will be deleted.
+    /// </summary>
+    public uint? MaxHistoryPerPrerelease { get; set; }
+}

src/BaGetter.Core/Entities/Package.cs

@@ -1,10 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using NuGet.Versioning;
 
 namespace BaGetter.Core;
 
 // See NuGetGallery's: https://github.com/NuGet/NuGetGallery/blob/master/src/NuGetGallery.Core/Entities/Package.cs
+[DebuggerDisplay("{Id} {Version}")]
 public class Package
 {
     public int Key { get; set; }

src/BaGetter.Core/Extensions/DependencyInjectionExtensions.cs

@@ -68,6 +68,7 @@ private static void AddConfiguration(this IServiceCollection services)
         services.AddBaGetterOptions<DatabaseOptions>(nameof(BaGetterOptions.Database));
         services.AddBaGetterOptions<FileSystemStorageOptions>(nameof(BaGetterOptions.Storage));
         services.AddBaGetterOptions<MirrorOptions>(nameof(BaGetterOptions.Mirror));
+        services.AddBaGetterOptions<RetentionOptions>(nameof(BaGetterOptions.Retention));
         services.AddBaGetterOptions<SearchOptions>(nameof(BaGetterOptions.Search));
         services.AddBaGetterOptions<StorageOptions>(nameof(BaGetterOptions.Storage));
         services.AddBaGetterOptions<StatisticsOptions>(nameof(BaGetterOptions.Statistics));

src/BaGetter.Core/Indexing/IPackageDeletionService.cs

@@ -7,13 +7,19 @@
 public interface IPackageDeletionService
 {
     /// <summary>
-    /// Delete old versions of packages
+    /// This method deletes old versions of a package.
+    /// This leverages semver 2.0 - and assume a package is major.minor.patch-prerelease.build
+    /// It can leverage the <see cref="IPackageDatabase"/> to list all versions of a package and then delete all but the last <paramref name="maxMajor"/> versions.
+    /// It also takes into account the <paramref name="maxMinor"/>, <paramref name="maxPath"/> and <paramref name="maxPrerelease"/> parameters to further filter the versions to delete.
     /// </summary>
-    /// <param name="package">Current package object to clean</param>
-    /// <param name="maxPackagesToKeep">Maximum number of packages to keep</param>
-    /// <param name="cancellationToken"></param>
+    /// <param name="package">Package name</param>
+    /// <param name="maxMajor">Maximum of major versions to keep (optional)</param>
+    /// <param name="maxMinor">Maximum of minor versions to keep (optional)</param>
+    /// <param name="maxPatch">Maximum of patch versions to keep (optional)</param>
+    /// <param name="maxPrerelease">Maximum of pre-release versions (optional)</param>
+    /// <param name="cancellationToken">Cancel the operation</param>
     /// <returns>Number of packages deleted</returns>
-    Task<int> DeleteOldVersionsAsync(Package package, uint maxPackagesToKeep, CancellationToken cancellationToken);
+    Task<int> DeleteOldVersionsAsync(Package package, uint? maxMajor, uint? maxMinor, uint? maxPatch, uint? maxPrerelease, CancellationToken cancellationToken);
 
     /// <summary>
     /// Attempt to delete a package.

src/BaGetter.Core/Indexing/PackageDeletionService.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -90,21 +91,105 @@
         return found;
     }
 
-    public async Task<int> DeleteOldVersionsAsync(Package package, uint maxPackages, CancellationToken cancellationToken)
+    private static IList<NuGetVersion> GetValidVersions<S, T>(IEnumerable<NuGetVersion> versions, Func<NuGetVersion, S> getParent, Func<NuGetVersion,T> getSelector, int versionsToKeep)
+            where S : IComparable<S>, IEquatable<S>
+            where T : IComparable<T>, IEquatable<T>
+    {
+        var validVersions = versions
+            // for each parent group
+            .GroupBy(v => getParent(v))
+            // get all versions by selector
+            .SelectMany(g => g.Select(k => (parent: g.Key, selector: getSelector(k)))
+                .Distinct()
+                .OrderByDescending(k => k.selector)
+                .Take(versionsToKeep))            
+            .ToList();
+        return versions.Where(k => validVersions.Any(v => getParent(k).Equals(v.parent) && getSelector(k).Equals(v.selector))).ToList();
+    }
+
+    public async Task<int> DeleteOldVersionsAsync(Package package, uint? maxMajor, uint? maxMinor, uint? maxPatch, uint? maxPrerelease, CancellationToken cancellationToken)
     {
         // list all versions of the package
-        var versions = await _packages.FindAsync(package.Id, includeUnlisted: true, cancellationToken);
-        if (versions is null || versions.Count <= maxPackages) return 0;
+        var packages = await _packages.FindAsync(package.Id, includeUnlisted: true, cancellationToken);
+        if (packages is null || packages.Count <= maxMajor) return 0;
+
+        var goodVersions = new HashSet<NuGetVersion>();
+
+        if (maxMajor.HasValue)
+        {
+            goodVersions = GetValidVersions(packages.Select(t => t.Version), v => 0, v => v.Major, (int)maxMajor).ToHashSet();
+        }
+        else
+        {
+            goodVersions = packages.Select(p => p.Version).ToHashSet();
+        }
+
+        if (maxMinor.HasValue)
+        {
+            goodVersions.IntersectWith(GetValidVersions(goodVersions, v => (v.Major), v => v.Minor, (int)maxMinor));
+        }
+
+        if (maxPatch.HasValue)
+        {
+            goodVersions.IntersectWith(GetValidVersions(goodVersions, v => (v.Major, v.Minor), v => v.Patch, (int)maxPatch));
+        }
+
+        if (maxPrerelease.HasValue)
+        {
+            // this assume we have something like 1.1.1-alpha.1 - alpha is the release type
+            var preReleases = packages.Select(p => p.Version).Where(p => p.IsPrerelease).ToList();
+            // this will give us 'alpha' or 'beta' etc
+            var prereleaseTypes = preReleases
+                .Select(v => v.ReleaseLabels?.FirstOrDefault())
+                .Where(lb => lb is not null)
+                .Distinct();
+
+            var allPreReleaseValidVersions = new HashSet<NuGetVersion>();
+            foreach (var preReleaseType in prereleaseTypes)
+            {
+                var preReleaseVersions = preReleases.Where(p => p.ReleaseLabels!.FirstOrDefault() == preReleaseType
+                        && GetPreReleaseBuild(p) is not null).ToList();
+
+                allPreReleaseValidVersions.UnionWith
+                    (GetValidVersions(preReleaseVersions,
+                        v => (v.Major, v.Minor, v.Patch), v => GetPreReleaseBuild(v).Value, (int)maxPrerelease));
+
+            }
+            goodVersions.IntersectWith(allPreReleaseValidVersions);
+        }
+
         // sort by version and take everything except the last maxPackages
-        var versionsToDelete = versions
-            .OrderByDescending(p => p.Version)
-            .Skip((int)maxPackages)
-            .ToList();
+        var versionsToDelete = packages.Where(p => !goodVersions.Contains(p.Version)).ToList();
+
         var deleted = 0;
         foreach (var version in versionsToDelete)
         {
             if (await TryHardDeletePackageAsync(package.Id, version.Version, cancellationToken)) deleted++;
         }
         return deleted;
     }
+
+    /// <summary>
+    /// Tries to get the version number of a pre-release build.<br/>
+    /// If we have 1.1.1-alpha.1 , this will return 1 or <c>null</c> if not valid.
+    /// </summary>
+    /// <returns>The version as <c>int</c> or <c>null</c> if not found.</returns>
+    private int? GetPreReleaseBuild(NuGetVersion nuGetVersion)
+    {
+        if (nuGetVersion.IsPrerelease && nuGetVersion.ReleaseLabels != null)
+        {
+            // Assuming the last part of the release label is the build number
+            var lastLabel = nuGetVersion.ReleaseLabels.LastOrDefault();
+            if (int.TryParse(lastLabel, out var buildNumber))
+            {
+                return buildNumber;
+            }
+            else
+            {
+                _logger.LogWarning("Could not parse build number from prerelease label {PrereleaseLabel} - prerelease number is expected to be like 2.3.4-alpha.1 where 1 is prerelease", nuGetVersion);
+            }
+        }
+        return null;
+    }
+
 }

src/BaGetter.Core/Indexing/PackageIndexingService.cs

@@ -15,6 +15,7 @@ public class PackageIndexingService : IPackageIndexingService
     private readonly ISearchIndexer _search;
     private readonly SystemTime _time;
     private readonly IOptionsSnapshot<BaGetterOptions> _options;
+    private readonly IOptionsSnapshot<RetentionOptions> _retentionOptions;
     private readonly ILogger<PackageIndexingService> _logger;
     private readonly IPackageDeletionService _packageDeletionService;
 
@@ -25,13 +26,15 @@ public PackageIndexingService(
         ISearchIndexer search,
         SystemTime time,
         IOptionsSnapshot<BaGetterOptions> options,
+        IOptionsSnapshot<RetentionOptions> retentionOptions,
         ILogger<PackageIndexingService> logger)
     {
         _packages = packages ?? throw new ArgumentNullException(nameof(packages));
         _storage = storage ?? throw new ArgumentNullException(nameof(storage));
         _search = search ?? throw new ArgumentNullException(nameof(search));
         _time = time ?? throw new ArgumentNullException(nameof(time));
         _options = options ?? throw new ArgumentNullException(nameof(options));
+        _retentionOptions = retentionOptions ?? throw new ArgumentNullException(nameof(retentionOptions));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         _packageDeletionService = packageDeletionService ?? throw new ArgumentNullException(nameof(packageDeletionService));
     }
@@ -156,14 +159,20 @@ await _storage.SavePackageContentAsync(
 
         await _search.IndexAsync(package, cancellationToken);
 
-        if (_options.Value.MaxVersionsPerPackage.HasValue)
+        if (_retentionOptions.Value.MaxHistoryPerMajorVersion.HasValue)
         {
             try { 
                 _logger.LogInformation(
                     "Deleting older packages for package {PackageId} {PackageVersion}",
                     package.Id,
                     package.NormalizedVersionString);
-                var deleted = await _packageDeletionService.DeleteOldVersionsAsync(package, _options.Value.MaxVersionsPerPackage.Value, cancellationToken);
+                var deleted = await _packageDeletionService.DeleteOldVersionsAsync(
+                    package,
+                    _retentionOptions.Value.MaxHistoryPerMajorVersion,
+                    _retentionOptions.Value.MaxHistoryPerMinorVersion,
+                    _retentionOptions.Value.MaxHistoryPerPatch,
+                    _retentionOptions.Value.MaxHistoryPerPrerelease,
+                    cancellationToken);
                 if (deleted > 0)
                 {
                     _logger.LogInformation(