Skip to content

Commit

Permalink
Merge branch 'main' into fix-loading-packages
Browse files Browse the repository at this point in the history
Signed-off-by: geffzhang <[email protected]>
  • Loading branch information
geffzhang authored Sep 14, 2024
2 parents 5802cf8 + 087ca20 commit db533b6
Show file tree
Hide file tree
Showing 17 changed files with 568 additions and 35 deletions.
82 changes: 77 additions & 5 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,10 @@ jobs:
needs: [version, verify]
name: Release Helm chart to GitHub
runs-on: ubuntu-latest
env:
CHART_DIR: deployment templates/chart
CHART_REPO: bagetter/helm-chart-repo
CHART_REPO_BRANCH: master
steps:
- uses: actions/[email protected]
- name: Update version in helm chart
Expand All @@ -139,8 +143,76 @@ jobs:
uses: losisin/helm-docs-github-action@v1
with:
chart-search-root: "deployment templates"
- name: Commit changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: "Release helm chart version ${{ needs.version.outputs.RELEASE_VERSION }}"
file_pattern: 'deployment\ templates/chart/bagetter/Chart.yaml deployment\ templates/chart/bagetter/README.md'

- name: Install yq - portable yaml processor
uses: mikefarah/[email protected]

- name: Collect charts
id: charts
run: |
set -e
find -L '${{ env.CHART_DIR }}' -mindepth 2 -maxdepth 2 -type f \( -name 'Chart.yaml' -o -name 'Chart.yml' \) -exec dirname "{}" \; \
| sort -u \
| sed -E 's/^/- /' \
| yq --no-colors --indent 0 --output-format json '.' \
| sed -E 's/^/charts=/' >> $GITHUB_OUTPUT
- name: Install chart releaser
run: |
set -e
arch="$(dpkg --print-architecture)"
curl -s https://api.github.com/repos/helm/chart-releaser/releases/latest \
| yq --indent 0 --no-colors --input-format json --unwrapScalar \
".assets[] | select(.name | test("\""^chart-releaser_.+_linux_${arch}\.tar\.gz$"\"")) | .browser_download_url" \
| xargs curl -SsL \
| tar zxf - -C /usr/local/bin
- name: Install Helm
uses: azure/[email protected]

- name: Helm Deps
run: |
set -ex
echo '${{ steps.charts.outputs.charts }}' \
| yq --indent 0 --no-colors --input-format json --unwrapScalar '.[]' \
| while read -r dir; do
helm dependency update $dir;
if [ -f "$dir/Chart.lock" ]; then
yq --indent 0 \
'.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' \
"$dir/Chart.lock" \
| sh --;
fi
done
- name: Package charts
id: package
run: |
set -ex
PACKAGES=.cr-release-packages
echo '${{ steps.charts.outputs.charts }}' \
| yq --indent 0 --no-colors --input-format json --unwrapScalar '.[]' \
| xargs -d$'\n' cr package --package-path "$PACKAGES"
echo "dir=${PACKAGES}" >> $GITHUB_OUTPUT
- name: Upload packages
run: |
set -ex
git config --list
owner=$(cut -d '/' -f 1 <<< '${{ env.CHART_REPO }}')
repo=$(cut -d '/' -f 2 <<< '${{ env.CHART_REPO }}')
cr upload --commit '${{ github.sha }}' --git-repo "$repo" --owner "$owner" --token '${{ github.token }}' \
--package-path '${{ steps.package.outputs.dir }}' --skip-existing
- name: Update charts index
working-directory: .helm-chart-repo
run: |
set -ex
git config --local user.name "$GITHUB_ACTOR"
git config --local user.email "[email protected]"
mkdir -p .cr-index
owner=$(cut -d '/' -f 1 <<< '${{ env.CHART_REPO }}')
repo=$(cut -d '/' -f 2 <<< '${{ env.CHART_REPO }}')
cr index --git-repo "$repo" --owner "$owner" --pages-branch '${{ env.CHART_REPO_BRANCH }}' \
--package-path '../${{ steps.package.outputs.dir }}' \
--index-path .cr-index --push
49 changes: 24 additions & 25 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -3,49 +3,48 @@
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="Azure.Data.Tables" Version="12.8.3" />
<PackageVersion Include="Azure.Search.Documents" Version="11.5.1" />
<PackageVersion Include="Azure.Storage.Blobs" Version="12.19.1" />
<PackageVersion Include="Azure.Storage.Common" Version="12.18.1" />
<PackageVersion Include="Azure.Data.Tables" Version="12.9.0" />
<PackageVersion Include="Azure.Search.Documents" Version="11.6.0" />
<PackageVersion Include="Azure.Storage.Blobs" Version="12.21.2" />
<PackageVersion Include="Azure.Storage.Common" Version="12.20.1" />
<PackageVersion Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="8.0.3" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="8.0.8" />
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.38.1" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.3" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="8.0.3" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.8" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="8.0.8" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.3" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" Version="8.0.3" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.8" />
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" Version="8.0.8" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
<PackageVersion Include="NuGet.Frameworks" Version="6.11.0" />
<PackageVersion Include="NuGet.Protocol" Version="6.9.1" />
<PackageVersion Include="NuGet.Protocol" Version="6.11.0" />
<PackageVersion Include="NuGet.Versioning" Version="6.11.0" />
<PackageVersion Include="System.Reflection.Metadata" Version="8.0.0" />
<PackageVersion Include="System.Text.Json" Version="8.0.4" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.0" />
<PackageVersion Include="Moq" Version="4.20.70" />
<PackageVersion Include="xunit" Version="2.7.0" />
<PackageVersion Include="xunit.runner.visualstudio" Version="2.5.7" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.3" />
<PackageVersion Include="xunit" Version="2.9.0" />
<PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.8" />
<PackageVersion Include="coverlet.collector" Version="6.0.2" />
<PackageVersion Include="Humanizer" Version="2.14.1" />
<PackageVersion Include="Markdig" Version="0.36.2" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.3" />
<PackageVersion Include="Microsoft.Data.SqlClient" Version="5.2.0" />
<PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.2" />
<PackageVersion Include="Markdig" Version="0.37.0" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.8" />
<PackageVersion Include="Microsoft.Data.SqlClient" Version="5.2.2" />
<PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
<PackageVersion Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.2" />
<PackageVersion Include="Aliyun.OSS.SDK.NetCore" Version="2.13.0" />
<PackageVersion Include="AWSSDK.S3" Version="3.7.307" />
<PackageVersion Include="AWSSDK.SecurityToken" Version="3.7.300.60" />
<PackageVersion Include="Aliyun.OSS.SDK.NetCore" Version="2.14.1" />
<PackageVersion Include="AWSSDK.S3" Version="3.7.402.3" />
<PackageVersion Include="AWSSDK.SecurityToken" Version="3.7.400.12" />
<PackageVersion Include="Microsoft.Azure.Cosmos.Table" Version="1.0.8" />
<PackageVersion Include="Microsoft.Azure.Search" Version="10.1.0" />
<PackageVersion Include="Microsoft.Azure.Storage.Blob" Version="11.2.3" />
<PackageVersion Include="Google.Cloud.Storage.V1" Version="4.9.0" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.3" />
<PackageVersion Include="Tencent.QCloud.Cos.Sdk" Version="5.4.40" />

<PackageVersion Include="Google.Cloud.Storage.V1" Version="4.10.0" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
<PackageVersion Include="Tencent.QCloud.Cos.Sdk" Version="5.4.40" />
</ItemGroup>
</Project>
14 changes: 14 additions & 0 deletions docs/docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,20 @@ downloaded if you know the package's id and version. You can override this behav
}
```

## Enable package auto-deletion

If your build server generates many nuget packages, your BaGet server can quickly run out of space. To avoid this issue, `MaxVersionsPerPackage` can be configured to auto-delete packages older packages when a new one is uploaded. This will use the `HardDelete` option detailed above and will unlist and delete the files for the older packages. By default this value is not configured and no packages will be deleted automatically.

```json
{
...

"MaxVersionsPerPackage ": 5,

...
}
```

## Enable package overwrites

Normally, BaGetter will reject a package upload if the id and version are already taken. This is to maintain the [immutability of semantically versioned packages](https://learn.microsoft.com/azure/devops/artifacts/artifacts-key-concepts?view=azure-devops#immutability).
Expand Down
2 changes: 1 addition & 1 deletion global.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"sdk": {
"version": "8.0.100",
"version": "8.0.401",
"rollForward": "latestFeature",
"allowPrerelease": false
}
Expand Down
6 changes: 6 additions & 0 deletions src/BaGetter.Core/Configuration/BaGetterOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ public class BaGetterOptions
/// </summary>
public uint MaxPackageSizeGiB { get; set; } = 8;

/// <summary>
/// If this is set to a value, it will limit the number of versions that can be pushed for a package.
/// the older versions will be deleted.
/// </summary>
public uint? MaxVersionsPerPackage { get; set; } = null;

public DatabaseOptions Database { get; set; }

public StorageOptions Storage { get; set; }
Expand Down
9 changes: 9 additions & 0 deletions src/BaGetter.Core/Indexing/IPackageDeletionService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@ namespace BaGetter.Core;

public interface IPackageDeletionService
{
/// <summary>
/// Delete old versions of packages
/// </summary>
/// <param name="package">Current package object to clean</param>
/// <param name="maxPackagesToKeep">Maximum number of packages to keep</param>
/// <param name="cancellationToken"></param>
/// <returns>Number of packages deleted</returns>
Task<int> DeleteOldVersionsAsync(Package package, uint maxPackagesToKeep, CancellationToken cancellationToken);

/// <summary>
/// Attempt to delete a package.
/// </summary>
Expand Down
19 changes: 19 additions & 0 deletions src/BaGetter.Core/Indexing/PackageDeletionService.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
Expand Down Expand Up @@ -88,4 +89,22 @@ private async Task<bool> TryHardDeletePackageAsync(string id, NuGetVersion versi

return found;
}

public async Task<int> DeleteOldVersionsAsync(Package package, uint maxPackages, CancellationToken cancellationToken)
{
// list all versions of the package
var versions = await _packages.FindAsync(package.Id, includeUnlisted: true, cancellationToken);
if (versions is null || versions.Count <= maxPackages) return 0;
// sort by version and take everything except the last maxPackages
var versionsToDelete = versions
.OrderByDescending(p => p.Version)
.Skip((int)maxPackages)
.ToList();
var deleted = 0;
foreach (var version in versionsToDelete)
{
if (await TryHardDeletePackageAsync(package.Id, version.Version, cancellationToken)) deleted++;
}
return deleted;
}
}
30 changes: 30 additions & 0 deletions src/BaGetter.Core/Indexing/PackageIndexingService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@ public class PackageIndexingService : IPackageIndexingService
private readonly SystemTime _time;
private readonly IOptionsSnapshot<BaGetterOptions> _options;
private readonly ILogger<PackageIndexingService> _logger;
private readonly IPackageDeletionService _packageDeletionService;

public PackageIndexingService(
IPackageDatabase packages,
IPackageStorageService storage,
IPackageDeletionService packageDeletionService,
ISearchIndexer search,
SystemTime time,
IOptionsSnapshot<BaGetterOptions> options,
Expand All @@ -31,6 +33,7 @@ public PackageIndexingService(
_time = time ?? throw new ArgumentNullException(nameof(time));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_packageDeletionService = packageDeletionService ?? throw new ArgumentNullException(nameof(packageDeletionService));
}

public async Task<PackageIndexingResult> IndexAsync(Stream packageStream, CancellationToken cancellationToken)
Expand Down Expand Up @@ -153,6 +156,33 @@ await _storage.SavePackageContentAsync(

await _search.IndexAsync(package, cancellationToken);

if (_options.Value.MaxVersionsPerPackage.HasValue)
{
try {
_logger.LogInformation(
"Deleting older packages for package {PackageId} {PackageVersion}",
package.Id,
package.NormalizedVersionString);
var deleted = await _packageDeletionService.DeleteOldVersionsAsync(package, _options.Value.MaxVersionsPerPackage.Value, cancellationToken);
if (deleted > 0)
{
_logger.LogInformation(
"Deleted {packages} older packages for package {PackageId} {PackageVersion}",
deleted,
package.Id,
package.NormalizedVersionString);
}
}
catch (Exception e)
{
_logger.LogError(
e,
"Failed to cleanup older versions of package {PackageId} {PackageVersion}",
package.Id,
package.NormalizedVersionString);
}
}

_logger.LogInformation(
"Successfully indexed package {Id} {Version} in search",
package.Id,
Expand Down
3 changes: 1 addition & 2 deletions src/BaGetter.Tencent/BucketClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ public BucketClient(CosXmlServer cosXml, string fullBucketName, string appId, st
_region = region;
}


public bool UploadStream(string key, Stream stream)
{
var result = false;
Expand Down Expand Up @@ -69,7 +68,7 @@ public byte[] DownloadFileBytes(string key)
throw new Exception(serverEx.Message);
}
return Array.Empty<byte>();
}
}

public void DeleteDir(string dir)
{
Expand Down
8 changes: 6 additions & 2 deletions src/BaGetter.Tencent/TencentStorageService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ public TencentStorageService(TencentCosClient cosClient, IOptionsSnapshot<Tencen
public async Task DeleteAsync(string path, CancellationToken cancellationToken = default)

Check warning on line 18 in src/BaGetter.Tencent/TencentStorageService.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 18 in src/BaGetter.Tencent/TencentStorageService.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.
{
ArgumentException.ThrowIfNullOrEmpty(path);

var found = _cosClient.BucketClient.DoesObjectExist(PrepareKey(path)) ? PrepareKey(path) : string.Empty;
if (string.IsNullOrEmpty(found))
{
Expand All @@ -30,7 +29,9 @@ public async Task DeleteAsync(string path, CancellationToken cancellationToken =
public async Task<Stream> GetAsync(string path, CancellationToken cancellationToken = default)

Check warning on line 29 in src/BaGetter.Tencent/TencentStorageService.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 29 in src/BaGetter.Tencent/TencentStorageService.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.
{
ArgumentException.ThrowIfNullOrEmpty(path);

var fileStorageUrl = _cosClient.BucketClient.DoesObjectExist(PrepareKey(path)) ? PrepareKey(path) : string.Empty;

if (string.IsNullOrEmpty(fileStorageUrl))
{
throw new KeyNotFoundException($"The file {path} was not found.");
Expand All @@ -42,7 +43,9 @@ public async Task<Stream> GetAsync(string path, CancellationToken cancellationTo
public async Task<Uri> GetDownloadUriAsync(string path, CancellationToken cancellationToken = default)

Check warning on line 43 in src/BaGetter.Tencent/TencentStorageService.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 43 in src/BaGetter.Tencent/TencentStorageService.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.
{
ArgumentException.ThrowIfNullOrEmpty(path);

var fileStorageUrl = _cosClient.BucketClient.DoesObjectExist(PrepareKey(path)) ? PrepareKey(path) : string.Empty;

if (string.IsNullOrEmpty(fileStorageUrl))
{
throw new KeyNotFoundException($"The file {path} was not found.");
Expand All @@ -60,7 +63,8 @@ public async Task<StoragePutResult> PutAsync(string path, Stream content, string
seekableContent.Seek(0, SeekOrigin.Begin);

var fileRet = _cosClient.BucketClient.UploadStream(PrepareKey(path), seekableContent);
if (!fileRet)

if(!fileRet)
{
return StoragePutResult.AlreadyExists;
}
Expand Down
7 changes: 7 additions & 0 deletions tests/BaGetter.Core.Tests/BaGetter.Core.Tests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,11 @@
<PackageReference Include="Microsoft.Extensions.Configuration" />
</ItemGroup>

<ItemGroup>
<PackageReference Update="xunit.runner.visualstudio">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
</ItemGroup>

</Project>
Loading

0 comments on commit db533b6

Please sign in to comment.