Merge branch 'aptly-dev:master' into master

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: aptly-dev

.github/workflows/ci.yml

@@ -46,12 +46,6 @@ jobs:
            make version
            go generate
 
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          version: v1.54.2
-          args: --timeout=5m
-
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
@@ -77,7 +71,7 @@ jobs:
       - name: Make
         env:
           RUN_LONG_TESTS: 'yes'
-          AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
           AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
           AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
         run: |
@@ -95,19 +89,61 @@ jobs:
           files: coverage.txt
 
   release:
-    name: release
     needs: build
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        name: [Ubuntu-20, Ubuntu-22, Debian-10, Debian-11, Debian-12]
+        include:
+          - name: Ubuntu-20
+            image: "ubuntu:20.04"
+            suite: focal
+            GOVER: '1.21'
+            install: "make ca-certificates git curl"
+          - name: Ubuntu-22
+            image: "ubuntu:22.04"
+            suite: jammy
+            install: "make ca-certificates git curl golang"
+          - name: Debian-10
+            image: "debian:buster"
+            suite: buster
+            GOVER: '1.21'
+            install: "make ca-certificates git curl"
+          - name: Debian-11
+            image: "debian:bullseye"
+            suite: bullseye
+            GOVER: '1.21'
+            install: "make ca-certificates git curl"
+          - name: Debian-12
+            image: "debian:bookworm"
+            suite: bookworm
+            install: "make ca-certificates git curl golang"
+    container:
+      image: ${{ matrix.image }}
+      env:
+        APT_LISTCHANGES_FRONTEND: none
+        DEBIAN_FRONTEND: noninteractive
     steps:
+      - name: Install O/S packages
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends ${{ matrix.install }}
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          echo GOVER: ${{ env.GOVER }}
+
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
-          # fetch the whole repot for `git describe` to
+          # fetch the whole repo for `git describe` to
           # work and get the nightly verion
           fetch-depth: 0
 
       - name: Setup Go
-        uses: actions/setup-go@v4
+        if: ${{ matrix.GOVER != '' }}
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.GOVER }}
 
       - name: Make Release
         env:
@@ -121,19 +157,12 @@ jobs:
           APTLY_USER: ${{ secrets.APTLY_USER }}
           APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
         run: |
-          ./upload-artifacts.sh nightly
+          ./upload-artifacts.sh nightly ${{ matrix.suite }}
 
       - name: Publish release to aptly
         if: startsWith(github.event.ref, 'refs/tags')
         env:
           APTLY_USER: ${{ secrets.APTLY_USER }}
           APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
         run: |
-          ./upload-artifacts.sh release
-
-      - name: Upload artifacts to GitHub Release
-        if: startsWith(github.event.ref, 'refs/tags')
-        uses: softprops/action-gh-release@v1
-        with:
-          body: Release ${{ github.ref }} generated by the CI.
-          files: build/*
+          ./upload-artifacts.sh release ${{ matrix.suite }}

.github/workflows/golangci-lint.yml

@@ -0,0 +1,58 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+          cache: false
+      - name: Create VERSION file
+        run: |
+          make -s version | tr -d '\n' > VERSION
+        shell: sh
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v4
+        with:
+          # Require: The version of golangci-lint to use.
+          # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
+          # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
+          version: v1.56.2
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          #
+          # Note: By default, the `.golangci.yml` file should be at the root of the repository.
+          # The location of the configuration file can be changed by using `--config=`
+          # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true, then all caching functionality will be completely disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true
+
+          # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
+          # install-mode: "goinstall"

AUTHORS

@@ -58,3 +58,6 @@ List of contributors, in chronological order:
 * Ryan Gonzalez (https://github.com/refi64)
 * Paul Cacheux (https://github.com/paulcacheux)
 * Nic Waller (https://github.com/sf-nwaller)
+* iofq (https://github.com/iofq)
+* Noa Resare (https://github.com/nresare)
+* Ramón N.Rodriguez (https://github.com/runitonmetal)

CONTRIBUTING.md

@@ -74,7 +74,7 @@ This section describes local setup to start contributing to aptly source.
 
 ### Go & Python
 
-You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+You would need `Go` (latest version is recommended) and `Python` 3.9 (or newer, the CI currently tests against 3.11).
 
 If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
 initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.

Makefile

@@ -7,14 +7,20 @@ PYTHON?=python3
 TESTS?=
 BINPATH?=$(GOPATH)/bin
 RUN_LONG_TESTS?=yes
+GOLANGCI_LINT_VERSION=v1.56.2
 COVERAGE_DIR?=$(shell mktemp -d)
 # Uncomment to update test outputs
 # CAPTURE := "--capture"
 
 all: modules test bench check system-test
 
+# Self-documenting Makefile
+# https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+help:  ## Print this help
+	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
 prepare:
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.2
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin $(GOLANGCI_LINT_VERSION)
 
 modules:
 	go mod download
@@ -28,13 +34,14 @@ dev:
 
 check: system/env
 ifeq ($(RUN_LONG_TESTS), yes)
-	golangci-lint run
 	system/env/bin/flake8
 endif
 
 install:
+	@echo Building aptly ...
 	go generate
-	go install -v
+	@echo go install -v
+	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
 system/env: system/requirements.txt
 ifeq ($(RUN_LONG_TESTS), yes)
@@ -49,9 +56,18 @@ ifeq ($(RUN_LONG_TESTS), yes)
 	go test -v -coverpkg="./..." -c -tags testruncli
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE)
+	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE)
 endif
 
+docker-test: install
+	@echo Building aptly.test ...
+	@rm -f aptly.test
+	go test -v -coverpkg="./..." -c -tags testruncli
+	@echo Running python tests ...
+	export PATH=$(BINPATH)/:$(PATH); \
+	export APTLY_VERSION=$(VERSION); \
+	$(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
+
 test:
 	go test -v ./... -gocheck.v=true -coverprofile=unit.out
 
@@ -78,16 +94,25 @@ release: goxc
 	mkdir -p build/
 	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
 
-man:
+man:  ## Create man pages
 	make -C man
 
-version:
+version:  ## Print aptly version
 	@echo $(VERSION)
 
-docker-build:
-	docker build -f system/Dockerfile --no-cache . -t aptly-system-test
+docker-build-system-tests:  ## Build system-test docker image
+	docker build -f system/Dockerfile . -t aptly-system-test
+
+docker-unit-tests:  ## Run unit tests in docker container
+	docker run -it --rm -v ${PWD}:/app aptly-system-test go test -v ./... -gocheck.v=true
+
+docker-system-tests:  ## Run system tests in docker container (add TEST=t04_mirror to run only specific tests)
+	docker run -it --rm -v ${PWD}:/app aptly-system-test /app/system/run-system-tests $(TEST)
+
+golangci-lint:  ## Run golangci-line in docker container
+	docker run -it --rm -v ~/.cache/golangci-lint/$(GOLANGCI_LINT_VERSION):/root/.cache -v ${PWD}:/app -w /app golangci/golangci-lint:$(GOLANGCI_LINT_VERSION) golangci-lint run
 
-docker-system-tests:
-	docker run --rm -it -v ${PWD}:/app aptly-system-test
+flake8:
+	flake8 system
 
-.PHONY: man modules version release goxc docker-build docker-system-tests
+.PHONY: help man modules version release goxc docker-build docker-system-tests

README.rst

@@ -9,7 +9,7 @@ aptly
     :target: https://codecov.io/gh/aptly-dev/aptly
 
 .. image:: https://badges.gitter.im/Join Chat.svg
-    :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+    :target: https://matrix.to/#/#aptly:gitter.im
 
 .. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
     :target: https://goreportcard.com/report/aptly-dev/aptly

api/api.go

@@ -155,18 +155,26 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
 	// Run this task in background if configured globally or per-request
 	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
 	if background {
-		log.Info().Msg("Executing task asynchronously")
+		log.Debug().Msg("Executing task asynchronously")
 		task, conflictErr := runTaskInBackground(name, resources, proc)
 		if conflictErr != nil {
 			AbortWithJSONError(c, 409, conflictErr)
 			return
 		}
 		c.JSON(202, task)
 	} else {
-		log.Info().Msg("Executing task synchronously")
-		out := context.Progress()
-		detail := task.Detail{}
-		retValue, err := proc(out, &detail)
+		log.Debug().Msg("Executing task synchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			AbortWithJSONError(c, 409, conflictErr)
+			return
+		}
+
+		// wait for task to finish
+		context.TaskList().WaitForTaskByID(task.ID)
+
+		retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
+		err, _ := context.TaskList().GetTaskErrorByID(task.ID)
 		if err != nil {
 			AbortWithJSONError(c, retValue.Code, err)
 			return
@@ -226,6 +234,36 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 		}
 	}
 
+	// filter packages by version
+	if c.Request.URL.Query().Get("maximumVersion") == "1" {
+		list.PrepareIndex()
+		list.ForEach(func(p *deb.Package) error {
+			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
+			if err != nil {
+				fmt.Println("filter packages by version, query string parse err: ", err)
+				c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
+			} else {
+				tmpList, err := list.Filter([]deb.PackageQuery{versionQ}, false,
+					nil, 0, []string{})
+
+				if err == nil {
+					if tmpList.Len() > 0 {
+						tmpList.ForEach(func(tp *deb.Package) error {
+							list.Remove(tp)
+							return nil
+						})
+						list.Add(p)
+					}
+				} else {
+					fmt.Println("filter packages by version, filter err: ", err)
+					c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
+				}
+			}
+
+			return nil
+		})
+	}
+
 	if c.Request.URL.Query().Get("format") == "details" {
 		list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)