From 42dfc8c9bf0bb6d63d1ffbc54b7c401ac5077d34 Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Tue, 13 Feb 2024 07:58:15 +0100 Subject: [PATCH 1/4] use given version as input version to allow overriding git describe Signed-off-by: Reuben Miller --- ci/build_scripts/version.sh | 79 ++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/ci/build_scripts/version.sh b/ci/build_scripts/version.sh index e031dd56980..cc08356edaf 100755 --- a/ci/build_scripts/version.sh +++ b/ci/build_scripts/version.sh @@ -14,34 +14,30 @@ The following environment variables are set: * CONTAINER_VERSION * TARBALL_VERSION -NOTES - -If you have previously sourced the script into your current shell environment then you will need to unset the GIT_SEMVER variable otherwise the previously created GIT_SEMVER value will be used. -Alternatively, you can limit to calling the script only from other scripts to avoid leakage of the environment variables between script calls. - -Example: - unset GIT_SEMVER - . $0 - USAGE - $0 [apk|deb|rpm|container|tarball|all] + $0 [apk|deb|rpm|container|tarball|all] [--version ] # Print out a version # importing values via a script . $0 [--version ] +FLAGS + --version Input version to use to generate the package version values + EXAMPLES . $0 # Export env variables for use in packaging -unset GIT_SEMVER . $0 # Export env variables for use in package but ignore any previously set value . $0 --version 1.2.3 # Export env variables but use an explicit value +. $0 --version 1.0.1-100-gabcdef all +# Export env variables but use an explicit value + EOT } @@ -91,43 +87,36 @@ set_version_variables() { BASE_VERSION= BUMP_VERSION=0 - if [ -z "$GIT_SEMVER" ]; then + # Allow overriding the GIT_DESCRIBE_RAW value to simulate git describe values + if [ -n "$GIT_DESCRIBE_RAW" ]; then + echo "Using (raw) version set by user: $GIT_DESCRIBE_RAW" >&2 + else GIT_DESCRIBE_RAW=$(git describe --always --tags --abbrev=7 2>/dev/null || true) + fi - if [[ "$GIT_DESCRIBE_RAW" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - # Tagged release - BASE_VERSION="$GIT_DESCRIBE_RAW" - elif [[ "$GIT_DESCRIBE_RAW" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then - # Pre-release tagged release, e.g. 1.0.0-rc.1 - BASE_VERSION="$GIT_DESCRIBE_RAW" - elif [[ "$GIT_DESCRIBE_RAW" =~ ^[a-z0-9]+$ ]]; then - # Note: Sometimes git describe only prints out the git hash when run on a PR branch - # from someone else. In such instances this causes the version to be incompatible with - # linux package types. For instance, debian versions must start with a digit. - # When this situation is detected, git describe is run on the main branch however the - # git hash is replaced with the current git hash of the current branch. - echo "Using git describe from origin/main" >&2 - BUILD_COMMIT_HASH="g$GIT_DESCRIBE_RAW" - GIT_DESCRIBE_RAW=$(git describe --always --tags --abbrev=7 origin/main 2>/dev/null || true) - BASE_VERSION=$(echo "$GIT_DESCRIBE_RAW" | cut -d- -f1) - BUILD_COMMITS_SINCE=$(echo "$GIT_DESCRIBE_RAW" | cut -d- -f2) - else - BASE_VERSION=$(echo "$GIT_DESCRIBE_RAW" | sed -E 's|-[0-9]+-g[a-f0-9]+$||g') - BUILD_COMMITS_SINCE=$(echo "$GIT_DESCRIBE_RAW" | rev | cut -d- -f2 | rev) - BUILD_COMMIT_HASH=$(echo "$GIT_DESCRIBE_RAW" | rev | cut -d- -f1 | rev) - fi - BUMP_VERSION=1 + if [[ "$GIT_DESCRIBE_RAW" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + # Tagged release + BASE_VERSION="$GIT_DESCRIBE_RAW" + elif [[ "$GIT_DESCRIBE_RAW" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then + # Pre-release tagged release, e.g. 1.0.0-rc.1 + BASE_VERSION="$GIT_DESCRIBE_RAW" + elif [[ "$GIT_DESCRIBE_RAW" =~ ^[a-z0-9]+$ ]]; then + # Note: Sometimes git describe only prints out the git hash when run on a PR branch + # from someone else. In such instances this causes the version to be incompatible with + # linux package types. For instance, debian versions must start with a digit. + # When this situation is detected, git describe is run on the main branch however the + # git hash is replaced with the current git hash of the current branch. + echo "Using git describe from origin/main" >&2 + BUILD_COMMIT_HASH="g$GIT_DESCRIBE_RAW" + GIT_DESCRIBE_RAW=$(git describe --always --tags --abbrev=7 origin/main 2>/dev/null || true) + BASE_VERSION=$(echo "$GIT_DESCRIBE_RAW" | cut -d- -f1) + BUILD_COMMITS_SINCE=$(echo "$GIT_DESCRIBE_RAW" | cut -d- -f2) else - echo "Using version set by user: $GIT_SEMVER" >&2 - if echo "$GIT_SEMVER" | grep -Eq '.+~.+\+.+'; then - BASE_VERSION=$(echo "$GIT_SEMVER" | cut -d'~' -f1) - VERSION_META=$(echo "$GIT_SEMVER" | cut -d'~' -f2) - BUILD_COMMITS_SINCE=$(echo "$VERSION_META" | cut -d'+' -f1) - BUILD_COMMIT_HASH=$(echo "$VERSION_META" | cut -d'+' -f2) - else - BASE_VERSION="$GIT_SEMVER" - fi + BASE_VERSION=$(echo "$GIT_DESCRIBE_RAW" | sed -E 's|-[0-9]+-g[a-f0-9]+$||g') + BUILD_COMMITS_SINCE=$(echo "$GIT_DESCRIBE_RAW" | rev | cut -d- -f2 | rev) + BUILD_COMMIT_HASH=$(echo "$GIT_DESCRIBE_RAW" | rev | cut -d- -f1 | rev) fi + BUMP_VERSION=1 if [ -n "$BUILD_COMMITS_SINCE" ]; then # If there is build info, it means we are building an unofficial version (e.g. it does not have a git tag) @@ -232,7 +221,7 @@ while [ $# -gt 0 ]; do case "$1" in --version) if [ -n "$2" ]; then - GIT_SEMVER="$2" + GIT_DESCRIBE_RAW="$2" fi shift ;; From ea394ea1c793b29ec25b3d84a5426aa4b20dce5d Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Tue, 13 Feb 2024 07:58:59 +0100 Subject: [PATCH 2/4] promote cloudsmith artifacts on tagging Signed-off-by: Reuben Miller --- .github/workflows/build-workflow.yml | 12 ++++++ ci/admin/cloudsmith_admin.sh | 59 ++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 3e90438233d..0125481618d 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -334,6 +334,11 @@ jobs: needs: [publish-virtual-packages, publish-containers] if: startsWith(github.ref, 'refs/tags/') steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false - name: Release uses: softprops/action-gh-release@v1 env: @@ -348,3 +353,10 @@ jobs: env: # Triggering another workflow requires more additional credentials GITHUB_TOKEN: ${{ secrets.ACTIONS_PAT }} + + - name: Promote cloudsmith packages + env: + VERSION: ${{ github.ref_name }} + PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }} + run: | + ./ci/admin/cloudsmith_admin.sh promote "$VERSION" diff --git a/ci/admin/cloudsmith_admin.sh b/ci/admin/cloudsmith_admin.sh index 7351f834ad8..40308eaad07 100755 --- a/ci/admin/cloudsmith_admin.sh +++ b/ci/admin/cloudsmith_admin.sh @@ -17,6 +17,8 @@ ACTION cleanup Remove old versions from the tedge-main and tedge-main-armv6 repositories which where uploaded longer than x days ago. + + promote Promote already published packages from the dev repo to the release repo Env: PUBLISH_TOKEN Cloudsmith API token used for authorize the delete commands @@ -56,6 +58,7 @@ if [ $# -lt 1 ]; then fi COMMAND="$1" +shift COMMON_ARGS=() @@ -117,6 +120,55 @@ delete_old_versions() { done < <(list_old_versions) } +promote_version() { + # + # Promote version from the main repo to the official release repo + # + + # Get the versions + # shellcheck disable=SC1091 + . ./ci/build_scripts/version.sh all --version "$1" + + if [ -z "$APK_VERSION" ]; then + echo "APK_VERSION variable is empty" >&2 + return 1 + fi + if [ -z "$RPM_VERSION" ]; then + echo "RPM_VERSION variable is empty" >&2 + return 1 + fi + if [ -z "$DEB_VERSION" ]; then + echo "DEB_VERSION variable is empty" >&2 + return 1 + fi + if [ -z "$TARBALL_VERSION" ]; then + echo "TARBALL_VERSION variable is empty" >&2 + return 1 + fi + + # Build cloudsmith query using the different package version variants + APK_VERSION_QUERY="(version:^${APK_VERSION}-r0\$ AND format:alpine)" + RPM_VERSION_QUERY="(version:^${RPM_VERSION}-1\$ AND format:rpm)" + DEB_VERSION_QUERY="(version:^${DEB_VERSION}\$ AND format:deb)" + TARBALL_VERSION_QUERY="(version:^${TARBALL_VERSION}\$ AND format:raw)" + query="$APK_VERSION_QUERY OR $RPM_VERSION_QUERY OR $DEB_VERSION_QUERY OR $TARBALL_VERSION_QUERY" + + if [ -z "$query" ]; then + echo "Unknown package query" + exit 1 + fi + + printf "Cloudsmith package selection query:\n %s\n\n" "$query" >&2 + + cloudsmith ls pkg thinedge/tedge-main -q "$query" -F json -l 500 \ + | jq '.data[] | .namespace + "/" + .repository + "/" + .slug' -r \ + | xargs -Ipackage cloudsmith cp package thin-edge/tedge-release --yes "${COMMON_ARGS[@]}" + + cloudsmith ls pkg thinedge/tedge-main-armv6 -q "$query" -F json -l 500 \ + | jq '.data[] | .namespace + "/" + .repository + "/" + .slug' -r \ + | xargs -Ipackage cloudsmith cp package thin-edge/tedge-release-armv6 --yes "${COMMON_ARGS[@]}" +} + # # Main # @@ -124,6 +176,13 @@ case "$COMMAND" in cleanup) delete_old_versions ;; + promote) + if [ $# -lt 1 ]; then + echo "Missing version. Please provide the version as the first positional argument" >&2 + exit 1 + fi + promote_version "$1" + ;; *) echo "Unknown action" >&2 help From 95998142fa5604cfb299b0b3b8bfe2929726de7a Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Tue, 13 Feb 2024 19:44:25 +0100 Subject: [PATCH 3/4] don't set explicit version to avoid double conversions Signed-off-by: Reuben Miller --- ci/build_scripts/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_scripts/build.sh b/ci/build_scripts/build.sh index d7f1f2662fe..64d34065426 100755 --- a/ci/build_scripts/build.sh +++ b/ci/build_scripts/build.sh @@ -208,7 +208,7 @@ if [ "$INCLUDE_DEPRECATED_PACKAGES" = "1" ]; then ) fi -./ci/build_scripts/package.sh build "$ARCH" "${PACKAGES[@]}" --version "$GIT_SEMVER" --output "$OUTPUT_DIR" +./ci/build_scripts/package.sh build "$ARCH" "${PACKAGES[@]}" --output "$OUTPUT_DIR" if [ "$INCLUDE_TEST_PACKAGES" = 1 ]; then if [ "$BUILD" = 1 ]; then @@ -220,5 +220,5 @@ if [ "$INCLUDE_TEST_PACKAGES" = 1 ]; then fi # Package test binaries (deb only) - ./ci/build_scripts/package.sh build "$ARCH" "${TEST_PACKAGES[@]}" --version "$GIT_SEMVER" --types deb --output "$OUTPUT_DIR" --no-clean + ./ci/build_scripts/package.sh build "$ARCH" "${TEST_PACKAGES[@]}" --types deb --output "$OUTPUT_DIR" --no-clean fi From 2e25b52b997d3208d733f804d10b6b8c52795cf7 Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Tue, 13 Feb 2024 19:44:45 +0100 Subject: [PATCH 4/4] log git describe in build script Signed-off-by: Reuben Miller --- ci/build_scripts/version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_scripts/version.sh b/ci/build_scripts/version.sh index cc08356edaf..820aca66c8e 100755 --- a/ci/build_scripts/version.sh +++ b/ci/build_scripts/version.sh @@ -94,6 +94,7 @@ set_version_variables() { GIT_DESCRIBE_RAW=$(git describe --always --tags --abbrev=7 2>/dev/null || true) fi + echo "Using GIT_DESCRIBE_RAW=$GIT_DESCRIBE_RAW" >&2 if [[ "$GIT_DESCRIBE_RAW" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then # Tagged release BASE_VERSION="$GIT_DESCRIBE_RAW"