From 9dc3e61091c97cabe02b6cfff7a20a17411f87da Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 19 Aug 2024 17:54:34 +0200 Subject: [PATCH 01/15] soft fail for RosettaIntegration Tests --- buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index ab4eed58015..eaee44624b3 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -1,5 +1,7 @@ let Cmd = ../../Lib/Cmds.dhall +let B = ../../External/Buildkite.dhall + let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall @@ -14,6 +16,8 @@ let Command = ../../Command/Base.dhall let Size = ../../Command/Size.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -49,6 +53,7 @@ in Pipeline.build ] , label = "Rosetta integration tests Bullseye Long" , key = "rosetta-integration-tests-bullseye-long" + , soft_fail = Some (B/SoftFail.Boolean True) , target = Size.Small , depends_on = [ { name = "MinaArtifactBullseye" From 32a5b8370df177530415e61c66b2cf3a02169ccf Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 24 Aug 2024 14:53:33 +0200 Subject: [PATCH 02/15] Remove buster support --- README-ci-failures.md | 6 +-- buildkite/scripts/run_promote_build_job.sh | 4 +- .../scripts/run_verify_promoted_build_job.sh | 4 +- buildkite/src/Command/RunInToolchain.dhall | 10 ----- buildkite/src/Constants/ContainerImages.dhall | 2 - buildkite/src/Constants/DebianVersions.dhall | 13 +----- buildkite/src/Constants/DockerVersions.dhall | 9 +--- buildkite/src/Constants/Toolchain.dhall | 2 - .../Entrypoints/GenerateHardforkPackage.dhall | 2 +- .../src/Jobs/Promote/AutoPromoteNightly.dhall | 2 - .../src/Jobs/Release/MinaArtifactBuster.dhall | 21 ---------- .../Release/MinaArtifactMainnetBuster.dhall | 35 ---------------- .../Release/MinaToolchainArtifactBuster.dhall | 41 ------------------- buildkite/src/README.md | 8 ++-- dockerfiles/Dockerfile-mina-daemon | 8 ---- dockerfiles/Dockerfile-mina-rosetta | 8 ---- dockerfiles/Dockerfile-mina-test-suite | 8 ---- dockerfiles/stages/1-build-deps | 2 +- scripts/deb-builder-helpers.sh | 4 -- scripts/debian/verify.sh | 1 - scripts/release-docker.sh | 4 +- 21 files changed, 17 insertions(+), 177 deletions(-) delete mode 100644 buildkite/src/Jobs/Release/MinaArtifactBuster.dhall delete mode 100644 buildkite/src/Jobs/Release/MinaArtifactMainnetBuster.dhall delete mode 100644 buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall diff --git a/README-ci-failures.md b/README-ci-failures.md index cfd8f4bc81d..6e4f678c8b9 100644 --- a/README-ci-failures.md +++ b/README-ci-failures.md @@ -29,8 +29,8 @@ details and links to the failures. ## CI environment mismatch The CI runs its jobs in multiple Docker images. The images that it is using are -specified in `buildkite/src/Constants/COntainerImages.dhall`: the CI uses all -Debian images prefixed by `minaToolchainBuster`. +specified in `buildkite/src/Constants/ContainerImages.dhall`: the CI uses all +Debian images prefixed by `minaToolchainBullseye`. These images are generated by the CI itself, in particular based on the content of the `dockerfiles` directory and the `opam.export` file (which describes versions of @@ -46,7 +46,7 @@ In this case, you should: [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain) where they have been automatically uploaded. Look for your branch name and commit hash in the second column. You should find several images suffixed - with `-buster`, `-bullseye`, `-stretch` and `-focal` (Debian versions). + with `-bullseye` and `-focal` (Debian versions). 3. For each such image, retrieve its full name and hash by hovering its link and clicking the `Copy full image name` tooltip that appears (or retrieve it on the image's page). diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 26feee3e23a..3de86fc13de 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -18,7 +18,7 @@ # - "NETWORK=Devnet" # - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" # - "NEW_VERSION=3.0.0fake-ddb6fc4" -# - "CODENAMES=Focal,Buster,Bullseye" +# - "CODENAMES=Focal,Bullseye" # - "FROM_CHANNEL=Unstable" # - "TO_CHANNEL=Experimental" # image: codaprotocol/ci-toolchain-base:v3 @@ -41,7 +41,7 @@ function usage() { fi echo " DEBIANS The comma delimitered debian names. For example: 'Daemon,Archive' " echo " DOCKERS The comma delimitered docker names. For example: 'Daemon,Archive' " - echo " CODENAMES The Debian codenames (Bullseye, Buster etc.)" + echo " CODENAMES The Debian codenames (Bullseye, Focal etc.)" echo " FROM_VERSION The Source Docker or Debian version " echo " NEW_VERSION The new Debian version or new Docker tag" echo " REMOVE_PROFILE_FROM_NAME Should we remove profile suffix from debian name" diff --git a/buildkite/scripts/run_verify_promoted_build_job.sh b/buildkite/scripts/run_verify_promoted_build_job.sh index ed093b1a44b..da2a3789d3f 100755 --- a/buildkite/scripts/run_verify_promoted_build_job.sh +++ b/buildkite/scripts/run_verify_promoted_build_job.sh @@ -18,7 +18,7 @@ # - "NETWORK=Devnet" # - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" # - "NEW_VERSION=3.0.0fake-ddb6fc4" -# - "CODENAMES=Focal,Buster,Bullseye" +# - "CODENAMES=Focal,Bullseye" # - "FROM_CHANNEL=Unstable" # - "TO_CHANNEL=Experimental" # image: codaprotocol/ci-toolchain-base:v3 @@ -41,7 +41,7 @@ function usage() { fi echo " DEBIANS The comma delimitered debian names. For example: 'Daemon,Archive' " echo " DOCKERS The comma delimitered docker names. For example: 'Daemon,Archive' " - echo " CODENAMES The Debian codenames (Bullseye, Buster etc.)" + echo " CODENAMES The Debian codenames (Bullseye, Focal etc.)" echo " NEW_VERSION The new Debian version or new Docker tag" echo " REMOVE_PROFILE_FROM_NAME Should we remove profile suffix from debian name" echo " PROFILE The Docker and Debian profile (Standard, Lightnet)" diff --git a/buildkite/src/Command/RunInToolchain.dhall b/buildkite/src/Command/RunInToolchain.dhall index 6d4616b605d..ecc05ca00f1 100644 --- a/buildkite/src/Command/RunInToolchain.dhall +++ b/buildkite/src/Command/RunInToolchain.dhall @@ -33,15 +33,6 @@ let runInToolchainBullseye environment innerScript -let runInToolchainBuster - : List Text -> Text -> List Cmd.Type - = \(environment : List Text) - -> \(innerScript : Text) - -> runInToolchainImage - ContainerImages.minaToolchainBuster - environment - innerScript - let runInToolchain : List Text -> Text -> List Cmd.Type = \(environment : List Text) @@ -55,5 +46,4 @@ in { runInToolchain = runInToolchain , runInToolchainImage = runInToolchainImage , runInToolchainBookworm = runInToolchainBookworm , runInToolchainBullseye = runInToolchainBullseye - , runInToolchainBuster = runInToolchainBuster } diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall index 65f7d567043..6b80b2c046e 100644 --- a/buildkite/src/Constants/ContainerImages.dhall +++ b/buildkite/src/Constants/ContainerImages.dhall @@ -3,8 +3,6 @@ -- NOTE: minaToolchainBullseye is also used for building Ubuntu Focal packages in CI -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI { toolchainBase = "codaprotocol/ci-toolchain-base:v3" -, minaToolchainBuster = - "gcr.io/o1labs-192920/mina-toolchain@sha256:d27c15e3143a99b86155ba57696020c00e2a296b388499b3e6fb364478ddda3a" , minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f" , minaToolchainBookworm = diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index 98cd6882689..627fb3cb659 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -4,14 +4,13 @@ let BuildFlags = ./BuildFlags.dhall let S = ../Lib/SelectFiles.dhall -let DebVersion = < Bookworm | Bullseye | Buster | Jammy | Focal > +let DebVersion = < Bookworm | Bullseye | Jammy | Focal > let capitalName = \(debVersion : DebVersion) -> merge { Bookworm = "Bookworm" , Bullseye = "Bullseye" - , Buster = "Buster" , Jammy = "Jammy" , Focal = "Focal" } @@ -22,7 +21,6 @@ let lowerName = -> merge { Bookworm = "bookworm" , Bullseye = "bullseye" - , Buster = "buster" , Jammy = "jammy" , Focal = "focal" } @@ -53,14 +51,6 @@ let dependsOnStep = , key = "${step}-deb-pkg" } ] - , Buster = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] , Jammy = [ { name = "${prefix}${capitalName @@ -122,7 +112,6 @@ let dirtyWhen = -> merge { Bookworm = minimalDirtyWhen , Bullseye = bullseyeDirtyWhen - , Buster = minimalDirtyWhen , Jammy = minimalDirtyWhen , Focal = minimalDirtyWhen } diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index b57d9785ed0..fe094476b0f 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -2,14 +2,13 @@ let Profiles = ./Profiles.dhall let Docker : Type - = < Bookworm | Bullseye | Buster | Jammy | Focal > + = < Bookworm | Bullseye | Jammy | Focal > let capitalName = \(docker : Docker) -> merge { Bookworm = "Bookworm" , Bullseye = "Bullseye" - , Buster = "Buster" , Jammy = "Jammy" , Focal = "Focal" } @@ -20,7 +19,6 @@ let lowerName = -> merge { Bookworm = "bookworm" , Bullseye = "bullseye" - , Buster = "buster" , Jammy = "jammy" , Focal = "focal" } @@ -47,11 +45,6 @@ let dependsOn = , key = "${binary}-${lowerName docker}-${suffix}" } ] - , Buster = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" - , key = "${binary}-${lowerName docker}-${suffix}" - } - ] , Jammy = [ { name = "${prefix}${capitalName docker}${profileSuffix}" , key = "${binary}-${lowerName docker}-${suffix}" diff --git a/buildkite/src/Constants/Toolchain.dhall b/buildkite/src/Constants/Toolchain.dhall index b70181349bf..21cbde37fe5 100644 --- a/buildkite/src/Constants/Toolchain.dhall +++ b/buildkite/src/Constants/Toolchain.dhall @@ -13,7 +13,6 @@ let runner = -> merge { Bookworm = RunInToolchain.runInToolchainBookworm , Bullseye = RunInToolchain.runInToolchainBullseye - , Buster = RunInToolchain.runInToolchainBuster , Jammy = RunInToolchain.runInToolchainBookworm , Focal = RunInToolchain.runInToolchainBullseye } @@ -34,7 +33,6 @@ let image = -> merge { Bookworm = ContainerImages.minaToolchainBookworm , Bullseye = ContainerImages.minaToolchainBullseye - , Buster = ContainerImages.minaToolchainBuster , Jammy = ContainerImages.minaToolchainBookworm , Focal = ContainerImages.minaToolchainBullseye } diff --git a/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall b/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall index 31756138e28..64bfb78f805 100644 --- a/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall +++ b/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall @@ -35,7 +35,7 @@ let generate_hardfork_package = , Cmd.quietly "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{=}).pipeline' | buildkite-agent pipeline upload" , Cmd.quietly - "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{codename = DebianVersions.DebVersion.Buster}).pipeline' | buildkite-agent pipeline upload" + "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{codename = DebianVersions.DebVersion.Bullseye}).pipeline' | buildkite-agent pipeline upload" ] , label = "Generate hardfork package" , key = "generate-hardfork-package" diff --git a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall index e0bd2c3a365..dd3d22fcbdb 100644 --- a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall +++ b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall @@ -44,7 +44,6 @@ let promotePackages = , codenames = [ DebianVersions.DebVersion.Bullseye , DebianVersions.DebVersion.Focal - , DebianVersions.DebVersion.Buster ] , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.NightlyCompatible @@ -71,7 +70,6 @@ let verifyPackages = , codenames = [ DebianVersions.DebVersion.Bullseye , DebianVersions.DebVersion.Focal - , DebianVersions.DebVersion.Buster ] , channel = DebianChannel.Type.NightlyCompatible , new_tags = diff --git a/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall b/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall deleted file mode 100644 index 3230e96725c..00000000000 --- a/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall +++ /dev/null @@ -1,21 +0,0 @@ -let ArtifactPipelines = ../../Command/MinaArtifact.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let Artifacts = ../../Constants/Artifacts.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -in Pipeline.build - ( ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - , artifacts = - [ Artifacts.Type.Daemon - , Artifacts.Type.Archive - , Artifacts.Type.BatchTxn - , Artifacts.Type.Rosetta - , Artifacts.Type.ZkappTestTransaction - ] - , debVersion = DebianVersions.DebVersion.Buster - } - ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBuster.dhall b/buildkite/src/Jobs/Release/MinaArtifactMainnetBuster.dhall deleted file mode 100644 index 49b2adcc03c..00000000000 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBuster.dhall +++ /dev/null @@ -1,35 +0,0 @@ -let ArtifactPipelines = ../../Command/MinaArtifact.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let Network = ../../Constants/Network.dhall - -let Artifacts = ../../Constants/Artifacts.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineMode = ../../Pipeline/Mode.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -in Pipeline.build - ( ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - , artifacts = - [ Artifacts.Type.Daemon - , Artifacts.Type.Archive - , Artifacts.Type.BatchTxn - , Artifacts.Type.Rosetta - , Artifacts.Type.ZkappTestTransaction - ] - , debVersion = DebianVersions.DebVersion.Buster - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" - } - ) diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall deleted file mode 100644 index 3057316b1cc..00000000000 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall +++ /dev/null @@ -1,41 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let PipelineMode = ../../Pipeline/Mode.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "dockerfiles/stages/1-") - , S.strictlyStart (S.contains "dockerfiles/stages/2-") - , S.strictlyStart (S.contains "dockerfiles/stages/3-") - , S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact") - , S.strictly (S.contains "opam.export") - , S.strictlyEnd (S.contains "rust-toolchain.toml") - ] - , path = "Release" - , name = "MinaToolchainArtifactBuster" - , tags = [ PipelineTag.Type.Toolchain ] - , mode = PipelineMode.Type.Stable - } - , steps = - [ let toolchainBusterSpec = - DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "buster" - , extra_args = "--no-cache" - , step_key = "toolchain-buster-docker-image" - } - - in DockerImage.generateStep toolchainBusterSpec - ] - } diff --git a/buildkite/src/README.md b/buildkite/src/README.md index d9b577e81b5..6299bad1610 100644 --- a/buildkite/src/README.md +++ b/buildkite/src/README.md @@ -134,7 +134,7 @@ steps: - "NETWORK=Devnet" - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" - "NEW_VERSION=3.0.0fake-ddb6fc4" - - "CODENAMES=Focal,Buster,Bullseye" + - "CODENAMES=Focal,Bullseye" - "FROM_CHANNEL=Unstable" - "TO_CHANNEL=Experimental" image: codaprotocol/ci-toolchain-base:v3 @@ -150,7 +150,7 @@ All list of available parameters: - DOCKERS - The comma delimitered docker names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/Artifacts.dhall` files. Only CamelCase format is supported -- CODENAMES - The Debian codenames `Bullseye,Buster,Focal`. All available names are located in `buildkite/src/Constans/DebianVersions.dhall`. Only CamelCase format is supported +- CODENAMES - The Debian codenames `Bullseye,Focal`. All available names are located in `buildkite/src/Constans/DebianVersions.dhall`. Only CamelCase format is supported - FROM_VERSION - The Source Docker or Debian version @@ -187,7 +187,7 @@ Pipeline with create 6 jobs for each Docker and Debian component separately. - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" - "NEW_VERSION=3.0.0-ddb6fc4" - "PUBLISH"=1 - - "CODENAMES=Focal,Buster,Bullseye" + - "CODENAMES=Focal,Bullseye" - "FROM_CHANNEL=Unstable" - "TO_CHANNEL=Devnet" @@ -202,6 +202,6 @@ We want only to move dockers from gcr to dockerhub without changing version. Cur - "NETWORK=Devnet" - "FROM_VERSION=3.0.0-dc6bf78" - "NEW_VERSION=3.0.0-dc6bf78" - - "CODENAMES=Focal,Buster,Bullseye" + - "CODENAMES=Focal,Bullseye" - "PUBLISH=1" ``` \ No newline at end of file diff --git a/dockerfiles/Dockerfile-mina-daemon b/dockerfiles/Dockerfile-mina-daemon index 4f718e5ea3f..7d43e27bd0e 100644 --- a/dockerfiles/Dockerfile-mina-daemon +++ b/dockerfiles/Dockerfile-mina-daemon @@ -22,14 +22,6 @@ ENV DEBIAN_FRONTEND noninteractive # We do not install the below platform-specific dependencies, # and instead assume that apt will install the proper deps based on the package definition. # The packages are noted here just for clarity/documentation. -# Stretch-only Deps: -# libffi6 -# libprocps6 -# libjemalloc1 -# Buster-only Deps: -# libffi7 -# libprocps7 -# libjemalloc2 # Bullseye/Focal-only Deps: # libffi7 # libprocps8 diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 53355cc049f..51c90c78c0f 100644 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -22,14 +22,6 @@ ENV DEBIAN_FRONTEND noninteractive # We do not install the below platform-specific dependencies, # and instead assume that apt will install the proper deps based on the package definition. # The packages are noted here just for clarity/documentation. -# Stretch-only Deps: -# libffi6 -# libprocps6 -# libjemalloc1 -# Buster-only Deps: -# libffi7 -# libprocps7 -# libjemalloc2 # Bullseye/Focal-only Deps: # libffi7 # libprocps8 diff --git a/dockerfiles/Dockerfile-mina-test-suite b/dockerfiles/Dockerfile-mina-test-suite index 98ac49ce9cd..1162bfa5e9e 100644 --- a/dockerfiles/Dockerfile-mina-test-suite +++ b/dockerfiles/Dockerfile-mina-test-suite @@ -13,14 +13,6 @@ ENV DEBIAN_FRONTEND noninteractive # We do not install the below platform-specific dependencies, # and instead assume that apt will install the proper deps based on the package definition. # The packages are noted here just for clarity/documentation. -# Stretch-only Deps: -# libffi6 -# libprocps6 -# libjemalloc1 -# Buster-only Deps: -# libffi7 -# libprocps7 -# libjemalloc2 # Bullseye/Focal-only Deps: # libffi7 # libprocps8 diff --git a/dockerfiles/stages/1-build-deps b/dockerfiles/stages/1-build-deps index 3c57e9d40fd..3e5348cc6e5 100644 --- a/dockerfiles/stages/1-build-deps +++ b/dockerfiles/stages/1-build-deps @@ -2,7 +2,7 @@ # The "build-deps" stage # - Installs all compilers/interpreters, tools, and OS packages on the given debian or ubuntu image ################################################################################################# -# Supports debian:buster-slim, debian:bullseye-slim, and ubuntu:focal +# Supports debian:bullseye-slim, and ubuntu:focal ARG image=ubuntu:focal FROM ${image} AS build-deps diff --git a/scripts/deb-builder-helpers.sh b/scripts/deb-builder-helpers.sh index 8c2f418b76d..a236d1e1677 100755 --- a/scripts/deb-builder-helpers.sh +++ b/scripts/deb-builder-helpers.sh @@ -38,10 +38,6 @@ case "${MINA_DEB_CODENAME}" in DAEMON_DEPS=", libffi7, libjemalloc2, libpq-dev, libprocps8, mina-logproc" ARCHIVE_DEPS="libssl1.1, libgomp1, libpq-dev, libjemalloc2" ;; - buster) - DAEMON_DEPS=", libffi6, libjemalloc2, libpq-dev, libprocps7, mina-logproc" - ARCHIVE_DEPS="libssl1.1, libgomp1, libpq-dev, libjemalloc2" - ;; stretch|bionic) DAEMON_DEPS=", libffi6, libjemalloc1, libpq-dev, libprocps6, mina-logproc" ARCHIVE_DEPS="libssl1.1, libgomp1, libpq-dev, libjemalloc1" diff --git a/scripts/debian/verify.sh b/scripts/debian/verify.sh index 65fce185d12..c61d3636a7e 100755 --- a/scripts/debian/verify.sh +++ b/scripts/debian/verify.sh @@ -37,7 +37,6 @@ SCRIPT=' set -x \ ' case $CODENAME in - buster) DOCKER_IMAGE="debian:buster" ;; bullseye) DOCKER_IMAGE="debian:bullseye" ;; focal) DOCKER_IMAGE="ubuntu:focal" ;; *) echo "Unknown codename passed: $CODENAME"; exit 1;; diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index bb07cba3d56..199fd47ef92 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -23,7 +23,7 @@ function usage() { echo " -n, --network The network configuration to use (devnet or mainnet). Default=devnet" echo " -b, --branch The branch of the mina repository to use for staged docker builds. Default=compatible" echo " -r, --repo The currently used mina repository" - echo " --deb-codename The debian codename (stretch or buster) to build the docker image from. Default=stretch" + echo " --deb-codename The debian codename (bullseye or focal) to build the docker image from. Default=stretch" echo " --deb-release The debian package release channel to pull from (unstable,alpha,beta,stable). Default=unstable" echo " --deb-version The version string for the debian package to install" echo " --deb-profile The profile string for the debian package to install" @@ -57,7 +57,7 @@ case "${DEB_CODENAME##*=}" in bionic|focal|impish|jammy) IMAGE="ubuntu:${DEB_CODENAME##*=}" ;; - stretch|buster|bullseye|bookworm|sid) + stretch|bullseye|bookworm|sid) IMAGE="debian:${DEB_CODENAME##*=}-slim" ;; esac From 3067bdf1c9275d71f96ba908904a5397af41d702 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 24 Aug 2024 15:05:55 +0200 Subject: [PATCH 03/15] lint & format --- buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall index dd3d22fcbdb..829a815ec25 100644 --- a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall +++ b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall @@ -42,9 +42,7 @@ let promotePackages = , profile = Profiles.Type.Standard , network = Network.Type.Devnet , codenames = - [ DebianVersions.DebVersion.Bullseye - , DebianVersions.DebVersion.Focal - ] + [ DebianVersions.DebVersion.Bullseye, DebianVersions.DebVersion.Focal ] , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.NightlyCompatible , new_tags = @@ -68,9 +66,7 @@ let verifyPackages = , profile = Profiles.Type.Standard , network = Network.Type.Devnet , codenames = - [ DebianVersions.DebVersion.Bullseye - , DebianVersions.DebVersion.Focal - ] + [ DebianVersions.DebVersion.Bullseye, DebianVersions.DebVersion.Focal ] , channel = DebianChannel.Type.NightlyCompatible , new_tags = [ "latest-compatible-nightly" From ae8c5a584ca65d4c8efd592a06ba17bb04654d11 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 16 Aug 2024 11:42:46 +0200 Subject: [PATCH 04/15] exclude remotes/origin use proper command to get branch name from buildkite fix dhall remove test executive move fetch tags after branch detection introduce no-cache arg instead of -extra-args remove extra args from docker build debug docker/build.sh use folder above for export-git-env add prefix to docker step even more debug more debug dhall format debug export git env vars before building fix location for debian to gs script fix name again fix name Fix uplaod to gs filename and not to build docker on debian only job Add experimental debian run WIP fix ./buildkite/scripts/upload-deb-to-gs.sh path update occurences of build scripts --- Makefile | 17 +- .../modules/services/daemon/README.md | 24 ++- .../modules/services/faucet/README.md | 18 +- .../modules/services/graphql-proxy/README.md | 11 +- buildkite/scripts/build-release.sh | 2 +- .../{promote-deb.sh => debian/promote.sh} | 2 +- .../{publish-deb.sh => debian/publish.sh} | 4 +- .../upload-to-gs.sh} | 0 buildkite/src/Command/DockerImage.dhall | 26 ++- .../Command/HardforkPackageGeneration.dhall | 6 +- buildkite/src/Command/MinaArtifact.dhall | 198 +++++++++++------- buildkite/src/Constants/DebianVersions.dhall | 22 +- .../Jobs/Release/MinaArtifactBullseye.dhall | 7 + .../MinaArtifactOnlyDebianBullseye.dhall | 27 +++ .../MinaToolchainArtifactBullseye.dhall | 2 +- buildkite/src/Pipeline/Filter.dhall | 6 + buildkite/src/Pipeline/Tag.dhall | 8 + scripts/debian/aptly.sh | 17 +- scripts/debian/build.sh | 39 ++++ .../builder-helpers.sh} | 19 +- .../clear-s3-lockfile.sh} | 0 scripts/{publish-deb.sh => debian/publish.sh} | 10 +- .../{reversion-deb.sh => debian/reversion.sh} | 2 +- scripts/docker/build.sh | 175 ++++++++++++++++ scripts/docker/helper.sh | 64 ++++++ scripts/docker/release.sh | 57 +++++ scripts/export-git-env-vars.sh | 31 +++ scripts/rebuild-deb.sh | 99 --------- 28 files changed, 655 insertions(+), 238 deletions(-) rename buildkite/scripts/{promote-deb.sh => debian/promote.sh} (98%) rename buildkite/scripts/{publish-deb.sh => debian/publish.sh} (85%) rename buildkite/scripts/{upload-deb-to-gs.sh => debian/upload-to-gs.sh} (100%) create mode 100644 buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall create mode 100755 scripts/debian/build.sh rename scripts/{deb-builder-helpers.sh => debian/builder-helpers.sh} (97%) rename scripts/{clear-deb-s3-lockfile.sh => debian/clear-s3-lockfile.sh} (100%) rename scripts/{publish-deb.sh => debian/publish.sh} (87%) rename scripts/{reversion-deb.sh => debian/reversion.sh} (94%) create mode 100755 scripts/docker/build.sh create mode 100644 scripts/docker/helper.sh create mode 100755 scripts/docker/release.sh create mode 100755 scripts/export-git-env-vars.sh delete mode 100755 scripts/rebuild-deb.sh diff --git a/Makefile b/Makefile index 922473e1460..8db8d616f49 100644 --- a/Makefile +++ b/Makefile @@ -102,9 +102,9 @@ build_archive: ocaml_checks reformat-diff (ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe --profile=$(DUNE_PROFILE) $(info Build complete) -build_archive_all_sigs: ocaml_checks reformat-diff +build_archive_utils: ocaml_checks reformat-diff $(info Starting Build) - (ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/archive/archive_testnet_signatures.exe src/app/archive/archive_mainnet_signatures.exe --profile=$(DUNE_PROFILE) + (ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/replayer/replayer.exe src/app/archive_blocks/archive_blocks.exe src/app/extract_blocks/extract_blocks.exe src/app/missing_blocks_auditor/missing_blocks_auditor.exe --profile=$(DUNE_PROFILE) $(info Build complete) build_rosetta: ocaml_checks @@ -208,14 +208,7 @@ publish-macos: @./scripts/publish-macos.sh deb: - ./scripts/rebuild-deb.sh - ./scripts/archive/build-release-archives.sh - @mkdir -p /tmp/artifacts - @cp _build/mina*.deb /tmp/artifacts/. - -deb_optimized: - ./scripts/rebuild-deb.sh "optimized" - ./scripts/archive/build-release-archives.sh + ./scripts/debian/builder.sh @mkdir -p /tmp/artifacts @cp _build/mina*.deb /tmp/artifacts/. @@ -229,14 +222,12 @@ build_or_download_pv_keys: ocaml_checks (ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune exec --profile=$(DUNE_PROFILE) src/lib/snark_keys/gen_keys/gen_keys.exe -- --generate-keys-only $(info Keys built) -publish_debs: - @./buildkite/scripts/publish-deb.sh - genesiskeys: @mkdir -p /tmp/artifacts @cp _build/default/src/lib/key_gen/sample_keypairs.ml /tmp/artifacts/. @cp _build/default/src/lib/key_gen/sample_keypairs.json /tmp/artifacts/. + ############################################## ## Genesis ledger in OCaml from running daemon diff --git a/automation/terraform/modules/services/daemon/README.md b/automation/terraform/modules/services/daemon/README.md index 8fd214c5b48..b55f6a78e18 100644 --- a/automation/terraform/modules/services/daemon/README.md +++ b/automation/terraform/modules/services/daemon/README.md @@ -41,17 +41,31 @@ The manual commands to release each container are the following: ### Mina-Daemon Container *(From the root of the `MinaProtocol/mina` repository)* -`./scripts/release-docker.sh -s mina-daemon -v .. --extra-args "--build-arg deb_version= --build-arg deb_release="` +`./scripts/docker/build.sh -s mina-daemon -v .. --deb_version= --deb_release="` + +and + +`./scripts/docker/release.sh -s mina-daemon -v .. - --deb_version= --deb_release="` + Example: -`./scripts/release-docker.sh -s mina-daemon -v 0.0.10-beta4 --extra-args "--build-arg deb_version=0.0.10-beta4-fff3b856 --build-arg deb_release=alpha` +`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha` + +`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha` The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `'deb_version` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release` ### daemon Container -*(From the root of the `MinaProtocol/mina/automation` folder in the mina repository)* -`./scripts/release-docker.sh -s daemon -v .. --extra-args "--build-arg base_image_tag= "` +*(From the root of the `MinaProtocol/mina` repository)* +`./scripts/docker/build.sh -s mina-daemon -v .. --deb_version= --deb_release="` + +and + +`./scripts/docker/release.sh -s mina-daemon -v .. - --deb_version= --deb_release="` + Example: -`./scripts/release-docker.sh -s daemon -v 0.0.10-beta4 --extra-args "--build-arg base_image_tag=0.0.10-beta4"` +`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha` + +`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha` diff --git a/automation/terraform/modules/services/faucet/README.md b/automation/terraform/modules/services/faucet/README.md index c34de574bae..f0dae0b88e0 100644 --- a/automation/terraform/modules/services/faucet/README.md +++ b/automation/terraform/modules/services/faucet/README.md @@ -63,14 +63,17 @@ The manual commands to release each container are the following: ### Mina-Daemon Container *(From the root of the `MinaProtocol/mina` repository)* +`./scripts/docker/build.sh -s mina-daemon -v .. --deb_version= --deb_release="` -`./scripts/release-docker.sh -s mina-daemon -v .. --extra-args "--build-arg deb_version="` +and -### daemon Container +`./scripts/docker/release.sh -s mina-daemon -v .. - --deb_version= --deb_release="` -*(From the root of the `MinaProtocol/mina/automation` repository)* -`./scripts/release-docker.sh -s daemon -v .. --extra-args "--build-arg base_image_tag="` +Example: +`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha` + +`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha` The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `base_image_tag` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release` @@ -80,5 +83,10 @@ The Faucet Dockerfile lives in the `MinaProtocol/mina` repository [here](https:/ *(From the root of the `MinaProtocol/mina` repository)* -`./scripts/release-docker.sh -s bot -v ..` +``` +./scripts/docker/build.sh -s bot -v .. +./scripts/docker/release.sh -s bot -v .. +``` + + diff --git a/automation/terraform/modules/services/graphql-proxy/README.md b/automation/terraform/modules/services/graphql-proxy/README.md index a8d85fc6571..61ff6177c0b 100644 --- a/automation/terraform/modules/services/graphql-proxy/README.md +++ b/automation/terraform/modules/services/graphql-proxy/README.md @@ -54,13 +54,18 @@ The manual commands to release each container are the following: *(From the root of the `MinaProtocol/mina` repository)* -`./scripts/release-docker.sh -s mina-daemon -v .. --extra-args "--build-arg deb_version="` +`./scripts/docker/release.sh -s mina-daemon -v .. --deb_version="` + +and + +`./scripts/docker/release.sh -s mina-daemon -v .. - --deb_version= --deb_release="` + ### daemon Container *(From the root of the `MinaProtocol/mina/automation` repository)* -`./scripts/release-docker.sh -s daemon -v .. --extra-args "--build-arg base_image_tag="` +`./scripts/docker/release.sh -s daemon -v .. ` The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `base_image_tag` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release` @@ -70,4 +75,4 @@ The Faucet Dockerfile lives in the `MinaProtocol/mina` repository [here](https:/ *(From the root of the `MinaProtocol/mina` repository)* -`./scripts/release-docker.sh -s graphql-public-proxy -v ..` +`./scripts/docker/release.sh -s graphql-public-proxy -v ..` diff --git a/buildkite/scripts/build-release.sh b/buildkite/scripts/build-release.sh index f1242fb94f5..fe90606bd1a 100755 --- a/buildkite/scripts/build-release.sh +++ b/buildkite/scripts/build-release.sh @@ -14,7 +14,7 @@ echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkele echo "--- Prepare debian packages" -./scripts/rebuild-deb.sh $@ +./scripts/debian/build.sh $@ echo "--- Git diff after build is complete:" git diff --exit-code -- . \ No newline at end of file diff --git a/buildkite/scripts/promote-deb.sh b/buildkite/scripts/debian/promote.sh similarity index 98% rename from buildkite/scripts/promote-deb.sh rename to buildkite/scripts/debian/promote.sh index b0af08cc3f5..f0a59ac369e 100755 --- a/buildkite/scripts/promote-deb.sh +++ b/buildkite/scripts/debian/promote.sh @@ -56,7 +56,7 @@ echo "Promoting debs: ${PACKAGE}_${VERSION} to Release: ${TO_COMPONENT} and Code if [[ -z "$NEW_VERSION" ]] || [[ "$NEW_VERSION" == "$VERSION" ]]; then deb-s3 copy --s3-region=us-west-2 --lock --bucket packages.o1test.net --preserve-versions --cache-control=max-age=120 $PACKAGE $CODENAME $TO_COMPONENT --versions $VERSION --arch $ARCH --component ${FROM_COMPONENT} --codename ${CODENAME} else - source scripts/reversion-deb.sh \ + source scripts/debian/reversion.sh \ --deb $PACKAGE \ --codename $CODENAME \ --new-release $TO_COMPONENT \ diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/debian/publish.sh similarity index 85% rename from buildkite/scripts/publish-deb.sh rename to buildkite/scripts/debian/publish.sh index f92c44b2aa3..8b96a974591 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/debian/publish.sh @@ -2,7 +2,7 @@ set -eox pipefail SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -source "${SCRIPTPATH}/export-git-env-vars.sh" +source "${SCRIPTPATH}/../export-git-env-vars.sh" DOWNLOAD_FOLDER=_build @@ -15,7 +15,7 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi -source scripts/publish-deb.sh \ +source scripts/debian/publish.sh \ --names "${DOWNLOAD_FOLDER}/mina-*.deb" \ --release $MINA_DEB_RELEASE \ --version $MINA_DEB_VERSION \ diff --git a/buildkite/scripts/upload-deb-to-gs.sh b/buildkite/scripts/debian/upload-to-gs.sh similarity index 100% rename from buildkite/scripts/upload-deb-to-gs.sh rename to buildkite/scripts/debian/upload-to-gs.sh diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 1ac61447144..9c3e7c2a1dc 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -26,13 +26,13 @@ let ReleaseSpec = , version : Text , branch : Text , repo : Text + , no_cache : Bool , deb_codename : Text , deb_release : Text , deb_version : Text , deb_profile : Profiles.Type , deb_repo : DebianRepo.Type , build_flags : BuildFlags.Type - , extra_args : Text , step_key : Text , if : Optional B/If } @@ -49,7 +49,7 @@ let ReleaseSpec = , deb_profile = Profiles.Type.Standard , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.PackagesO1Test - , extra_args = "" + , no_cache = False , step_key = "daemon-standard-docker-image" , if = None B/If } @@ -59,12 +59,15 @@ let generateStep = \(spec : ReleaseSpec.Type) -> let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" + let maybeCacheOption = if spec.no_cache then "--no-cache" else "" + let buildDockerCmd = - "./scripts/release-docker.sh" + "./scripts/docker/build.sh" ++ " --service ${spec.service}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" ++ " --branch ${spec.branch}" + ++ " ${maybeCacheOption} " ++ " --deb-codename ${spec.deb_codename}" ++ " --deb-repo ${DebianRepo.address spec.deb_repo}" ++ " --deb-release ${spec.deb_release}" @@ -73,7 +76,20 @@ let generateStep = ++ " --deb-build-flags ${BuildFlags.lowerName spec.build_flags}" ++ " --repo ${spec.repo}" - ++ " --extra-args \\\"${spec.extra_args}\\\"" + + let releaseDockerCmd = + "./scripts/docker/release.sh" + ++ " --service ${spec.service}" + ++ " --version ${spec.version}" + ++ " --network ${spec.network}" + ++ " --branch ${spec.branch}" + ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-release ${spec.deb_release}" + ++ " --deb-version ${spec.deb_version}" + ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" + ++ " --deb-build-flags ${BuildFlags.lowerName + spec.build_flags}" + ++ " --repo ${spec.repo}" let commands = merge @@ -83,6 +99,7 @@ let generateStep = ++ " && source ./buildkite/scripts/export-git-env-vars.sh " ++ " && " ++ buildDockerCmd + ++ releaseDockerCmd ) ] , Local = @@ -93,6 +110,7 @@ let generateStep = ++ " && source ./buildkite/scripts/export-git-env-vars.sh " ++ " && " ++ buildDockerCmd + ++ releaseDockerCmd ++ " && ./scripts/debian/aptly.sh stop" ) ] diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index f45da4fb027..7353a8e9bb3 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -125,8 +125,8 @@ let pipeline ) "./buildkite/scripts/build-hardfork-package.sh" # [ Cmd.run - "./buildkite/scripts/upload-deb-to-gs.sh ${DebianVersions.lowerName - debVersion}" + "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName + debVersion}" ] , label = "Build Mina Hardfork Package for ${DebianVersions.capitalName @@ -144,7 +144,7 @@ let pipeline , "MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}" ] - "./buildkite/scripts/publish-deb.sh" + "./buildkite/scripts/debian/publish.sh" , label = "Publish Mina for ${DebianVersions.capitalName debVersion} Hardfork" diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 59a7bbb5876..fe4d203fc50 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -24,6 +24,8 @@ let DebianVersions = ../Constants/DebianVersions.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let DebianChannel = ../Constants/DebianChannel.dhall + let Profiles = ../Constants/Profiles.dhall let Network = ../Constants/Network.dhall @@ -45,6 +47,7 @@ let MinaBuildSpec = , toolchainSelectMode : Toolchain.SelectionMode , mode : PipelineMode.Type , tags : List PipelineTag.Type + , channel : DebianChannel.Type } , default = { prefix = "MinaArtifact" @@ -56,6 +59,7 @@ let MinaBuildSpec = , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] + , channel = DebianChannel.Type.Unstable } } @@ -83,8 +87,8 @@ let build_artifacts spec.artifacts spec.networks}" # [ Cmd.run - "./buildkite/scripts/upload-deb-to-gs.sh ${DebianVersions.lowerName - spec.debVersion}" + "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName + spec.debVersion}" ] , label = "Build Mina for ${DebianVersions.capitalName @@ -101,25 +105,48 @@ let build_artifacts ] } +let publish_to_debian_repo = + \(spec : MinaBuildSpec.Type) + -> Command.build + Command.Config::{ + , commands = + Toolchain.select + spec.toolchainSelectMode + spec.debVersion + [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" + ] + "./buildkite/scripts/debian/publish.sh" + , label = + "Publish Mina for ${DebianVersions.capitalName + spec.debVersion} ${Profiles.toSuffixUppercase + spec.profile}" + , key = "publish-deb-pkg" + , depends_on = + DebianVersions.dependsOnStep + (Some spec.prefix) + spec.debVersion + spec.profile + spec.buildFlags + "build" + , target = Size.Small + } + let docker_step - : Artifacts.Type - -> DebianVersions.DebVersion - -> List Network.Type - -> Profiles.Type - -> BuildFlags.Type - -> List DockerImage.ReleaseSpec.Type + : Artifacts.Type -> MinaBuildSpec.Type -> List DockerImage.ReleaseSpec.Type = \(artifact : Artifacts.Type) - -> \(debVersion : DebianVersions.DebVersion) - -> \(networks : List Network.Type) - -> \(profile : Profiles.Type) - -> \(buildFlags : BuildFlags.Type) + -> \(spec : MinaBuildSpec.Type) -> let step_dep_name = "build" let deps = DebianVersions.dependsOnStep - debVersion - profile - buildFlags + (Some spec.prefix) + spec.debVersion + spec.profile + spec.buildFlags step_dep_name in merge @@ -133,48 +160,50 @@ let docker_step , service = "mina-daemon" , network = Network.lowerName n , deb_codename = - "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , build_flags = buildFlags + "${DebianVersions.lowerName spec.debVersion}" + , deb_profile = spec.profile + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , step_key = "daemon-${Network.lowerName n}-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}${BuildFlags.toLabelSegment - buildFlags}-docker-image" + spec.debVersion}${Profiles.toLabelSegment + spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}-docker-image" } ) - networks + spec.networks , TestExecutive = [] : List DockerImage.ReleaseSpec.Type , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-batch-txn" , network = "berkeley" - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , build_flags = buildFlags + , deb_codename = + "${DebianVersions.lowerName spec.debVersion}" + , deb_profile = spec.profile + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , step_key = "batch-txn-${DebianVersions.lowerName - debVersion}${BuildFlags.toLabelSegment - buildFlags}--docker-image" + spec.debVersion}${BuildFlags.toLabelSegment + spec.buildFlags}--docker-image" } ] , Archive = [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-archive" - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , build_flags = buildFlags + , deb_codename = + "${DebianVersions.lowerName spec.debVersion}" + , deb_profile = spec.profile + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , step_key = "archive-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}${BuildFlags.toLabelSegment - buildFlags}-docker-image" + spec.debVersion}${Profiles.toLabelSegment + spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}-docker-image" } ] , Rosetta = @@ -182,44 +211,47 @@ let docker_step , deps = deps , service = "mina-rosetta" , network = "berkeley" - , build_flags = buildFlags + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , deb_profile = profile - , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_profile = spec.profile + , deb_codename = + "${DebianVersions.lowerName spec.debVersion}" , step_key = "rosetta-${DebianVersions.lowerName - debVersion}${BuildFlags.toLabelSegment - buildFlags}-docker-image" + spec.debVersion}${BuildFlags.toLabelSegment + spec.buildFlags}-docker-image" } ] , ZkappTestTransaction = [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-zkapp-test-transaction" - , build_flags = buildFlags + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , deb_profile = profile - , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_profile = spec.profile + , deb_codename = + "${DebianVersions.lowerName spec.debVersion}" , step_key = "zkapp-test-transaction-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}${BuildFlags.toLabelSegment - buildFlags}--docker-image" + spec.debVersion}${Profiles.toLabelSegment + spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}--docker-image" } ] , FunctionalTestSuite = [ DockerImage.ReleaseSpec::{ , deps = deps , service = "mina-test-suite" - , deb_codename = "${DebianVersions.lowerName debVersion}" - , build_flags = buildFlags + , deb_codename = + "${DebianVersions.lowerName spec.debVersion}" + , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , deb_profile = profile + , deb_profile = spec.profile , step_key = "test-suite-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}${BuildFlags.toLabelSegment - buildFlags}--docker-image" + spec.debVersion}${Profiles.toLabelSegment + spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}--docker-image" , network = "berkeley" } ] @@ -233,14 +265,7 @@ let docker_commands List/map Artifacts.Type (List DockerImage.ReleaseSpec.Type) - ( \(artifact : Artifacts.Type) - -> docker_step - artifact - spec.debVersion - spec.networks - spec.profile - spec.buildFlags - ) + (\(artifact : Artifacts.Type) -> docker_step artifact spec) spec.artifacts let flattened_docker_steps = @@ -262,27 +287,48 @@ let docker_commands ) flattened_docker_steps +let pipelineBuilder + : MinaBuildSpec.Type -> List Command.Type -> Pipeline.Config.Type + = \(spec : MinaBuildSpec.Type) + -> \(steps : List Command.Type) + -> Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion + , path = "Release" + , name = + "${spec.prefix}${DebianVersions.capitalName + spec.debVersion}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + , tags = spec.tags + , mode = spec.mode + } + , steps = steps + } + +let onlyDebianPipeline + : MinaBuildSpec.Type -> Pipeline.Config.Type + = \(spec : MinaBuildSpec.Type) + -> pipelineBuilder + spec + [ Libp2p.step spec.debVersion spec.buildFlags + , build_artifacts spec + , publish_to_debian_repo spec + ] + let pipeline : MinaBuildSpec.Type -> Pipeline.Config.Type = \(spec : MinaBuildSpec.Type) - -> let steps = - [ Libp2p.step spec.debVersion spec.buildFlags + -> pipelineBuilder + spec + ( [ Libp2p.step spec.debVersion spec.buildFlags , build_artifacts spec + , publish_to_debian_repo spec ] + # docker_commands spec + ) - in Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion - , path = "Release" - , name = - "${spec.prefix}${DebianVersions.capitalName - spec.debVersion}${Profiles.toSuffixUppercase - spec.profile}${BuildFlags.toSuffixUppercase - spec.buildFlags}" - , tags = spec.tags - , mode = spec.mode - } - , steps = steps # docker_commands spec - } - -in { pipeline = pipeline, MinaBuildSpec = MinaBuildSpec } +in { pipeline = pipeline + , onlyDebianPipeline = onlyDebianPipeline + , MinaBuildSpec = MinaBuildSpec + } diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index 627fb3cb659..f3042f5f067 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -1,3 +1,7 @@ +let Prelude = ../External/Prelude.dhall + +let Optional/default = Prelude.Optional.default + let Profiles = ./Profiles.dhall let BuildFlags = ./BuildFlags.dhall @@ -27,13 +31,14 @@ let lowerName = debVersion let dependsOnStep = - \(debVersion : DebVersion) + \(prefix : Optional Text) + -> \(debVersion : DebVersion) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) -> let profileSuffix = Profiles.toSuffixUppercase profile - let prefix = "MinaArtifact" + let prefix = Optional/default Text "MinaArtifact" prefix in merge { Bookworm = @@ -73,7 +78,12 @@ let dependsOnStep = let dependsOn = \(debVersion : DebVersion) -> \(profile : Profiles.Type) - -> dependsOnStep debVersion profile BuildFlags.Type.None "build" + -> dependsOnStep + (None Text) + debVersion + profile + BuildFlags.Type.None + "build" let minimalDirtyWhen = [ S.exactly "buildkite/src/Constants/DebianVersions" "dhall" @@ -82,8 +92,10 @@ let minimalDirtyWhen = , S.exactly "buildkite/src/Command/MinaArtifact" "dhall" , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact") , S.strictlyStart (S.contains "dockerfiles/stages") - , S.exactly "scripts/rebuild-deb" "sh" - , S.exactly "scripts/release-docker" "sh" + , S.exactly "scripts/debian/build" "sh" + , S.exactly "scripts/debian/builder-helpers" "sh" + , S.exactly "scripts/docker/release" "sh" + , S.exactly "scripts/docker/build" "sh" , S.exactly "buildkite/scripts/build-artifact" "sh" , S.exactly "buildkite/scripts/build-hardfork-package" "sh" , S.exactly "buildkite/scripts/check-compatibility" "sh" diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall index d2ee7bc60f1..3f0af8423fb 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall @@ -4,6 +4,8 @@ let Artifacts = ../../Constants/Artifacts.dhall let Pipeline = ../../Pipeline/Dsl.dhall +let PipelineTag = ../../Pipeline/Tag.dhall + in Pipeline.build ( ArtifactPipelines.pipeline ArtifactPipelines.MinaBuildSpec::{ @@ -16,5 +18,10 @@ in Pipeline.build , Artifacts.Type.ZkappTestTransaction , Artifacts.Type.FunctionalTestSuite ] + , tags = + [ PipelineTag.Type.Long + , PipelineTag.Type.Release + , PipelineTag.Type.Docker + ] } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall new file mode 100644 index 00000000000..850cafe0adf --- /dev/null +++ b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall @@ -0,0 +1,27 @@ +let ArtifactPipelines = ../../Command/MinaArtifact.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let DebianChannel = ../../Constants/DebianChannel.dhall + +in Pipeline.build + ( ArtifactPipelines.onlyDebianPipeline + ArtifactPipelines.MinaBuildSpec::{ + , artifacts = + [ Artifacts.Type.Daemon + , Artifacts.Type.Archive + , Artifacts.Type.BatchTxn + , Artifacts.Type.TestExecutive + , Artifacts.Type.Rosetta + , Artifacts.Type.ZkappTestTransaction + , Artifacts.Type.FunctionalTestSuite + ] + , tags = [ PipelineTag.Type.Debian ] + , channel = DebianChannel.Type.Experimental + , prefix = "MinaArtifactOnlyDebian" + } + ) diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall index 8603e4001fd..86e483e1e31 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -29,7 +29,7 @@ in Pipeline.build DockerImage.ReleaseSpec::{ , service = "mina-toolchain" , deb_codename = "bullseye" - , extra_args = "--no-cache" + , no_cache = True , step_key = "toolchain-bullseye-docker-image" } diff --git a/buildkite/src/Pipeline/Filter.dhall b/buildkite/src/Pipeline/Filter.dhall index f8eba8ab070..1755e99294a 100644 --- a/buildkite/src/Pipeline/Filter.dhall +++ b/buildkite/src/Pipeline/Filter.dhall @@ -15,6 +15,8 @@ let Filter | Release | HardforkPackageGeneration | Promote + | DebianBuild + | DockerBuild > let tags @@ -26,6 +28,8 @@ let tags , Long = [ Tag.Type.Long ] , TearDownOnly = [ Tag.Type.TearDown ] , ToolchainsOnly = [ Tag.Type.Toolchain ] + , DebianBuild = [ Tag.Type.Debian ] + , DockerBuild = [ Tag.Type.Docker ] , AllTests = [ Tag.Type.Lint , Tag.Type.Release @@ -50,6 +54,8 @@ let show , AllTests = "AllTests" , Release = "Release" , Promote = "Promote" + , DebianBuild = "DebianBuild" + , DockerBuild = "DockerBuild" , HardforkPackageGeneration = "HardforkPackageGeneration" } filter diff --git a/buildkite/src/Pipeline/Tag.dhall b/buildkite/src/Pipeline/Tag.dhall index 43058b2c82a..8325058f581 100644 --- a/buildkite/src/Pipeline/Tag.dhall +++ b/buildkite/src/Pipeline/Tag.dhall @@ -17,6 +17,8 @@ let Tag | Test | Stable | Toolchain + | Docker + | Debian | Hardfork | Promote > @@ -36,6 +38,8 @@ let toNatural , Hardfork = 9 , Stable = 10 , Promote = 11 + , Debian = 12 + , Docker = 13 } tag @@ -71,6 +75,8 @@ let capitalName = , Hardfork = "Hardfork" , Stable = "Stable" , Promote = "Promote" + , Docker = "Docker" + , Debian = "Debian" } tag @@ -88,6 +94,8 @@ let lowerName = , Hardfork = "hardfork" , Stable = "stable" , Promote = "promote" + , Docker = "docker" + , Debian = "debian" } tag diff --git a/scripts/debian/aptly.sh b/scripts/debian/aptly.sh index f2e16dbed75..8eae74c3544 100755 --- a/scripts/debian/aptly.sh +++ b/scripts/debian/aptly.sh @@ -16,6 +16,8 @@ declare CLI_VERSION='1.0.0'; declare CLI_NAME='aptly.sh'; declare PS4='debug($LINENO) ${FUNCNAME[0]:+${FUNCNAME[0]}}(): '; +PORT=8080 + # functions function check_required() { @@ -33,6 +35,7 @@ function start_aptly() { local __clean=$4 local __component=$5 local __repo="$__distribution-$__component" + local __port=$6 if [ $__clean = 1 ]; then rm -rf ~/.aptly @@ -47,9 +50,9 @@ function start_aptly() { aptly publish snapshot -distribution=$__distribution -skip-signing $__component if [ $__background = 1 ]; then - aptly serve -listen localhost:8080 & + aptly serve -listen localhost:$__port & else - aptly serve -listen localhost:8080 + aptly serve -listen localhost:$__port fi @@ -70,6 +73,7 @@ function start_help(){ echo " -d, --debians The Debian(s) to be available in aptly. Supports regular expression" echo " -m, --component The Component for debian repository. For example: unstable" echo " -l, --clean Removes existing aptly installation" + echo " -p, --port Server port. default=8080" echo "" echo "Example: $0 start --background --codename focal --debs *.deb --component unstable " echo "" @@ -86,6 +90,7 @@ function start(){ local __background=0 local __clean=0 local __component="unstable" + local __port=$PORT while [ ${#} -gt 0 ]; do @@ -102,6 +107,10 @@ function start(){ __distribution=${2:?$error_message} shift 2; ;; + -p | --port ) + __port=${2:?$error_message} + shift 2; + ;; -d | --debians ) __debs=${2:?$error_message} shift 2; @@ -126,7 +135,9 @@ function start(){ $__debs \ $__background \ $__clean \ - $__component + $__component \ + $__port + } diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh new file mode 100755 index 00000000000..c3e55f3e49d --- /dev/null +++ b/scripts/debian/build.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Script collects binaries and keys and builds deb archives. + +set -eox pipefail + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +source ${SCRIPTPATH}/../export-git-env-vars.sh + +echo "after export" + +source ${SCRIPTPATH}/builder-helpers.sh + +if [ $# -eq 0 ] + then + echo "No arguments supplied. Building all known debian packages" + build_logproc_deb + build_keypair_deb + build_archive_deb + build_batch_txn_deb + build_daemon_berkeley_deb + build_daemon_mainnet_deb + build_daemon_devnet_deb + build_test_executive_deb + build_functional_test_suite_deb + build_zkapp_test_transaction_deb + + else + for i in "$@"; do + if [[ $(type -t "build_${i}_deb") == function ]] + then + echo "Building $i debian package" + "build_${i}_deb" + else + echo "invalid debian package name '$i'" + exit 1 + fi + done +fi \ No newline at end of file diff --git a/scripts/deb-builder-helpers.sh b/scripts/debian/builder-helpers.sh similarity index 97% rename from scripts/deb-builder-helpers.sh rename to scripts/debian/builder-helpers.sh index a236d1e1677..76e67d3695c 100755 --- a/scripts/deb-builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -1,5 +1,10 @@ #!/bin/bash +BUILD_URL=${BUILD_URL:-"local build from '$(hostname)' host"} +MINA_DEB_CODENAME=${MINA_DEB_CODENAME:-"bullseye"} +MINA_DEB_VERSION=${MINA_DEB_VERSION:-"0.0.0-experimental"} +MINA_DEB_RELEASE=${MINA_DEB_RELEASE:-"unstable"} + # Helper script to include when building deb archives. echo "--- Setting up the envrionment to build debian packages..." @@ -7,20 +12,12 @@ echo "--- Setting up the envrionment to build debian packages..." set -euo pipefail SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -cd "${SCRIPTPATH}/../_build" +cd "${SCRIPTPATH}/../../_build" GITHASH=$(git rev-parse --short=7 HEAD) GITHASH_CONFIG=$(git rev-parse --short=8 --verify HEAD) -set +u -BUILD_NUM=${BUILDKITE_BUILD_NUM} -BUILD_URL=${BUILDKITE_BUILD_URL} -set -u - -# Load in env vars for githash/branch/etc. -source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" - -cd "${SCRIPTPATH}/../_build" +cd "${SCRIPTPATH}/../../_build" # Set dependencies based on debian release SHARED_DEPS="libssl1.1, libgmp10, libgomp1, tzdata, rocksdb-tools" @@ -62,7 +59,7 @@ case "${DUNE_PROFILE}" in esac -#ADd suffix to debian to distinguish instrumented packages +#Add suffix to debian to distinguish instrumented packages if [[ -v DUNE_INSTRUMENT_WITH ]]; then INSTRUMENTED_SUFFIX=instrumented MINA_DEB_NAME="${MINA_DEB_NAME}-${INSTRUMENTED_SUFFIX}" diff --git a/scripts/clear-deb-s3-lockfile.sh b/scripts/debian/clear-s3-lockfile.sh similarity index 100% rename from scripts/clear-deb-s3-lockfile.sh rename to scripts/debian/clear-s3-lockfile.sh diff --git a/scripts/publish-deb.sh b/scripts/debian/publish.sh similarity index 87% rename from scripts/publish-deb.sh rename to scripts/debian/publish.sh index b00a094817d..f20d9b53d33 100755 --- a/scripts/publish-deb.sh +++ b/scripts/debian/publish.sh @@ -18,14 +18,13 @@ function usage() { if [[ -n "$1" ]]; then echo -e "${RED}☞ $1${CLEAR}\n"; fi - echo "Usage: $0 [-n name] [-r release] [-v version] [-c codename]" - echo " -n, --name The Debian archive name" - echo " --names The Debians archive names" + echo "Usage: $0 -n names -r release -v version -c codename" + echo " -n, --names The Debians archive names" echo " -r, --release The Debian release" echo " -v, --version The Debian version" echo " -c, --codename The Debian codename" echo "" - echo "Example: $0 --name mina-archive_2.0.0berkeley-rc1-berkeley-48efea4 --release unstable --version 2.0.0berkeley-rc1-berkeley-48efea4 --codename bullseye " + echo "Example: $0 --name mina-archive --release unstable --version 2.0.0berkeley-rc1-berkeley-48efea4 --codename bullseye " exit 1 } @@ -45,6 +44,7 @@ S3_REGION_ARG='--s3-region=us-west-2' #>> Attempting to obtain a lock #/var/lib/gems/2.3.0/gems/deb-s3-0.10.0/lib/deb/s3/lock.rb:24:in `throw': uncaught throw #"Unable to obtain a lock after 60, giving up." DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ + --fail-if-exists \ --lock \ --preserve-versions \ --cache-control=max-age=120" @@ -57,7 +57,7 @@ for i in {1..10}; do ( --component "${DEB_RELEASE}" \ --codename "${DEB_CODENAME}" \ "${DEB_NAMES}" -) && break || scripts/clear-deb-s3-lockfile.sh; done +) && break || scripts/debian/clear-s3-lockfile.sh; done # Verify integrity of debs on remote repo function verify_o1test_repo_has_package { diff --git a/scripts/reversion-deb.sh b/scripts/debian/reversion.sh similarity index 94% rename from scripts/reversion-deb.sh rename to scripts/debian/reversion.sh index 0c00ef1e265..0aa2c16faf3 100755 --- a/scripts/reversion-deb.sh +++ b/scripts/debian/reversion.sh @@ -60,4 +60,4 @@ function rebuild_deb() { rebuild_deb -source scripts/publish-deb.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE} +source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE} diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh new file mode 100755 index 00000000000..5a32ba771d1 --- /dev/null +++ b/scripts/docker/build.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + + +set -x +# Author's Note: Because the structure of this repo is inconsistent (Dockerfiles and build contexts placed willy-nilly) +# we have to trustlist and configure image builds individually because each one is going to be slightly different. +# This is needed as opposed to trusting the structure of the each project to be consistent for every deployable. + +CLEAR='\033[0m' +RED='\033[0;31m' + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +source ${SCRIPTPATH}/helper.sh + +function usage() { + if [[ -n "$1" ]]; then + echo -e "${RED}☞ $1${CLEAR}\n"; + fi + echo "Usage: $0 [-s service-to-release] [-v service-version] [-n network]" + echo " -s, --service The Service being released to Dockerhub" + echo " -v, --version The version to be used in the docker image tag" + echo " -n, --network The network configuration to use (devnet or mainnet). Default=devnet" + echo " -b, --branch The branch of the mina repository to use for staged docker builds. Default=compatible" + echo " -r, --repo The currently used mina repository" + echo " --deb-codename The debian codename (stretch or buster) to build the docker image from. Default=stretch" + echo " --deb-release The debian package release channel to pull from (unstable,alpha,beta,stable). Default=unstable" + echo " --deb-version The version string for the debian package to install" + echo " --deb-profile The profile string for the debian package to install" + echo " --deb-build-flags The build-flags string for the debian package to install" + echo "" + echo "Example: $0 --service faucet --version v0.1.0" + echo "Valid Services: ${VALID_SERVICES[*]}" + exit 1 +} + +while [[ "$#" -gt 0 ]]; do case $1 in + -s|--service) SERVICE="$2"; shift;; + -v|--version) VERSION="$2"; shift;; + -n|--network) INPUT_NETWORK="$2"; shift;; + -b|--branch) INPUT_BRANCH="$2"; shift;; + -c|--cache-from) INPUT_CACHE="$2"; shift;; + -r|--repo) MINA_REPO="$2"; shift;; + --no-cache) NO_CACHE="--no-cache"; shift;; + --deb-codename) INPUT_CODENAME="$2"; shift;; + --deb-release) INPUT_RELEASE="$2"; shift;; + --deb-version) INPUT_VERSION="$2"; shift;; + --deb-profile) INPUT_PROFILE="$2"; shift;; + --deb-repo) INPUT_REPO="$2"; shift;; + --deb-build-flags) DEB_BUILD_FLAGS="$2"; shift;; + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + +# Verify Required Parameters are Present +if [[ -z "$SERVICE" ]]; then usage "Service is not set!"; fi; +if [[ -z "$VERSION" ]]; then usage "Version is not set!"; fi; + +NETWORK="--build-arg network=$INPUT_NETWORK" +if [[ -z "$INPUT_NETWORK" ]]; then + echo "Network is not set. Using the default (devnet)" + NETWORK="--build-arg network=devnet" +fi + +BRANCH="--build-arg MINA_BRANCH=$INPUT_BRANCH" +if [[ -z "$INPUT_BRANCH" ]]; then + echo "Branch is not set. Using the default (compatible)" + BRANCH="--build-arg MINA_BRANCH=compatible" +fi + +REPO="--build-arg MINA_REPO=${MINA_REPO}" +if [[ -z "${MINA_REPO}" ]]; then + echo "Repository is not set. Using the default (https://github.com/MinaProtocol/mina)" + REPO="--build-arg MINA_REPO=https://github.com/MinaProtocol/mina" +fi + +DEB_CODENAME="--build-arg deb_codename=$INPUT_CODENAME" +if [[ -z "$INPUT_CODENAME" ]]; then + echo "Debian codename is not set. Using the default (bullseye)" + DEB_CODENAME="--build-arg deb_codename=bullseye" +fi + +DEB_RELEASE="--build-arg deb_release=$INPUT_RELEASE" +if [[ -z "$INPUT_RELEASE" ]]; then + echo "Debian release is not set. Using the default (unstable)" + DEB_RELEASE="--build-arg deb_release=unstable" +fi + +DEB_VERSION="--build-arg deb_version=$INPUT_VERSION" +if [[ -z "$INPUT_VERSION" ]]; then + echo "Debian version is not set. Using the default ($VERSION)" + DEB_VERSION="--build-arg deb_version=$VERSION" +fi + +if [[ -z "$DEB_PROFILE" ]]; then + echo "Debian profile is not set. Using the default (standard)" + DEB_PROFILE="standard" +fi + +if [[ -z "$DEB_BUILD_FLAGS" ]]; then + DEB_BUILD_FLAGS="" +fi + +CACHE="--cache-from $INPUT_CACHE" +if [[ -z "$INPUT_CACHE" ]]; then + CACHE="" +fi + +DEB_REPO="--build-arg deb_repo=$INPUT_REPO" +if [[ -z "$INPUT_REPO" ]]; then + echo "Debian repository is not set. Using the default (http://localhost:8080)" + DEB_REPO="--build-arg deb_repo=http://localhost:8080" +fi + +if [[ $(echo ${VALID_SERVICES[@]} | grep -o "$SERVICE" - | wc -w) -eq 0 ]]; then usage "Invalid service!"; fi + +case "${SERVICE}" in + mina-archive) + DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-archive" + DOCKER_CONTEXT="dockerfiles/" + ;; + bot) + DOCKERFILE_PATH="frontend/bot/Dockerfile" + DOCKER_CONTEXT="frontend/bot" + ;; + mina-daemon) + DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-daemon" + DOCKER_CONTEXT="dockerfiles/" + VERSION="${VERSION}-${NETWORK##*=}" + ;; + mina-toolchain) + DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-opam-deps dockerfiles/stages/3-toolchain" + ;; + mina-batch-txn) + DOCKERFILE_PATH="dockerfiles/Dockerfile-txn-burst" + DOCKER_CONTEXT="dockerfiles/" + VERSION="${VERSION}-${NETWORK##*=}" + ;; + mina-rosetta) + DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-rosetta" + ;; + mina-zkapp-test-transaction) + DOCKERFILE_PATH="dockerfiles/Dockerfile-zkapp-test-transaction" + ;; + leaderboard) + DOCKERFILE_PATH="frontend/leaderboard/Dockerfile" + DOCKER_CONTEXT="frontend/leaderboard" + ;; + delegation-backend) + DOCKERFILE_PATH="dockerfiles/Dockerfile-delegation-backend" + DOCKER_CONTEXT="src/app/delegation_backend" + ;; + delegation-backend-toolchain) + DOCKERFILE_PATH="dockerfiles/Dockerfile-delegation-backend-toolchain" + DOCKER_CONTEXT="src/app/delegation_backend" + ;; + itn-orchestrator) + DOCKERFILE_PATH="dockerfiles/Dockerfile-itn-orchestrator" + DOCKER_CONTEXT="src/app/itn_orchestrator" + ;; + mina-test-suite) + DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-test-suite" + DOCKER_CONTEXT="dockerfiles/" + ;; +esac + +export_suffixes +export_docker_tag + +BUILD_NETWORK="--network=host" + +# If DOCKER_CONTEXT is not specified, assume none and just pipe the dockerfile into docker build +if [[ -z "${DOCKER_CONTEXT}" ]]; then + cat $DOCKERFILE_PATH | docker build $NO_CACHE $BUILD_NETWORK $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $DOCKER_DEB_SUFFIX $DEB_REPO $BRANCH $REPO -t "$TAG" - +else + docker build $NO_CACHE $BUILD_NETWORK $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $DOCKER_DEB_SUFFIX $DEB_REPO $BRANCH $REPO "$DOCKER_CONTEXT" -t "$TAG" -f $DOCKERFILE_PATH +fi diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh new file mode 100644 index 00000000000..6a6a1af86dc --- /dev/null +++ b/scripts/docker/helper.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Array of valid service names +export VALID_SERVICES=('mina-archive' 'mina-daemon' 'mina-rosetta' 'mina-test-suite' 'mina-batch-txn' 'mina-zkapp-test-transaction' 'mina-toolchain' 'bot' 'leaderboard' 'delegation-backend' 'delegation-backend-toolchain' 'itn-orchestrator') + +function export_base_image () { + # Determine the proper image for ubuntu or debian + case "${DEB_CODENAME##*=}" in + bionic|focal|impish|jammy) + IMAGE="ubuntu:${DEB_CODENAME##*=}" + ;; + stretch|buster|bullseye|bookworm|sid) + IMAGE="debian:${DEB_CODENAME##*=}-slim" + ;; + esac + export IMAGE="--build-arg image=${IMAGE}" +} + +function export_suffixes () { + # Determine suffix for mina name. Suffix is combined from profile and service name + # Possible outcomes: + # - instrumented + # - hardfork + # - lightnet + # - hardfork-instrumented + case "${DEB_PROFILE}" in + standard) + case "${DEB_BUILD_FLAGS}" in + *instrumented) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" + export BUILD_FLAG_SUFFIX="-instrumented" + ;; + *) + export DOCKER_DEB_SUFFIX="" + export BUILD_FLAG_SUFFIX="" + ;; + esac + ;; + *) + case "${DEB_BUILD_FLAGS}" in + *instrumented) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=${DEB_PROFILE}-instrumented" + export BUILD_FLAG_SUFFIX="-instrumented" + ;; + *) + export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=${DEB_PROFILE}" + export BUILD_FLAG_SUFFIX="" + ;; + esac + ;; + esac +} + + +function export_docker_tag() { + export_suffixes + + export DOCKER_REGISTRY="gcr.io/o1labs-192920" + export TAG="${DOCKER_REGISTRY}/${SERVICE}:${VERSION}${BUILD_FLAG_SUFFIX}" + # friendly, predictable tag + export GITHASH=$(git rev-parse --short=7 HEAD) + export HASHTAG="${DOCKER_REGISTRY}/${SERVICE}:${GITHASH}-${DEB_CODENAME##*=}-${NETWORK##*=}${BUILD_FLAG_SUFFIX}" + +} diff --git a/scripts/docker/release.sh b/scripts/docker/release.sh new file mode 100755 index 00000000000..35bc716b750 --- /dev/null +++ b/scripts/docker/release.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Author's Note: Because the structure of this repo is inconsistent (Dockerfiles and build contexts placed willy-nilly) +# we have to trustlist and configure image builds individually because each one is going to be slightly different. +# This is needed as opposed to trusting the structure of the each project to be consistent for every deployable. + +set -eo pipefail + +CLEAR='\033[0m' +RED='\033[0;31m' + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +source ${SCRIPTPATH}/helper.sh + +function usage() { + if [[ -n "$1" ]]; then + echo -e "${RED}☞ $1${CLEAR}\n"; + fi + echo "Usage: $0 [-s service-to-release] [-v service-version] [-n network]" + echo " -s, --service The Service being released to Dockerhub" + echo " -v, --version The version to be used in the docker image tag" + echo " -n, --network The network configuration to use (devnet or mainnet). Default=devnet" + echo " -b, --branch The branch of the mina repository to use for staged docker builds. Default=compatible" + echo " -r, --repo The currently used mina repository" + echo " --deb-codename The debian codename (stretch or buster) to build the docker image from. Default=stretch" + echo " --deb-release The debian package release channel to pull from (unstable,alpha,beta,stable). Default=unstable" + echo " --deb-version The version string for the debian package to install" + echo " --deb-profile The profile string for the debian package to install" + echo " --deb-build-flags The build-flags string for the debian package to install" + echo "" + echo "Example: $0 --service faucet --version v0.1.0" + echo "Valid Services: ${VALID_SERVICES[*]}" + exit 1 +} + +while [[ "$#" -gt 0 ]]; do case $1 in + -s|--service) SERVICE="$2"; shift;; + -v|--version) VERSION="$2"; shift;; + -n|--network) NETWORK="--build-arg network=$2"; shift;; + --deb-codename) DEB_CODENAME="--build-arg deb_codename=$2"; shift;; + --deb-version) DEB_VERSION="--build-arg deb_version=$2"; shift;; + --deb-profile) DEB_PROFILE="$2"; shift;; + --deb-build-flags) DEB_BUILD_FLAGS="$2"; shift;; + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + +export_base_image +export_suffixes +export_docker_tag + +# push to GCR +docker push "${TAG}" + +# retag and push again to GCR +docker tag "${TAG}" "${HASHTAG}" +docker push "${HASHTAG}" + diff --git a/scripts/export-git-env-vars.sh b/scripts/export-git-env-vars.sh new file mode 100755 index 00000000000..c1d08262488 --- /dev/null +++ b/scripts/export-git-env-vars.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -euo pipefail + +set -x + +echo "Exporting Git Variables: " + +git fetch + +function find_most_recent_numeric_tag() { + # We use the --prune flag because we've had problems with buildkite agents getting conflicting results here + git fetch --tags --prune --prune-tags --force + TAG=$(git describe --always --abbrev=0 $1 | sed 's!/!-!g; s!_!-!g; s!#!-!g') + if [[ $TAG != [0-9]* ]]; then + TAG=$(find_most_recent_numeric_tag $TAG~) + fi + echo $TAG +} + +export GITHASH=$(git rev-parse --short=7 HEAD) +export THIS_COMMIT_TAG=$(git tag --points-at HEAD) +export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) + +export GITTAG=$(find_most_recent_numeric_tag HEAD) + +export MINA_DEB_VERSION="${GITTAG}-${GITBRANCH}-${GITHASH}" +export MINA_DOCKER_TAG="$(echo "${MINA_DEB_VERSION}-${MINA_DEB_CODENAME}" | sed 's!/!-!g; s!_!-!g')" + +[[ -n ${THIS_COMMIT_TAG} ]] && export MINA_COMMIT_TAG="${THIS_COMMIT_TAG}" + +echo "after commit tag" \ No newline at end of file diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh deleted file mode 100755 index db063592a64..00000000000 --- a/scripts/rebuild-deb.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash - -# Script collects binaries and keys and builds deb archives. - -set -eo pipefail - -source scripts/deb-builder-helpers.sh - -# always build log proc since it is often an dependency -# TODO: remove logproc as external dependency -build_logproc_deb - -if [ $# -eq 0 ] - then - echo "No arguments supplied. Building all known debian packages" - build_keypair_deb - build_archive_deb - build_batch_txn_deb - build_daemon_berkeley_deb - build_daemon_mainnet_deb - build_daemon_devnet_deb - build_rosetta_berkeley_deb - build_rosetta_mainnet_deb - build_rosetta_devnet_deb - build_test_executive_deb - build_functional_test_suite_deb - build_zkapp_test_transaction_deb - - else - for i in "$@"; do - if [[ $(type -t "build_${i}_deb") == function ]] - then - echo "Building $i debian package" - "build_${i}_deb" - else - echo "invalid debian package name '$i'" - exit 1 - fi - done -fi - -# TODO: Find a way to package keys properly without blocking/locking in CI -# TODO: Keys should be their own package, which this 'non-noprovingkeys' deb depends on -# For now, deleting keys in /tmp/ so that the complicated logic below for moving them short-circuits and both packages are built without keys -rm -rf /tmp/s3_cache_dir /tmp/coda_cache_dir - -# Keys -# Identify actual keys used in build -# NOTE: Moving the keys from /tmp because of storage constraints. This is OK -# because building deb is the last step and therefore keys, genesis ledger, and -# proof are not required in /tmp -echo "Checking PV keys" -mkdir -p "${BUILDDIR}/var/lib/coda" -compile_keys=("step" "vk-step" "wrap" "vk-wrap") -for key in ${compile_keys[*]} -do - echo -n "Looking for keys matching: ${key} -- " - - # Awkward, you can't do a filetest on a wildcard - use loops - for f in /tmp/s3_cache_dir/${key}*; do - if [ -e "$f" ]; then - echo " [OK] found key in s3 key set" - mv /tmp/s3_cache_dir/${key}* "${BUILDDIR}/var/lib/coda/." - break - fi - done - - for f in /var/lib/coda/${key}*; do - if [ -e "$f" ]; then - echo " [OK] found key in stable key set" - mv /var/lib/coda/${key}* "${BUILDDIR}/var/lib/coda/." - break - fi - done - - for f in /tmp/coda_cache_dir/${key}*; do - if [ -e "$f" ]; then - echo " [WARN] found key in compile-time set" - mv /tmp/coda_cache_dir/${key}* "${BUILDDIR}/var/lib/coda/." - break - fi - done -done - -# Build mina block producer sidecar -if ${MINA_BUILD_MAINNET} # only builds on mainnet-like branches -then - ../automation/services/mina-bp-stats/sidecar/build.sh # only builds on mainnet-like branches - rm -rf "${BUILDDIR}" -fi - -if ${MINA_BUILD_MAINNET} # only builds on mainnet-like branches -then - echo "---- Built all packages including mainnet, devnet, and the sidecar" -else - echo "---- Not a mainnet-like branch, only built berkeley and beyond packages" -fi - -ls -lh mina*.deb From eaf7c11fa6c2ee076168cc6fd6812bc0ce2a102d Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 16 Aug 2024 12:29:20 +0200 Subject: [PATCH 05/15] introdude logproc debian package to dhall constants --- buildkite/src/Constants/Artifacts.dhall | 10 +++++++++- buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall | 1 + .../Release/MinaArtifactBullseyeInstrumented.dhall | 6 +++++- .../Jobs/Release/MinaArtifactBullseyeLightnet.dhall | 2 +- buildkite/src/Jobs/Release/MinaArtifactFocal.dhall | 1 + .../src/Jobs/Release/MinaArtifactMainnetBullseye.dhall | 1 + .../src/Jobs/Release/MinaArtifactMainnetFocal.dhall | 1 + .../Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall | 1 + 8 files changed, 20 insertions(+), 3 deletions(-) diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index acd41dff4e2..83d5be7de07 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -11,6 +11,7 @@ let Network = ./Network.dhall let Artifact : Type = < Daemon + | LogProc | Archive | TestExecutive | BatchTxn @@ -21,6 +22,7 @@ let Artifact let AllButTests = [ Artifact.Daemon + , Artifact.LogProc , Artifact.Archive , Artifact.BatchTxn , Artifact.TestExecutive @@ -28,7 +30,8 @@ let AllButTests = , Artifact.ZkappTestTransaction ] -let Main = [ Artifact.Daemon, Artifact.Archive, Artifact.Rosetta ] +let Main = + [ Artifact.Daemon, Artifact.LogProc, Artifact.Archive, Artifact.Rosetta ] let All = AllButTests # [ Artifact.FunctionalTestSuite ] @@ -36,6 +39,7 @@ let capitalName = \(artifact : Artifact) -> merge { Daemon = "Daemon" + , LogProc = "LogProc" , Archive = "Archive" , TestExecutive = "TestExecutive" , BatchTxn = "BatchTxn" @@ -49,6 +53,7 @@ let lowerName = \(artifact : Artifact) -> merge { Daemon = "daemon" + , LogProc = "logproc" , Archive = "archive" , TestExecutive = "test_executive" , BatchTxn = "batch_txn" @@ -64,6 +69,7 @@ let dockerName = { Daemon = "mina-daemon" , Archive = "mina-archive" , TestExecutive = "mina-test-executive" + , LogProc = "mina-logproc" , BatchTxn = "mina-batch-txn" , Rosetta = "mina-rosetta" , ZkappTestTransaction = "mina-zkapp-test-transaction" @@ -76,6 +82,7 @@ let toDebianName = -> \(network : Network.Type) -> merge { Daemon = "daemon_${Network.lowerName network}" + , LogProc = "logproc" , Archive = "archive" , TestExecutive = "test_executive" , BatchTxn = "batch_txn" @@ -101,6 +108,7 @@ let toDebianNames = (\(n : Network.Type) -> toDebianName a n) networks , Archive = [ "archive" ] + , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] , Rosetta = diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall index 3f0af8423fb..d4c2688f721 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall @@ -11,6 +11,7 @@ in Pipeline.build ArtifactPipelines.MinaBuildSpec::{ , artifacts = [ Artifacts.Type.Daemon + , Artifacts.Type.LogProc , Artifacts.Type.Archive , Artifacts.Type.BatchTxn , Artifacts.Type.TestExecutive diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall index 7181e5fd0a5..b36d1324dc3 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall @@ -9,7 +9,11 @@ let Pipeline = ../../Pipeline/Dsl.dhall in Pipeline.build ( ArtifactPipelines.pipeline ArtifactPipelines.MinaBuildSpec::{ - , artifacts = [ Artifacts.Type.Daemon, Artifacts.Type.Archive ] + , artifacts = + [ Artifacts.Type.Daemon + , Artifacts.Type.LogProc + , Artifacts.Type.Archive + ] , buildFlags = BuildFlags.Type.Instrumented } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall index 68ce331394c..7551cb02039 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall @@ -9,7 +9,7 @@ let Pipeline = ../../Pipeline/Dsl.dhall in Pipeline.build ( ArtifactPipelines.pipeline ArtifactPipelines.MinaBuildSpec::{ - , artifacts = [ Artifacts.Type.Daemon ] + , artifacts = [ Artifacts.Type.Daemon, Artifacts.Type.LogProc ] , profile = Profiles.Type.Lightnet } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall index f8269fab58a..492858b50ab 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall @@ -11,6 +11,7 @@ in Pipeline.build ArtifactPipelines.MinaBuildSpec::{ , artifacts = [ Artifacts.Type.Daemon + , Artifacts.Type.LogProc , Artifacts.Type.Archive , Artifacts.Type.BatchTxn , Artifacts.Type.TestExecutive diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall index 687d72aa43d..aa4037d8626 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall @@ -15,6 +15,7 @@ in Pipeline.build ArtifactPipelines.MinaBuildSpec::{ , artifacts = [ Artifacts.Type.Daemon + , Artifacts.Type.LogProc , Artifacts.Type.Archive , Artifacts.Type.BatchTxn , Artifacts.Type.Rosetta diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall index 06b05770530..82d9d49a532 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall @@ -17,6 +17,7 @@ in Pipeline.build ArtifactPipelines.MinaBuildSpec::{ , artifacts = [ Artifacts.Type.Daemon + , Artifacts.Type.LogProc , Artifacts.Type.Archive , Artifacts.Type.BatchTxn , Artifacts.Type.Rosetta diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall index 850cafe0adf..b4d2642bbd4 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall @@ -13,6 +13,7 @@ in Pipeline.build ArtifactPipelines.MinaBuildSpec::{ , artifacts = [ Artifacts.Type.Daemon + , Artifacts.Type.LogProc , Artifacts.Type.Archive , Artifacts.Type.BatchTxn , Artifacts.Type.TestExecutive From 45a8bf2800662340f6764e8d1683bcebb475d7be Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 16 Aug 2024 12:50:46 +0200 Subject: [PATCH 06/15] fix comp --- buildkite/src/Command/MinaArtifact.dhall | 1 + buildkite/src/Constants/Artifacts.dhall | 1 + 2 files changed, 2 insertions(+) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index fe4d203fc50..4fe0ed95ffd 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -174,6 +174,7 @@ let docker_step ) spec.networks , TestExecutive = [] : List DockerImage.ReleaseSpec.Type + , LogProc = [] : List DockerImage.ReleaseSpec.Type , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index 83d5be7de07..8397d0b27a5 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -158,6 +158,7 @@ let dockerTag = "${version_and_codename}-${Network.lowerName network}${profile_part}" , Archive = "${version_and_codename}" + , LogProc = "${version_and_codename}" , TestExecutive = "${version_and_codename}" , BatchTxn = "${version_and_codename}" , Rosetta = "${version_and_codename}" From ec30333ae5775a1904ce5399fac061258401eec2 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 17 Aug 2024 09:45:45 +0200 Subject: [PATCH 07/15] debug --- scripts/debian/builder-helpers.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 76e67d3695c..0e34178731d 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -1,4 +1,7 @@ #!/bin/bash +set -euox pipefail + + BUILD_URL=${BUILD_URL:-"local build from '$(hostname)' host"} MINA_DEB_CODENAME=${MINA_DEB_CODENAME:-"bullseye"} @@ -9,16 +12,12 @@ MINA_DEB_RELEASE=${MINA_DEB_RELEASE:-"unstable"} echo "--- Setting up the envrionment to build debian packages..." -set -euo pipefail - SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" cd "${SCRIPTPATH}/../../_build" GITHASH=$(git rev-parse --short=7 HEAD) GITHASH_CONFIG=$(git rev-parse --short=8 --verify HEAD) -cd "${SCRIPTPATH}/../../_build" - # Set dependencies based on debian release SHARED_DEPS="libssl1.1, libgmp10, libgomp1, tzdata, rocksdb-tools" From c7fbbae3c671438a3c656c1538b75f708d7a50fa Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 21 Aug 2024 19:15:16 +0200 Subject: [PATCH 08/15] use export_base_image --- scripts/docker/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index 5a32ba771d1..1846e9965f2 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -162,6 +162,7 @@ case "${SERVICE}" in ;; esac +export_base_image export_suffixes export_docker_tag From 83696f389debc3dfd1251b41a0494389c1fd0e95 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 21 Aug 2024 23:09:02 +0200 Subject: [PATCH 09/15] delimiter for build and release docker step --- buildkite/src/Command/DockerImage.dhall | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 9c3e7c2a1dc..143d35ce39e 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -99,6 +99,7 @@ let generateStep = ++ " && source ./buildkite/scripts/export-git-env-vars.sh " ++ " && " ++ buildDockerCmd + ++ " && " ++ releaseDockerCmd ) ] @@ -110,6 +111,7 @@ let generateStep = ++ " && source ./buildkite/scripts/export-git-env-vars.sh " ++ " && " ++ buildDockerCmd + ++ " && " ++ releaseDockerCmd ++ " && ./scripts/debian/aptly.sh stop" ) From ff4a9b648631622f113527d19dee48d2a8e5e84a Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 21 Aug 2024 23:14:37 +0200 Subject: [PATCH 10/15] fix release docker params --- buildkite/src/Command/DockerImage.dhall | 2 -- 1 file changed, 2 deletions(-) diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 143d35ce39e..cb7a7f5ddf2 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -82,14 +82,12 @@ let generateStep = ++ " --service ${spec.service}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" - ++ " --branch ${spec.branch}" ++ " --deb-codename ${spec.deb_codename}" ++ " --deb-release ${spec.deb_release}" ++ " --deb-version ${spec.deb_version}" ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ " --deb-build-flags ${BuildFlags.lowerName spec.build_flags}" - ++ " --repo ${spec.repo}" let commands = merge From e273e63cdff34e3c26bdfeff737f25b4854b2285 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 21 Aug 2024 23:22:25 +0200 Subject: [PATCH 11/15] one more fix --- buildkite/src/Command/DockerImage.dhall | 1 - 1 file changed, 1 deletion(-) diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index cb7a7f5ddf2..cc5131772c4 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -83,7 +83,6 @@ let generateStep = ++ " --version ${spec.version}" ++ " --network ${spec.network}" ++ " --deb-codename ${spec.deb_codename}" - ++ " --deb-release ${spec.deb_release}" ++ " --deb-version ${spec.deb_version}" ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ " --deb-build-flags ${BuildFlags.lowerName From bfbf4296ec7c09b54904772aaba4159ab2188564 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 22 Aug 2024 19:10:16 +0200 Subject: [PATCH 12/15] Add some docs + fix mina-daemon suffix --- Makefile | 8 +++----- README-dev.md | 41 +++++++++++++++++++++++++++++++++++++++ buildkite/src/README.md | 4 ++-- dev/Makefile | 8 ++++---- dev/README.md | 2 +- dev/docker-compose.yml | 4 +--- scripts/docker/build.sh | 4 +--- scripts/docker/helper.sh | 8 ++++++++ scripts/docker/release.sh | 6 +++++- 9 files changed, 66 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 8db8d616f49..4ec0ad34bbb 100644 --- a/Makefile +++ b/Makefile @@ -74,9 +74,7 @@ endif ocaml_checks: switch ocaml_version ocaml_word_size check_opam_switch libp2p_helper: -ifeq (, $(MINA_LIBP2P_HELPER_PATH)) make -C src/app/libp2p_helper -endif genesis_ledger: ocaml_checks $(info Building runtime_genesis_ledger) @@ -89,12 +87,12 @@ check: ocaml_checks libp2p_helper build: ocaml_checks reformat-diff libp2p_helper $(info Starting Build) - (ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe --profile=$(DUNE_PROFILE) + (ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe src/app/generate_keypair/generate_keypair.exe src/app/validate_keypair/validate_keypair.exe src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --profile=$(DUNE_PROFILE) $(info Build complete) -build_all_sigs: ocaml_checks reformat-diff libp2p_helper +build_all_sigs: ocaml_checks reformat-diff libp2p_helper build $(info Starting Build) - (ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe src/app/cli/src/mina_testnet_signatures.exe src/app/cli/src/mina_mainnet_signatures.exe --profile=$(DUNE_PROFILE) + (ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/cli/src/mina_testnet_signatures.exe src/app/cli/src/mina_mainnet_signatures.exe --profile=$(DUNE_PROFILE) $(info Build complete) build_archive: ocaml_checks reformat-diff diff --git a/README-dev.md b/README-dev.md index 2fec130e83b..a245dca2a13 100644 --- a/README-dev.md +++ b/README-dev.md @@ -250,6 +250,47 @@ dune exec src/app/cli/src/mina.exe -- -help The command line help is the place to learn about other options to the Mina CLI and how to connect to an existing network, such as Mainnet. +## Building debian package locally + +Debian package can be build locally by using below commands: + + +1. Build binaries +``` +make build +``` +2. Build debian for mina-devet (example for ubuntu 18.04): +``` +./scripts/debian/build.sh daemon_devnet +``` + +## Building docker locally + +Prerequisites: + +- debian package previously built +- aptly app + +Steps: + +1. Start local debian repository +``` +./scripts/debian/aptly.sh start -b -c focal -d _build/ -m unstable -l -p 8081 +``` + +IMPORTANT: debians should be placed in _build folder + +2. Build docker: +``` +./scripts/docker/build.sh --service mina-daemon -v 3.0.0-dkijania-local-debian-build-a099fc7 --network devnet --deb-codename focal --deb-version 3.0.0-dkijania-local-debian-build-a099fc7 +``` + +Where: + +`-v` - base docker tag +`--deb-codename` - input debian codename (buster,bullseye etc.) +`--deb-version` - version of debian which docker will host + ## Using the Makefile The Makefile contains placeholder targets for all the common tasks that need to be done and automatically knows how to use Docker. diff --git a/buildkite/src/README.md b/buildkite/src/README.md index 6299bad1610..e8bfe284ddb 100644 --- a/buildkite/src/README.md +++ b/buildkite/src/README.md @@ -146,9 +146,9 @@ Above definition one need to paste into steps edit box for given pipeline and th All list of available parameters: -- DEBIANS - The comma delimitered debian names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/DebianPackage.dhall` files. Only CamelCase format is supported +- DEBIANS - The comma delimited debian names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/DebianPackage.dhall` files. Only CamelCase format is supported -- DOCKERS - The comma delimitered docker names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/Artifacts.dhall` files. Only CamelCase format is supported +- DOCKERS - The comma delimited docker names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/Artifacts.dhall` files. Only CamelCase format is supported - CODENAMES - The Debian codenames `Bullseye,Focal`. All available names are located in `buildkite/src/Constans/DebianVersions.dhall`. Only CamelCase format is supported diff --git a/dev/Makefile b/dev/Makefile index 8e1f1764620..dfb72c5f38a 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -1,10 +1,10 @@ all: - docker-compose up + docker compose up rebuild: - docker-compose down --volumes - docker-compose build --no-cache - docker-compose up + docker compose down --volumes + docker compose build --no-cache + docker compose up ssh: docker exec -it -w "/mina" mina /bin/bash diff --git a/dev/README.md b/dev/README.md index 2ff5ad2eca5..205c4cf0bb4 100644 --- a/dev/README.md +++ b/dev/README.md @@ -4,7 +4,7 @@ To build your changes using docker, you can use the following flow: - in one window run `make` - in another one run `make ssh` to have a shell into the container - + don't forget to run `./scripts/setup-ocaml.sh` to set up opam/ocaml/etc. + + don't forget to run `sudo chown -R opam .` to set up opam/ocaml/etc. + don't forget to run `eval $(opam config env)` as well + `make build` should work after that :) diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index cfc2391cdb3..a4c49483d74 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -3,9 +3,7 @@ version: "3.9" services: mina: container_name: mina - build: - context: ../dockerfiles - dockerfile: Dockerfile-toolchain + image: gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f volumes: - "..:/mina" # mount the mina folder so that changes are reflected in the container - "_opam:/mina/_opam" # exclude any existing local switch diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index 1846e9965f2..810aae26e49 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -124,7 +124,6 @@ case "${SERVICE}" in mina-daemon) DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-daemon" DOCKER_CONTEXT="dockerfiles/" - VERSION="${VERSION}-${NETWORK##*=}" ;; mina-toolchain) DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-opam-deps dockerfiles/stages/3-toolchain" @@ -132,7 +131,6 @@ case "${SERVICE}" in mina-batch-txn) DOCKERFILE_PATH="dockerfiles/Dockerfile-txn-burst" DOCKER_CONTEXT="dockerfiles/" - VERSION="${VERSION}-${NETWORK##*=}" ;; mina-rosetta) DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-rosetta" @@ -162,8 +160,8 @@ case "${SERVICE}" in ;; esac +export_version export_base_image -export_suffixes export_docker_tag BUILD_NETWORK="--network=host" diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 6a6a1af86dc..4f256ed54bb 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -16,6 +16,14 @@ function export_base_image () { export IMAGE="--build-arg image=${IMAGE}" } +function export_version () { + export VERSION="${VERSION}-${DEB_CODENAME##*=}" + case "${SERVICE}" in + mina-daemon|mina-batch-txn) export VERSION="${VERSION}-${NETWORK##*=}" ;; + *) ;; +esac +} + function export_suffixes () { # Determine suffix for mina name. Suffix is combined from profile and service name # Possible outcomes: diff --git a/scripts/docker/release.sh b/scripts/docker/release.sh index 35bc716b750..d198725efaf 100755 --- a/scripts/docker/release.sh +++ b/scripts/docker/release.sh @@ -41,13 +41,17 @@ while [[ "$#" -gt 0 ]]; do case $1 in --deb-version) DEB_VERSION="--build-arg deb_version=$2"; shift;; --deb-profile) DEB_PROFILE="$2"; shift;; --deb-build-flags) DEB_BUILD_FLAGS="$2"; shift;; + --help) usage; exit 0;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done +export_version export_base_image -export_suffixes export_docker_tag +echo tag ${TAG} +echo hash ${HASHTAG} + # push to GCR docker push "${TAG}" From befd7a29c2e59496cfe96f24ec4731e30af0de45 Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Thu, 22 Aug 2024 20:41:43 +0200 Subject: [PATCH 13/15] Update helper.sh --- scripts/docker/helper.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 4f256ed54bb..20b7bca95af 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -17,7 +17,6 @@ function export_base_image () { } function export_version () { - export VERSION="${VERSION}-${DEB_CODENAME##*=}" case "${SERVICE}" in mina-daemon|mina-batch-txn) export VERSION="${VERSION}-${NETWORK##*=}" ;; *) ;; From d3a11010a6c0c1931d4d5931bf564abfee408f2f Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Thu, 22 Aug 2024 21:21:48 +0200 Subject: [PATCH 14/15] Update build.sh --- scripts/docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index 810aae26e49..53364f471ce 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -44,7 +44,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --deb-codename) INPUT_CODENAME="$2"; shift;; --deb-release) INPUT_RELEASE="$2"; shift;; --deb-version) INPUT_VERSION="$2"; shift;; - --deb-profile) INPUT_PROFILE="$2"; shift;; + --deb-profile) DEB_PROFILE="$2"; shift;; --deb-repo) INPUT_REPO="$2"; shift;; --deb-build-flags) DEB_BUILD_FLAGS="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; From 0b21989fc86908b9f7a729483fc62827dd384756 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 26 Aug 2024 11:01:40 +0200 Subject: [PATCH 15/15] revert ifreq from libp2p_helper --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 4ec0ad34bbb..7fc681d4c91 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,9 @@ endif ocaml_checks: switch ocaml_version ocaml_word_size check_opam_switch libp2p_helper: +ifeq (, $(MINA_LIBP2P_HELPER_PATH)) make -C src/app/libp2p_helper +endif genesis_ledger: ocaml_checks $(info Building runtime_genesis_ledger)