Merge pull request #15977 from MinaProtocol/merge_compatible_into_dev…

…elop_26AUG24

Merge compatible into develop 26 aug24

Makefile

@@ -89,22 +89,22 @@ check: ocaml_checks libp2p_helper
 
 build: ocaml_checks reformat-diff libp2p_helper
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe src/app/generate_keypair/generate_keypair.exe src/app/validate_keypair/validate_keypair.exe src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
-build_all_sigs: ocaml_checks reformat-diff libp2p_helper
+build_all_sigs: ocaml_checks reformat-diff libp2p_helper build
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe src/app/cli/src/mina_testnet_signatures.exe src/app/cli/src/mina_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/cli/src/mina_testnet_signatures.exe src/app/cli/src/mina_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
 build_archive: ocaml_checks reformat-diff
 	$(info Starting Build)
 	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
-build_archive_all_sigs: ocaml_checks reformat-diff
+build_archive_utils: ocaml_checks reformat-diff
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/archive/archive_testnet_signatures.exe src/app/archive/archive_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/replayer/replayer.exe src/app/archive_blocks/archive_blocks.exe src/app/extract_blocks/extract_blocks.exe src/app/missing_blocks_auditor/missing_blocks_auditor.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
 build_rosetta: ocaml_checks
@@ -208,14 +208,7 @@ publish-macos:
 	@./scripts/publish-macos.sh
 
 deb:
-	./scripts/rebuild-deb.sh
-	./scripts/archive/build-release-archives.sh
-	@mkdir -p /tmp/artifacts
-	@cp _build/mina*.deb /tmp/artifacts/.
-
-deb_optimized:
-	./scripts/rebuild-deb.sh "optimized"
-	./scripts/archive/build-release-archives.sh
+	./scripts/debian/builder.sh
 	@mkdir -p /tmp/artifacts
 	@cp _build/mina*.deb /tmp/artifacts/.
 
@@ -229,14 +222,12 @@ build_or_download_pv_keys: ocaml_checks
 	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune exec --profile=$(DUNE_PROFILE) src/lib/snark_keys/gen_keys/gen_keys.exe -- --generate-keys-only
 	$(info Keys built)
 
-publish_debs:
-	@./buildkite/scripts/publish-deb.sh
-
 genesiskeys:
 	@mkdir -p /tmp/artifacts
 	@cp _build/default/src/lib/key_gen/sample_keypairs.ml /tmp/artifacts/.
 	@cp _build/default/src/lib/key_gen/sample_keypairs.json /tmp/artifacts/.
 
+
 ##############################################
 ## Genesis ledger in OCaml from running daemon
 

README-ci-failures.md

@@ -29,8 +29,8 @@ details and links to the failures.
 ## CI environment mismatch
 
 The CI runs its jobs in multiple Docker images. The images that it is using are
-specified in `buildkite/src/Constants/COntainerImages.dhall`: the CI uses all
-Debian images prefixed by `minaToolchainBuster`.
+specified in `buildkite/src/Constants/ContainerImages.dhall`: the CI uses all
+Debian images prefixed by `minaToolchainBullseye`.
 
 These images are generated by the CI itself, in particular based on the content of the
 `dockerfiles` directory and the `opam.export` file (which describes versions of
@@ -46,7 +46,7 @@ In this case, you should:
    [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain)
    where they have been automatically uploaded. Look for your branch name and
    commit hash in the second column. You should find several images suffixed
-   with `-buster`, `-bullseye`, `-stretch` and `-focal` (Debian versions).
+   with `-bullseye` and `-focal` (Debian versions).
 3. For each such image, retrieve its full name and hash by hovering its link and
    clicking the `Copy full image name` tooltip that appears (or retrieve it on
    the image's page).

README-dev.md

@@ -250,6 +250,47 @@ dune exec src/app/cli/src/mina.exe -- -help
 
 The command line help is the place to learn about other options to the Mina CLI and how to connect to an existing network, such as Mainnet.
 
+## Building debian package locally
+
+Debian package can be build locally by using below commands:
+
+
+1. Build binaries
+```
+make build
+```
+2. Build debian for mina-devet (example for ubuntu 18.04):
+```
+./scripts/debian/build.sh daemon_devnet
+```
+
+## Building docker locally
+
+Prerequisites: 
+
+- debian package previously built
+- aptly app
+
+Steps:
+
+1. Start local debian repository
+```
+./scripts/debian/aptly.sh start -b -c focal -d _build/ -m unstable -l -p 8081
+```
+
+IMPORTANT: debians should be placed in _build folder
+
+2. Build docker:
+```
+./scripts/docker/build.sh --service mina-daemon -v 3.0.0-dkijania-local-debian-build-a099fc7 --network devnet --deb-codename focal --deb-version 3.0.0-dkijania-local-debian-build-a099fc7 
+```
+
+Where:
+
+`-v` - base docker tag
+`--deb-codename` - input debian codename (buster,bullseye etc.)
+`--deb-version` - version of debian which docker will host
+
 ## Using the Makefile
 
 The Makefile contains placeholder targets for all the common tasks that need to be done and automatically knows how to use Docker.

automation/terraform/modules/services/daemon/README.md

@@ -41,17 +41,31 @@ The manual commands to release each container are the following:
 ### Mina-Daemon Container
 
 *(From the root of the `MinaProtocol/mina` repository)*
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<DEB_VERSION> --build-arg deb_release=<mina package release channel, e.g. alpha>"`
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 Example:
-`./scripts/release-docker.sh -s mina-daemon -v 0.0.10-beta4 --extra-args "--build-arg deb_version=0.0.10-beta4-fff3b856 --build-arg deb_release=alpha`
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `'deb_version` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
 ### daemon Container
 
-*(From the root of the `MinaProtocol/mina/automation` folder in the mina repository)*
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step> "`
+*(From the root of the `MinaProtocol/mina` repository)*
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 Example:
-`./scripts/release-docker.sh -s daemon -v 0.0.10-beta4 --extra-args "--build-arg base_image_tag=0.0.10-beta4"`
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`

automation/terraform/modules/services/faucet/README.md

@@ -63,14 +63,17 @@ The manual commands to release each container are the following:
 ### Mina-Daemon Container
 
 *(From the root of the `MinaProtocol/mina` repository)*
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
 
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<deb_version>"`
+and 
 
-### daemon Container
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
 
-*(From the root of the `MinaProtocol/mina/automation` repository)*
 
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step>"`
+Example:
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `base_image_tag` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
@@ -80,5 +83,10 @@ The Faucet Dockerfile lives in the `MinaProtocol/mina` repository [here](https:/
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s bot -v <major>.<minor>.<patch>`
+```
+./scripts/docker/build.sh -s bot -v <major>.<minor>.<patch>
+./scripts/docker/release.sh -s bot -v <major>.<minor>.<patch>
+```
+
+
 

automation/terraform/modules/services/graphql-proxy/README.md

@@ -54,13 +54,18 @@ The manual commands to release each container are the following:
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<MINA_VERSION>"`
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<MINA_VERSION>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 ### daemon Container
 
 *(From the root of the `MinaProtocol/mina/automation` repository)*
 
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step>"`
+`./scripts/docker/release.sh -s daemon -v <major>.<minor>.<patch> `
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `base_image_tag` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
@@ -70,4 +75,4 @@ The Faucet Dockerfile lives in the `MinaProtocol/mina` repository [here](https:/
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s graphql-public-proxy -v <major>.<minor>.<patch>`
+`./scripts/docker/release.sh -s graphql-public-proxy -v <major>.<minor>.<patch>`

buildkite/scripts/build-release.sh

@@ -14,7 +14,7 @@ echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkele
 
 
 echo "--- Prepare debian packages"
-./scripts/rebuild-deb.sh $@
+./scripts/debian/build.sh $@
 
 echo "--- Git diff after build is complete:"
 git diff --exit-code -- .

buildkite/scripts/promote-deb.sh → buildkite/scripts/debian/promote.sh

@@ -56,7 +56,7 @@ echo "Promoting debs: ${PACKAGE}_${VERSION} to Release: ${TO_COMPONENT} and Code
 if [[ -z "$NEW_VERSION" ]] || [[ "$NEW_VERSION" == "$VERSION" ]]; then
   deb-s3 copy --s3-region=us-west-2 --lock --bucket packages.o1test.net --preserve-versions --cache-control=max-age=120  $PACKAGE $CODENAME $TO_COMPONENT --versions $VERSION --arch $ARCH --component ${FROM_COMPONENT} --codename ${CODENAME}
 else
-  source scripts/reversion-deb.sh \
+  source scripts/debian/reversion.sh \
     --deb $PACKAGE  \
     --codename $CODENAME \
     --new-release $TO_COMPONENT \

buildkite/scripts/publish-deb.sh → buildkite/scripts/debian/publish.sh

@@ -2,7 +2,7 @@
 set -eox pipefail
 
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
-source "${SCRIPTPATH}/export-git-env-vars.sh"
+source "${SCRIPTPATH}/../export-git-env-vars.sh"
 
 DOWNLOAD_FOLDER=_build
 
@@ -15,7 +15,7 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then
     exit 0
 fi
 
-source scripts/publish-deb.sh \
+source scripts/debian/publish.sh \
   --names "${DOWNLOAD_FOLDER}/mina-*.deb" \
   --release $MINA_DEB_RELEASE \
   --version $MINA_DEB_VERSION \

buildkite/scripts/run_promote_build_job.sh

@@ -18,7 +18,7 @@
 #          - "NETWORK=Devnet"
 #          - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e"
 #          - "NEW_VERSION=3.0.0fake-ddb6fc4"
-#          - "CODENAMES=Focal,Buster,Bullseye"
+#          - "CODENAMES=Focal,Bullseye"
 #          - "FROM_CHANNEL=Unstable"
 #          - "TO_CHANNEL=Experimental"
 #        image: codaprotocol/ci-toolchain-base:v3
@@ -41,7 +41,7 @@ function usage() {
   fi
   echo "  DEBIANS                     The comma delimitered debian names. For example: 'Daemon,Archive' "
   echo "  DOCKERS                     The comma delimitered docker names. For example: 'Daemon,Archive' "
-  echo "  CODENAMES                   The Debian codenames (Bullseye, Buster etc.)"
+  echo "  CODENAMES                   The Debian codenames (Bullseye, Focal etc.)"
   echo "  FROM_VERSION                The Source Docker or Debian version "
   echo "  NEW_VERSION                 The new Debian version or new Docker tag"
   echo "  REMOVE_PROFILE_FROM_NAME    Should we remove profile suffix from debian name"

buildkite/scripts/run_verify_promoted_build_job.sh

@@ -18,7 +18,7 @@
 #          - "NETWORK=Devnet"
 #          - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e"
 #          - "NEW_VERSION=3.0.0fake-ddb6fc4"
-#          - "CODENAMES=Focal,Buster,Bullseye"
+#          - "CODENAMES=Focal,Bullseye"
 #          - "FROM_CHANNEL=Unstable"
 #          - "TO_CHANNEL=Experimental"
 #        image: codaprotocol/ci-toolchain-base:v3
@@ -41,7 +41,7 @@ function usage() {
   fi
   echo "  DEBIANS                     The comma delimitered debian names. For example: 'Daemon,Archive' "
   echo "  DOCKERS                     The comma delimitered docker names. For example: 'Daemon,Archive' "
-  echo "  CODENAMES                   The Debian codenames (Bullseye, Buster etc.)"
+  echo "  CODENAMES                   The Debian codenames (Bullseye, Focal etc.)"
   echo "  NEW_VERSION                 The new Debian version or new Docker tag"
   echo "  REMOVE_PROFILE_FROM_NAME    Should we remove profile suffix from debian name"
   echo "  PROFILE                     The Docker and Debian profile (Standard, Lightnet)"

buildkite/src/Command/DockerImage.dhall

@@ -26,13 +26,13 @@ let ReleaseSpec =
           , version : Text
           , branch : Text
           , repo : Text
+          , no_cache : Bool
           , deb_codename : Text
           , deb_release : Text
           , deb_version : Text
           , deb_profile : Profiles.Type
           , deb_repo : DebianRepo.Type
           , build_flags : BuildFlags.Type
-          , extra_args : Text
           , step_key : Text
           , if : Optional B/If
           }
@@ -49,7 +49,7 @@ let ReleaseSpec =
           , deb_profile = Profiles.Type.Standard
           , build_flags = BuildFlags.Type.None
           , deb_repo = DebianRepo.Type.PackagesO1Test
-          , extra_args = ""
+          , no_cache = False
           , step_key = "daemon-standard-docker-image"
           , if = None B/If
           }
@@ -59,12 +59,15 @@ let generateStep =
           \(spec : ReleaseSpec.Type)
       ->  let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}"
 
+          let maybeCacheOption = if spec.no_cache then "--no-cache" else ""
+
           let buildDockerCmd =
-                    "./scripts/release-docker.sh"
+                    "./scripts/docker/build.sh"
                 ++  " --service ${spec.service}"
                 ++  " --version ${spec.version}"
                 ++  " --network ${spec.network}"
                 ++  " --branch ${spec.branch}"
+                ++  " ${maybeCacheOption} "
                 ++  " --deb-codename ${spec.deb_codename}"
                 ++  " --deb-repo ${DebianRepo.address spec.deb_repo}"
                 ++  " --deb-release ${spec.deb_release}"
@@ -73,7 +76,17 @@ let generateStep =
                 ++  " --deb-build-flags ${BuildFlags.lowerName
                                             spec.build_flags}"
                 ++  " --repo ${spec.repo}"
-                ++  " --extra-args \\\"${spec.extra_args}\\\""
+
+          let releaseDockerCmd =
+                    "./scripts/docker/release.sh"
+                ++  " --service ${spec.service}"
+                ++  " --version ${spec.version}"
+                ++  " --network ${spec.network}"
+                ++  " --deb-codename ${spec.deb_codename}"
+                ++  " --deb-version ${spec.deb_version}"
+                ++  " --deb-profile ${Profiles.lowerName spec.deb_profile}"
+                ++  " --deb-build-flags ${BuildFlags.lowerName
+                                            spec.build_flags}"
 
           let commands =
                 merge
@@ -83,6 +96,8 @@ let generateStep =
                           ++  " && source ./buildkite/scripts/export-git-env-vars.sh "
                           ++  " && "
                           ++  buildDockerCmd
+                          ++  " && "
+                          ++  releaseDockerCmd
                         )
                     ]
                   , Local =
@@ -93,6 +108,8 @@ let generateStep =
                           ++  " && source ./buildkite/scripts/export-git-env-vars.sh "
                           ++  " && "
                           ++  buildDockerCmd
+                          ++  " && "
+                          ++  releaseDockerCmd
                           ++  " && ./scripts/debian/aptly.sh stop"
                         )
                     ]

buildkite/src/Command/HardforkPackageGeneration.dhall

@@ -125,8 +125,8 @@ let pipeline
                             )
                             "./buildkite/scripts/build-hardfork-package.sh"
                         # [ Cmd.run
-                              "./buildkite/scripts/upload-deb-to-gs.sh ${DebianVersions.lowerName
-                                                                           debVersion}"
+                              "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName
+                                                                              debVersion}"
                           ]
                     , label =
                         "Build Mina Hardfork Package for ${DebianVersions.capitalName
@@ -144,7 +144,7 @@ let pipeline
                           , "MINA_DEB_CODENAME=${DebianVersions.lowerName
                                                    debVersion}"
                           ]
-                          "./buildkite/scripts/publish-deb.sh"
+                          "./buildkite/scripts/debian/publish.sh"
                     , label =
                         "Publish Mina for ${DebianVersions.capitalName
                                               debVersion} Hardfork"