From 6c51bac2ae675715daea399e4b35117a15b8da02 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 23 Jul 2024 17:31:23 +0200 Subject: [PATCH 1/3] nightly on develop cleanup. Marking HF test as optional and removing delegation backend unit test --- .../Jobs/Test/DelegationBackendUnitTest.dhall | 48 ------------ buildkite/src/Jobs/Test/HFTest.dhall | 74 ++++++++++--------- 2 files changed, 39 insertions(+), 83 deletions(-) delete mode 100644 buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall diff --git a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall b/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall deleted file mode 100644 index c9d2d787fc8..00000000000 --- a/buildkite/src/Jobs/Test/DelegationBackendUnitTest.dhall +++ /dev/null @@ -1,48 +0,0 @@ -let ContainerImages = ../../Constants/ContainerImages.dhall -let B = ../../External/Buildkite.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - -let Cmd = ../../Lib/Cmds.dhall -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Command = ../../Command/Base.dhall -let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall - -let B = ../../External/Buildkite.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - -in - -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src/app/delegation_backend"), - S.exactly "Makefile" "", - S.exactly "buildkite/src/Jobs/Test/DelegationBackendUnitTest" "dhall" - ], - path = "Test", - name = "DelegationBackendUnitTest", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.runInDocker Cmd.Docker::{image = ContainerImages.delegationBackendToolchain} "cd src/app/delegation_backend && mkdir -p result && cp -R /headers result && cd src/delegation_backend && go test" - ], - label = "delegation backend unit-tests", - soft_fail = Some (B/SoftFail.Boolean True), - key = "delegation-backend-unit-tests", - target = Size.Small, - docker = None Docker.Type - } - ] - } diff --git a/buildkite/src/Jobs/Test/HFTest.dhall b/buildkite/src/Jobs/Test/HFTest.dhall index af91ea30d8f..31d2f0b6df2 100644 --- a/buildkite/src/Jobs/Test/HFTest.dhall +++ b/buildkite/src/Jobs/Test/HFTest.dhall @@ -13,40 +13,44 @@ let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall -in +let B = ../../External/Buildkite.dhall -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/HFTest" "dhall", - S.strictlyStart (S.contains "scripts/hardfork"), - S.strictlyStart (S.contains "nix"), - S.exactly "flake" "nix", - S.exactly "flake" "lock", - S.exactly "default" "nix" - ], - path = "Test", - name = "HFTest", - mode = PipelineMode.Type.Stable, - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.runInDocker Cmd.Docker::{ - image = ContainerImages.nixos, - privileged = True - } "./scripts/hardfork/build-and-test.sh $BUILDKITE_BRANCH" - ], - label = "hard fork test", - key = "hard-fork-test", - target = Size.Small, - docker = None Docker.Type, - timeout_in_minutes = Some 420 +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/HFTest" "dhall" + , S.strictlyStart (S.contains "scripts/hardfork") + , S.strictlyStart (S.contains "nix") + , S.exactly "flake" "nix" + , S.exactly "flake" "lock" + , S.exactly "default" "nix" + ] + , path = "Test" + , name = "HFTest" + , mode = PipelineMode.Type.Stable + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.runInDocker + Cmd.Docker::{ + , image = ContainerImages.nixos + , privileged = True + } + "./scripts/hardfork/build-and-test.sh \$BUILDKITE_BRANCH" + ] + , label = "hard fork test" + , key = "hard-fork-test" + , target = Size.Small + , soft_fail = Some (B/SoftFail.Boolean True) + , docker = None Docker.Type + , timeout_in_minutes = Some +420 + } + ] + } From 146ce26beb83d69e94429b7d32ee51d98f325c73 Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Tue, 23 Jul 2024 18:42:48 +0200 Subject: [PATCH 2/3] Merge pull request #15858 from MinaProtocol/terraform-tests Fix Terraform tests by synthetically creating required seed peerid files --- .../terraform/modules/o1-testnet/files.tf | 19 ++++++++++++++++--- .../terraform/modules/o1-testnet/inputs.tf | 7 +++++++ automation/terraform/testnets/ci-net/main.tf | 8 ++++++++ buildkite/scripts/terraform-test.sh | 2 +- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/automation/terraform/modules/o1-testnet/files.tf b/automation/terraform/modules/o1-testnet/files.tf index 7eb37815c96..8f39b9982e3 100644 --- a/automation/terraform/modules/o1-testnet/files.tf +++ b/automation/terraform/modules/o1-testnet/files.tf @@ -15,10 +15,23 @@ # ] # } +# Generate random strings for peer IDs +resource "random_id" "peer_id" { + for_each = var.create_libp2p_files ? toset(local.seed_names) : [] + byte_length = 32 +} + +# Create the files with the generated peer IDs +resource "local_file" "libp2p_seed_peers" { + for_each = var.create_libp2p_files ? toset(local.seed_names) : [] + filename = "${var.artifact_path}/keys/libp2p-keys/${each.key}.peerid" + content = random_id.peer_id[each.key].hex +} + data "local_file" "libp2p_seed_peers" { for_each = toset(local.seed_names) filename = "${var.artifact_path}/keys/libp2p-keys/${each.key}.peerid" - # depends_on = [ - # null_resource.block_producer_key_generation - # ] + depends_on = [ + local_file.libp2p_seed_peers + ] } diff --git a/automation/terraform/modules/o1-testnet/inputs.tf b/automation/terraform/modules/o1-testnet/inputs.tf index 9841807a9eb..117f18e0e13 100644 --- a/automation/terraform/modules/o1-testnet/inputs.tf +++ b/automation/terraform/modules/o1-testnet/inputs.tf @@ -30,6 +30,13 @@ variable "artifact_path" { default = "/tmp" } +# Variable to control file creation +variable "create_libp2p_files" { + description = "Whether to create LibP2P peer ID files" + type = bool + default = false +} + variable "mina_image" { type = string default = "gcr.io/o1labs-192920/mina-daemon:1.2.0beta8-5b35b27-devnet" diff --git a/automation/terraform/testnets/ci-net/main.tf b/automation/terraform/testnets/ci-net/main.tf index 84bb70c2eb7..f9a41019e43 100644 --- a/automation/terraform/testnets/ci-net/main.tf +++ b/automation/terraform/testnets/ci-net/main.tf @@ -62,6 +62,12 @@ variable "ci_artifact_path" { default = "/tmp" } +variable "create_libp2p_files" { + type = bool + description = "Whether to create LibP2P peer ID files" + default = false +} + locals { seed_region = "us-west1" seed_zone = "us-west1-b" @@ -73,6 +79,8 @@ module "ci_testnet" { artifact_path = var.ci_artifact_path + create_libp2p_files = var.create_libp2p_files + # TODO: remove obsolete cluster_name var + cluster region cluster_name = "mina-integration-west1" cluster_region = var.ci_cluster_region diff --git a/buildkite/scripts/terraform-test.sh b/buildkite/scripts/terraform-test.sh index c117898cfa0..d5da2567f7e 100755 --- a/buildkite/scripts/terraform-test.sh +++ b/buildkite/scripts/terraform-test.sh @@ -5,7 +5,7 @@ function test_single_terraform_config { cd $1 terraform init - terraform plan + terraform plan -var="create_libp2p_files=true" RET=$? From 2cba6435b72ca5fa7325757e0057a7412f2bea37 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 23 Jul 2024 21:40:46 +0200 Subject: [PATCH 3/3] fix type for timeout_in_minutes --- buildkite/src/Jobs/Test/HFTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/HFTest.dhall b/buildkite/src/Jobs/Test/HFTest.dhall index 31d2f0b6df2..feb2f44267b 100644 --- a/buildkite/src/Jobs/Test/HFTest.dhall +++ b/buildkite/src/Jobs/Test/HFTest.dhall @@ -50,7 +50,7 @@ in Pipeline.build , target = Size.Small , soft_fail = Some (B/SoftFail.Boolean True) , docker = None Docker.Type - , timeout_in_minutes = Some +420 + , timeout_in_minutes = Some 420 } ] }