diff --git a/buildkite/src/Command/Base.dhall b/buildkite/src/Command/Base.dhall index 57266a40479..ce24c09acd2 100644 --- a/buildkite/src/Command/Base.dhall +++ b/buildkite/src/Command/Base.dhall @@ -1,254 +1,309 @@ -- Commands are the individual command steps that CI runs let Prelude = ../External/Prelude.dhall + let B = ../External/Buildkite.dhall let Map = Prelude.Map + let List/map = Prelude.List.map + let List/concat = Prelude.List.concat + let Optional/map = Prelude.Optional.map + let Optional/toList = Prelude.Optional.toList let B/Plugins/Partial = B.definitions/commandStep/properties/plugins/Type --- Retry bits + let B/ExitStatus = B.definitions/automaticRetry/properties/exit_status/Type + let B/AutoRetryChunk = B.definitions/automaticRetry/Type.Type -let B/Retry = B.definitions/commandStep/properties/retry/properties/automatic/Type + +let B/Retry = + B.definitions/commandStep/properties/retry/properties/automatic/Type + let B/Manual = B.definitions/commandStep/properties/retry/properties/manual/Type --- Job requirement/flake mgmt bits let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + let B/Skip = B.definitions/commandStep/properties/skip/Type let B/If = B.definitions/commandStep/properties/if/Type let Cmd = ../Lib/Cmds.dhall + let Decorate = ../Lib/Decorate.dhall + let SelectFiles = ../Lib/SelectFiles.dhall let Docker = ./Docker/Type.dhall + let DockerLogin = ./DockerLogin/Type.dhall -let Summon= ./Summon/Type.dhall + +let Summon = ./Summon/Type.dhall + let Size = ./Size.dhall --- If you are adding a new type of plugin, stick it here let Plugins = - < Docker : Docker.Type - | DockerLogin : DockerLogin.Type - | Summon : Summon.Type - > + < Docker : Docker.Type + | DockerLogin : DockerLogin.Type + | Summon : Summon.Type + > let B/Command = B.definitions/commandStep/Type Text Text Plugins Plugins + let B/Plugins = B/Plugins/Partial Plugins Plugins --- Depends on takes several layers of unions, but we can just choose the most --- general of them let B/DependsOn = - let OuterUnion/Type = B.definitions/dependsOn/Type - let InnerUnion/Type = B.definitions/dependsOn/union/Type - in - { Type = OuterUnion/Type, - depends = - \(keys : List Text) -> - OuterUnion/Type.ListDependsOn/Type - (List/map Text InnerUnion/Type (\(k: Text) -> InnerUnion/Type.DependsOn/Type { allow_failure = Some False, step = Some k }) keys) - } + let OuterUnion/Type = B.definitions/dependsOn/Type + + let InnerUnion/Type = B.definitions/dependsOn/union/Type + + in { Type = OuterUnion/Type + , depends = + \(keys : List Text) + -> OuterUnion/Type.ListDependsOn/Type + ( List/map + Text + InnerUnion/Type + ( \(k : Text) + -> InnerUnion/Type.DependsOn/Type + { allow_failure = Some False, step = Some k } + ) + keys + ) + } let B/ArtifactPaths = B.definitions/commandStep/properties/artifact_paths/Type -let B/Env = B.definitions/commandStep/properties/env/Type - --- A type to make sure we don't accidentally forget the prefix on keys -let TaggedKey = { - Type = { - name : Text, - key : Text - }, - default = {=} -} - -let ExitStatus = - --- Retry requires you feed an exit status (as a string so we can support --- negative codes), and optionally a limit to the number of times this command --- should be retried. -let Retry = { - Type = { - exit_status : ExitStatus, - limit : Optional Natural - }, - default = { - limit = None Natural - } -} - --- Everything here is taken directly from the buildkite Command documentation --- https://buildkite.com/docs/pipelines/command-step#command-step-attributes --- except "target" replaces "agents" --- --- Target is our explicit union of large or small instances. As we build more --- complicated targeting rules we can replace this abstraction with something --- more powerful. + +let TaggedKey = { Type = { name : Text, key : Text }, default = {=} } + +let ExitStatus = < Code : Integer | Any > + +let Retry = + { Type = { exit_status : ExitStatus, limit : Optional Natural } + , default.limit = None Natural + } + let Config = - { Type = - { commands : List Cmd.Type - , depends_on : List TaggedKey.Type - , artifact_paths : List SelectFiles.Type - , env : List TaggedKey.Type - , label : Text - , key : Text - , target : Size - , docker : Optional Docker.Type - , docker_login : Optional DockerLogin.Type - , summon : Optional Summon.Type - , retries : List Retry.Type - , flake_retry_limit: Optional Natural - , soft_fail : Optional B/SoftFail - , skip: Optional B/Skip - , `if` : Optional B/If - , timeout_in_minutes : Optional Integer + { Type = + { commands : List Cmd.Type + , depends_on : List TaggedKey.Type + , artifact_paths : List SelectFiles.Type + , env : List TaggedKey.Type + , label : Text + , key : Text + , target : Size + , docker : Optional Docker.Type + , docker_login : Optional DockerLogin.Type + , summon : Optional Summon.Type + , retries : List Retry.Type + , flake_retry_limit : Optional Natural + , soft_fail : Optional B/SoftFail + , skip : Optional B/Skip + , if : Optional B/If + , timeout_in_minutes : Optional Integer + } + , default = + { depends_on = [] : List TaggedKey.Type + , docker = None Docker.Type + , docker_login = None DockerLogin.Type + , summon = None Summon.Type + , artifact_paths = [] : List SelectFiles.Type + , env = [] : List TaggedKey.Type + , retries = [] : List Retry.Type + , flake_retry_limit = Some 0 + , soft_fail = None B/SoftFail + , skip = None B/Skip + , if = None B/If + , timeout_in_minutes = None Integer + } } - , default = - { depends_on = [] : List TaggedKey.Type - , docker = None Docker.Type - , docker_login = None DockerLogin.Type - , summon = None Summon.Type - , artifact_paths = [] : List SelectFiles.Type - , env = [] : List TaggedKey.Type - , retries = [] : List Retry.Type - , flake_retry_limit = Some 0 - , soft_fail = None B/SoftFail - , skip = None B/Skip - , `if` = None B/If - , timeout_in_minutes = None Integer - } - } - -let targetToAgent = \(target : Size) -> - merge { XLarge = toMap { size = "generic" }, - Large = toMap { size = "generic" }, - Medium = toMap { size = "generic" }, - Small = toMap { size = "generic" }, - Integration = toMap { size = "integration" }, - QA = toMap { size = "qa" }, - Hardfork = toMap {size = "hardfork"} - } - target - -let build : Config.Type -> B/Command.Type = \(c : Config.Type) -> - B/Command::{ - agents = - let agents = targetToAgent c.target in - if Prelude.List.null (Map.Entry Text Text) agents then None (Map.Type Text Text) else Some agents, - commands = - B.definitions/commandStep/properties/commands/Type.ListString (Decorate.decorateAll c.commands), - depends_on = - let flattened = - List/map - TaggedKey.Type - Text - (\(k : TaggedKey.Type) -> "_${k.name}-${k.key}") - c.depends_on - in - if Prelude.List.null Text flattened then - None B/DependsOn.Type - else - Some (B/DependsOn.depends flattened), - artifact_paths = if Prelude.List.null SelectFiles.Type c.artifact_paths - then None B/ArtifactPaths - else Some (B/ArtifactPaths.String (SelectFiles.compile c.artifact_paths)), - key = Some c.key, - label = Some c.label, - timeout_in_minutes = c.timeout_in_minutes, - retry = - Some { - -- we only consider automatic retries - automatic = Some ( - -- and for every retry - let xs : List B/AutoRetryChunk = + +let targetToAgent = + \(target : Size) + -> merge + { XLarge = toMap { size = "generic" } + , Large = toMap { size = "generic" } + , Medium = toMap { size = "generic" } + , Small = toMap { size = "generic" } + , Integration = toMap { size = "integration" } + , QA = toMap { size = "qa" } + , Hardfork = toMap { size = "hardfork" } + } + target + +let build + : Config.Type -> B/Command.Type + = \(c : Config.Type) + -> B/Command::{ + , agents = + let agents = targetToAgent c.target + + in if Prelude.List.null (Map.Entry Text Text) agents + + then None (Map.Type Text Text) + + else Some agents + , commands = + B.definitions/commandStep/properties/commands/Type.ListString + (Decorate.decorateAll c.commands) + , depends_on = + let flattened = List/map - Retry.Type - B/AutoRetryChunk - (\(retry : Retry.Type) -> - { - -- we always require the exit status - exit_status = Some ( - merge - { Code = \(i : Integer) -> B/ExitStatus.Integer i - , Any = B/ExitStatus.String "*" } - retry.exit_status), - -- but limit is optional - limit = - Optional/map - Natural - Integer - Natural/toInteger - retry.limit - }) - -- per https://buildkite.com/docs/agent/v3#exit-codes: - ( - [ - -- infra error - Retry::{ exit_status = ExitStatus.Code -1, limit = Some 4 }, - -- infra error - Retry::{ exit_status = ExitStatus.Code +255, limit = Some 4 }, - -- common/flake error - Retry::{ exit_status = ExitStatus.Code +1, limit = c.flake_retry_limit }, - -- apt-get update race condition error - Retry::{ exit_status = ExitStatus.Code +100, limit = Some 4 }, - -- Git checkout error - Retry::{ exit_status = ExitStatus.Code +128, limit = Some 4 }, - -- SIGTERM - Retry::{ exit_status = ExitStatus.Code +143, limit = Some 4 }, - -- Docker error - Retry::{ exit_status = ExitStatus.Code +125, limit = Some 4 } - ] # - -- and the retries that are passed in (if any) - c.retries) - in - B/Retry.ListAutomaticRetry/Type xs), - manual = Some (B/Manual.Manual/Type { - allowed = Some True, - permit_on_passed = Some True, - reason = None Text - }) - }, - soft_fail = c.soft_fail, - skip = c.skip, - `if` = c.`if`, - plugins = - let dockerPart = - Optional/toList - (Map.Type Text Plugins) - (Optional/map - Docker.Type - (Map.Type Text Plugins) - (\(docker: Docker.Type) -> - toMap { `docker#v3.5.0` = Plugins.Docker docker }) - c.docker) - let dockerLoginPart = - Optional/toList - (Map.Type Text Plugins) - (Optional/map - DockerLogin.Type - (Map.Type Text Plugins) - (\(dockerLogin: DockerLogin.Type) -> - toMap { `docker-login#v2.0.1` = Plugins.DockerLogin dockerLogin }) - c.docker_login) - let summonPart = - Optional/toList - (Map.Type Text Plugins) - (Optional/map - Summon.Type - (Map.Type Text Plugins) - (\(summon: Summon.Type) -> - toMap { `angaza/summon#v0.1.0` = Plugins.Summon summon }) - c.summon) - - -- Add more plugins here as needed, empty list omits that part from the - -- plugins map - let allPlugins = List/concat (Map.Entry Text Plugins) (dockerPart # summonPart # dockerLoginPart) in - if Prelude.List.null (Map.Entry Text Plugins) allPlugins then None B/Plugins else Some (B/Plugins.Plugins/Type allPlugins) - } - -in {Config = Config, build = build, Type = B/Command.Type, TaggedKey = TaggedKey, Retry = Retry, ExitStatus = ExitStatus} + TaggedKey.Type + Text + (\(k : TaggedKey.Type) -> "_${k.name}-${k.key}") + c.depends_on + + in if Prelude.List.null Text flattened + + then None B/DependsOn.Type + + else Some (B/DependsOn.depends flattened) + , artifact_paths = + if Prelude.List.null SelectFiles.Type c.artifact_paths + then None B/ArtifactPaths + + else Some + ( B/ArtifactPaths.String + (SelectFiles.compile c.artifact_paths) + ) + , key = Some c.key + , label = Some c.label + , timeout_in_minutes = c.timeout_in_minutes + , retry = Some + { automatic = Some + ( let xs + : List B/AutoRetryChunk + = List/map + Retry.Type + B/AutoRetryChunk + ( \(retry : Retry.Type) + -> { exit_status = Some + ( merge + { Code = + \(i : Integer) + -> B/ExitStatus.Integer i + , Any = B/ExitStatus.String "*" + } + retry.exit_status + ) + , limit = + Optional/map + Natural + Integer + Natural/toInteger + retry.limit + } + ) + ( [ Retry::{ + , exit_status = ExitStatus.Code -1 + , limit = Some 4 + } + , Retry::{ + , exit_status = ExitStatus.Code +255 + , limit = Some 4 + } + , Retry::{ + , exit_status = ExitStatus.Code +1 + , limit = c.flake_retry_limit + } + , Retry::{ + , exit_status = ExitStatus.Code +100 + , limit = Some 4 + } + , Retry::{ + , exit_status = ExitStatus.Code +128 + , limit = Some 4 + } + , Retry::{ + , exit_status = ExitStatus.Code +143 + , limit = Some 4 + } + , Retry::{ + , exit_status = ExitStatus.Code +125 + , limit = Some 4 + } + ] + # c.retries + ) + + in B/Retry.ListAutomaticRetry/Type xs + ) + , manual = Some + ( B/Manual.Manual/Type + { allowed = Some True + , permit_on_passed = Some True + , reason = None Text + } + ) + } + , soft_fail = c.soft_fail + , skip = c.skip + , if = c.if + , plugins = + let dockerPart = + Optional/toList + (Map.Type Text Plugins) + ( Optional/map + Docker.Type + (Map.Type Text Plugins) + ( \(docker : Docker.Type) + -> toMap + { `docker#v3.5.0` = Plugins.Docker docker } + ) + c.docker + ) + + let dockerLoginPart = + Optional/toList + (Map.Type Text Plugins) + ( Optional/map + DockerLogin.Type + (Map.Type Text Plugins) + ( \(dockerLogin : DockerLogin.Type) + -> toMap + { `docker-login#v2.0.1` = + Plugins.DockerLogin dockerLogin + } + ) + c.docker_login + ) + + let summonPart = + Optional/toList + (Map.Type Text Plugins) + ( Optional/map + Summon.Type + (Map.Type Text Plugins) + ( \(summon : Summon.Type) + -> toMap + { `angaza/summon#v0.1.0` = + Plugins.Summon summon + } + ) + c.summon + ) + + let allPlugins = + List/concat + (Map.Entry Text Plugins) + (dockerPart # summonPart # dockerLoginPart) + + in if Prelude.List.null (Map.Entry Text Plugins) allPlugins + + then None B/Plugins + + else Some (B/Plugins.Plugins/Type allPlugins) + } + +in { Config = Config + , build = build + , Type = B/Command.Type + , TaggedKey = TaggedKey + , Retry = Retry + , ExitStatus = ExitStatus + } diff --git a/buildkite/src/Command/CheckGraphQLSchema.dhall b/buildkite/src/Command/CheckGraphQLSchema.dhall index c9fc20367a9..bed77f7f58f 100644 --- a/buildkite/src/Command/CheckGraphQLSchema.dhall +++ b/buildkite/src/Command/CheckGraphQLSchema.dhall @@ -1,18 +1,20 @@ -let Prelude = ../External/Prelude.dhall - let Command = ./Base.dhall + let Size = ./Size.dhall -let RunInToolchain = ./RunInToolchain.dhall in -{ step = \(dependsOn : List Command.TaggedKey.Type) -> - Command.build - Command.Config::{ - commands = - RunInToolchain.runInToolchainBullseye ([] : List Text) - "./buildkite/scripts/check-graphql-schema.sh", - label = "Check GraphQL Schema", - key = "check-graphql-schema", - target = Size.Large, - depends_on = dependsOn - } -} +let RunInToolchain = ./RunInToolchain.dhall + +in { step = + \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchainBullseye + ([] : List Text) + "./buildkite/scripts/check-graphql-schema.sh" + , label = "Check GraphQL Schema" + , key = "check-graphql-schema" + , target = Size.Large + , depends_on = dependsOn + } + } diff --git a/buildkite/src/Command/ConnectToTestnet.dhall b/buildkite/src/Command/ConnectToTestnet.dhall index 7fc32580a5b..ba4bba1b974 100644 --- a/buildkite/src/Command/ConnectToTestnet.dhall +++ b/buildkite/src/Command/ConnectToTestnet.dhall @@ -1,32 +1,32 @@ -let Prelude = ../External/Prelude.dhall let B = ../External/Buildkite.dhall let Command = ./Base.dhall -let Docker = ./Docker/Type.dhall + let Size = ./Size.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type -let Cmd = ../Lib/Cmds.dhall in +let Cmd = ../Lib/Cmds.dhall -{ step = \(dependsOn : List Command.TaggedKey.Type) - -> \(testnet : Text) - -> \(wait_between_graphql_poll : Text) - -> \(wait_before_final_check: Text ) - -> \(soft_fail: B/SoftFail) - -> Command.build - Command.Config::{ - commands = [ - Cmd.runInDocker - Cmd.Docker::{ - image = (../Constants/ContainerImages.dhall).ubuntu2004 +in { step = + \(dependsOn : List Command.TaggedKey.Type) + -> \(testnet : Text) + -> \(wait_between_graphql_poll : Text) + -> \(wait_before_final_check : Text) + -> \(soft_fail : B/SoftFail) + -> Command.build + Command.Config::{ + , commands = + [ Cmd.runInDocker + Cmd.Docker::{ + , image = (../Constants/ContainerImages.dhall).ubuntu2004 + } + "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" + ] + , label = "Connect to ${testnet}" + , soft_fail = Some soft_fail + , key = "connect-to-${testnet}" + , target = Size.Large + , depends_on = dependsOn } - "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" - ], - label = "Connect to ${testnet}", - soft_fail = Some soft_fail, - key = "connect-to-${testnet}", - target = Size.Large, - depends_on = dependsOn - } -} + } diff --git a/buildkite/src/Command/DeployTestnet.dhall b/buildkite/src/Command/DeployTestnet.dhall index db062f3b98d..b9a45590042 100644 --- a/buildkite/src/Command/DeployTestnet.dhall +++ b/buildkite/src/Command/DeployTestnet.dhall @@ -1,63 +1,56 @@ -let Prelude = ../External/Prelude.dhall - let Command = ./Base.dhall + let Size = ./Size.dhall let Cmd = ../Lib/Cmds.dhall +let DeploySpec = + { Type = + { testnetLabel : Text + , deployEnvFile : Text + , workspace : Text + , artifactPath : Text + , postDeploy : Text + , testnetDir : Text + , deps : List Command.TaggedKey.Type + } + , default = + { testnetLabel = "ci-net" + , deployEnvFile = "export-git-env-vars.sh" + , workspace = "\\\${BUILDKITE_BRANCH//[_\\/]/-}" + , artifactPath = "/tmp/artifacts" + , postDeploy = "echo 'Deployment successfull!'" + , testnetDir = "automation/terraform/testnets/ci-net" + , deps = [] : List Command.TaggedKey.Type + } + } -let DeploySpec = { - Type = { - testnetLabel: Text, - deployEnvFile : Text, - workspace: Text, - artifactPath: Text, - postDeploy: Text, - testnetDir: Text, - deps : List Command.TaggedKey.Type - }, - default = { - testnetLabel = "ci-net", - deployEnvFile = "export-git-env-vars.sh", - workspace = "\\\${BUILDKITE_BRANCH//[_\\/]/-}", - artifactPath = "/tmp/artifacts", - postDeploy = "echo 'Deployment successfull!'", - testnetDir = "automation/terraform/testnets/ci-net", - deps = [] : List Command.TaggedKey.Type - } -} - -in - -{ - step = \(spec : DeploySpec.Type) -> - Command.build - Command.Config::{ - commands = [ - Cmd.run "cd ${spec.testnetDir}", - Cmd.run "terraform init", - -- create separate workspace based on build branch to isolate infrastructure states - Cmd.run "terraform workspace select ${spec.workspace} || terraform workspace new ${spec.workspace}", - -- download deployment dependencies and ensure artifact DIR exists - Cmd.run "mkdir -p ${spec.artifactPath}", - Cmd.run "artifact-cache-helper.sh ${spec.deployEnvFile}", - -- launch testnet based on deploy ENV and ensure auto-cleanup on `apply` failures - Cmd.run "source ${spec.deployEnvFile}", - Cmd.run ( - "terraform apply -auto-approve" ++ - " -var mina_image=gcr.io/o1labs-192920/mina-daemon:\\\$MINA_DOCKER_TAG" ++ - " -var ci_artifact_path=${spec.artifactPath}" ++ - " || (terraform destroy -auto-approve && exit 1)" - ), - -- upload/cache testnet genesis_ledger - Cmd.run "artifact-cache-helper.sh ${spec.artifactPath}/genesis_ledger.json --upload", - -- execute post-deploy operation - Cmd.run "${spec.postDeploy}" - ], - label = "Deploy ${spec.testnetLabel}", - key = "deploy-${spec.testnetLabel}", - target = Size.Large, - depends_on = spec.deps - }, - DeploySpec = DeploySpec -} +in { step = + \(spec : DeploySpec.Type) + -> Command.build + Command.Config::{ + , commands = + [ Cmd.run "cd ${spec.testnetDir}" + , Cmd.run "terraform init" + , Cmd.run + "terraform workspace select ${spec.workspace} || terraform workspace new ${spec.workspace}" + , Cmd.run "mkdir -p ${spec.artifactPath}" + , Cmd.run "artifact-cache-helper.sh ${spec.deployEnvFile}" + , Cmd.run "source ${spec.deployEnvFile}" + , Cmd.run + ( "terraform apply -auto-approve" + ++ " -var mina_image=gcr.io/o1labs-192920/mina-daemon:\\\$MINA_DOCKER_TAG" + ++ " -var ci_artifact_path=${spec.artifactPath}" + ++ " || (terraform destroy -auto-approve && exit 1)" + ) + , Cmd.run + "artifact-cache-helper.sh ${spec.artifactPath}/genesis_ledger.json --upload" + , Cmd.run "${spec.postDeploy}" + ] + , label = "Deploy ${spec.testnetLabel}" + , key = "deploy-${spec.testnetLabel}" + , target = Size.Large + , depends_on = spec.deps + } + , DeploySpec = DeploySpec + } diff --git a/buildkite/src/Command/Docker/Type.dhall b/buildkite/src/Command/Docker/Type.dhall index 084f93f93c4..96a7566a7f6 100644 --- a/buildkite/src/Command/Docker/Type.dhall +++ b/buildkite/src/Command/Docker/Type.dhall @@ -5,24 +5,23 @@ -- -- TODO: Move volume to something in the cloud or artifacts from gcloud storage -{ - Type = { - image: Text, - shell: Optional (List Text), - `propagate-environment`: Bool, - `mount-buildkite-agent`: Bool, - `mount-workdir`: Bool, - privileged: Bool, - environment: List Text, - user: Optional Text - }, - default = { - shell = Some ["/bin/sh", "-e", "-c"], - `propagate-environment` = True, - `mount-buildkite-agent` = False, - `mount-workdir` = False, - privileged = False, - environment = [ "BUILDKITE_AGENT_ACCESS_TOKEN" ], - user = None Text - } +{ Type = + { image : Text + , shell : Optional (List Text) + , propagate-environment : Bool + , mount-buildkite-agent : Bool + , mount-workdir : Bool + , privileged : Bool + , environment : List Text + , user : Optional Text + } +, default = + { shell = Some [ "/bin/sh", "-e", "-c" ] + , propagate-environment = True + , mount-buildkite-agent = False + , mount-workdir = False + , privileged = False + , environment = [ "BUILDKITE_AGENT_ACCESS_TOKEN" ] + , user = None Text + } } diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 465529a5839..1ac61447144 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -1,113 +1,113 @@ -- Execute Docker artifact release script according to build scoped DOCKER_DEPLOY_ENV -let Prelude = ../External/Prelude.dhall let B = ../External/Buildkite.dhall let B/If = B.definitions/commandStep/properties/if/Type let Command = ./Base.dhall + let Size = ./Size.dhall let Profiles = ../Constants/Profiles.dhall + let BuildFlags = ../Constants/BuildFlags.dhall let Cmd = ../Lib/Cmds.dhall let DockerLogin = ../Command/DockerLogin/Type.dhall -let DebianRepo = ../Constants/DebianRepo.dhall - - -let ReleaseSpec = { - Type = { - deps : List Command.TaggedKey.Type, - network: Text, - service: Text, - version: Text, - branch: Text, - repo: Text, - deb_codename: Text, - deb_release: Text, - deb_version: Text, - deb_profile: Profiles.Type, - deb_repo: DebianRepo.Type, - build_flags: BuildFlags.Type, - extra_args: Text, - step_key: Text, - `if`: Optional B/If - }, - default = { - deps = [] : List Command.TaggedKey.Type, - network = "devnet", - version = "\\\${MINA_DOCKER_TAG}", - service = "\\\${MINA_SERVICE}", - branch = "\\\${BUILDKITE_BRANCH}", - repo = "\\\${BUILDKITE_REPO}", - deb_codename = "bullseye", - deb_release = "\\\${MINA_DEB_RELEASE}", - deb_version = "\\\${MINA_DEB_VERSION}", - deb_profile = Profiles.Type.Standard, - build_flags = BuildFlags.Type.None, - deb_repo = DebianRepo.Type.PackagesO1Test, - extra_args = "", - step_key = "daemon-standard-docker-image", - `if` = None B/If - } -} - -let generateStep = \(spec : ReleaseSpec.Type) -> - let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" - - let buildDockerCmd = "./scripts/release-docker.sh" ++ - " --service ${spec.service}" ++ - " --version ${spec.version}" ++ - " --network ${spec.network}" ++ - " --branch ${spec.branch}" ++ - " --deb-codename ${spec.deb_codename}" ++ - " --deb-repo ${DebianRepo.address spec.deb_repo}" ++ - " --deb-release ${spec.deb_release}" ++ - " --deb-version ${spec.deb_version}" ++ - " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ - " --deb-build-flags ${BuildFlags.lowerName spec.build_flags}" ++ - " --repo ${spec.repo}" ++ - " --extra-args \\\"${spec.extra_args}\\\"" - let commands = merge { - PackagesO1Test = - [ - Cmd.run ( - exportMinaDebCmd ++ - " && source ./buildkite/scripts/export-git-env-vars.sh " ++ - " && " ++ buildDockerCmd - ) - ], - - Local = - [ - Cmd.run ( - exportMinaDebCmd ++ - " && apt update && apt install -y aptly" ++ - " && ./buildkite/scripts/debian/start_local_repo.sh" ++ - " && source ./buildkite/scripts/export-git-env-vars.sh " ++ - " && " ++ buildDockerCmd ++ - " && ./scripts/debian/aptly.sh stop" - ) - ] - - } spec.deb_repo - - in +let DebianRepo = ../Constants/DebianRepo.dhall - Command.build - Command.Config::{ - commands = commands, - label = "Docker: ${spec.step_key}", - key = spec.step_key, - target = Size.XLarge, - docker_login = Some DockerLogin::{=}, - depends_on = spec.deps, - `if` = spec.`if` +let ReleaseSpec = + { Type = + { deps : List Command.TaggedKey.Type + , network : Text + , service : Text + , version : Text + , branch : Text + , repo : Text + , deb_codename : Text + , deb_release : Text + , deb_version : Text + , deb_profile : Profiles.Type + , deb_repo : DebianRepo.Type + , build_flags : BuildFlags.Type + , extra_args : Text + , step_key : Text + , if : Optional B/If + } + , default = + { deps = [] : List Command.TaggedKey.Type + , network = "devnet" + , version = "\\\${MINA_DOCKER_TAG}" + , service = "\\\${MINA_SERVICE}" + , branch = "\\\${BUILDKITE_BRANCH}" + , repo = "\\\${BUILDKITE_REPO}" + , deb_codename = "bullseye" + , deb_release = "\\\${MINA_DEB_RELEASE}" + , deb_version = "\\\${MINA_DEB_VERSION}" + , deb_profile = Profiles.Type.Standard + , build_flags = BuildFlags.Type.None + , deb_repo = DebianRepo.Type.PackagesO1Test + , extra_args = "" + , step_key = "daemon-standard-docker-image" + , if = None B/If + } } -in - -{ generateStep = generateStep, ReleaseSpec = ReleaseSpec } +let generateStep = + \(spec : ReleaseSpec.Type) + -> let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" + + let buildDockerCmd = + "./scripts/release-docker.sh" + ++ " --service ${spec.service}" + ++ " --version ${spec.version}" + ++ " --network ${spec.network}" + ++ " --branch ${spec.branch}" + ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-repo ${DebianRepo.address spec.deb_repo}" + ++ " --deb-release ${spec.deb_release}" + ++ " --deb-version ${spec.deb_version}" + ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" + ++ " --deb-build-flags ${BuildFlags.lowerName + spec.build_flags}" + ++ " --repo ${spec.repo}" + ++ " --extra-args \\\"${spec.extra_args}\\\"" + + let commands = + merge + { PackagesO1Test = + [ Cmd.run + ( exportMinaDebCmd + ++ " && source ./buildkite/scripts/export-git-env-vars.sh " + ++ " && " + ++ buildDockerCmd + ) + ] + , Local = + [ Cmd.run + ( exportMinaDebCmd + ++ " && apt update && apt install -y aptly" + ++ " && ./buildkite/scripts/debian/start_local_repo.sh" + ++ " && source ./buildkite/scripts/export-git-env-vars.sh " + ++ " && " + ++ buildDockerCmd + ++ " && ./scripts/debian/aptly.sh stop" + ) + ] + } + spec.deb_repo + + in Command.build + Command.Config::{ + , commands = commands + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , docker_login = Some DockerLogin::{=} + , depends_on = spec.deps + , if = spec.if + } + +in { generateStep = generateStep, ReleaseSpec = ReleaseSpec } diff --git a/buildkite/src/Command/DockerLogin/Type.dhall b/buildkite/src/Command/DockerLogin/Type.dhall index 7af2e15b4d1..0fdf2c4fc07 100644 --- a/buildkite/src/Command/DockerLogin/Type.dhall +++ b/buildkite/src/Command/DockerLogin/Type.dhall @@ -3,15 +3,10 @@ -- See https://github.com/buildkite-plugins/docker-login-buildkite-plugin for options -- if you'd like to extend this definition for example -{ - Type = { - username: Text, - `password-env`: Text, - server: Text - }, - default = { - username = "o1botdockerhub", - `password-env` = "DOCKER_PASSWORD", - server = "" +{ Type = { username : Text, password-env : Text, server : Text } +, default = + { username = "o1botdockerhub" + , password-env = "DOCKER_PASSWORD" + , server = "" } } diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 8b694c9e5ca..f45da4fb027 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -1,199 +1,241 @@ let Prelude = ../External/Prelude.dhall -let List/map = Prelude.List.map + let Optional/toList = Prelude.Optional.toList + let Optional/map = Prelude.Optional.map let Command = ./Base.dhall let Cmd = ../Lib/Cmds.dhall + let S = ../Lib/SelectFiles.dhall -let D = S.PathPattern let B = ../External/Buildkite.dhall + let B/If = B.definitions/commandStep/properties/if/Type let Pipeline = ../Pipeline/Dsl.dhall + let PipelineTag = ../Pipeline/Tag.dhall + let PipelineMode = ../Pipeline/Mode.dhall let JobSpec = ../Pipeline/JobSpec.dhall -let Summon = ./Summon/Type.dhall let Size = ./Size.dhall -let Libp2p = ./Libp2pHelperBuild.dhall + let DockerImage = ./DockerImage.dhall + let DebianVersions = ../Constants/DebianVersions.dhall + let DebianRepo = ../Constants/DebianRepo.dhall + let Profiles = ../Constants/Profiles.dhall -let Artifacts = ../Constants/Artifacts.dhall + let Toolchain = ../Constants/Toolchain.dhall + let Network = ../Constants/Network.dhall -let Spec = { - Type = { - codename: DebianVersions.DebVersion, - network: Network.Type, - genesis_timestamp: Optional Text, - config_json_gz_url: Text, - profile: Profiles.Type, - suffix: Text - }, - default = { - codename = DebianVersions.DebVersion.Bullseye, - network = Network.Type.Devnet, - genesis_timestamp = Some "2024-04-07T11:45:00Z", - config_json_gz_url = "https://storage.googleapis.com/o1labs-gitops-infrastructure/devnet/devnet-state-dump-3NK4eDgbkCjKj9fFUXVkrJXsfpfXzJySoAvrFJVCropPW7LLF14F-676026c4d4d2c18a76b357d6422a06f932c3ef4667a8fd88717f68b53fd6b2d7.json.gz", - profile = Profiles.Type.Standard, - suffix = "hardfork" - } -} - -let spec_to_envs : Spec.Type -> List Text = - \(spec: Spec.Type ) -> - - [ - "NETWORK_NAME=${Network.lowerName spec.network}", - "TESTNET_NAME=${Network.lowerName spec.network}-${spec.suffix}", - "CONFIG_JSON_GZ_URL=${spec.config_json_gz_url}" - ] - - # - - (Optional/toList - Text - (Optional/map - Text - Text - (\(genesis_timestamp: Text) -> "GENESIS_TIMESTAMP=${genesis_timestamp}" ) - spec.genesis_timestamp) - ) - -let pipeline : Spec.Type -> Pipeline.Config.Type = - \(spec: Spec.Type) -> - let profile = spec.profile - let network = (Network.lowerName spec.network) - let network_name = "${network}-hardfork" - let pipelineName = "MinaArtifactHardfork${DebianVersions.capitalName spec.codename}${Profiles.toSuffixUppercase profile}" - let generateLedgersJobKey = "generate-ledger-tars-from-config" - let debVersion = spec.codename - let image = "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network_name}" - - in - - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ S.everything ] - , path = "Release" - , name = pipelineName - , tags = [ PipelineTag.Type.Release, PipelineTag.Type.Hardfork, PipelineTag.Type.Long ] - , mode = PipelineMode.Type.Stable - } - , steps = - [ Command.build - Command.Config::{ - commands = - Toolchain.runner debVersion - ( - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_BRANCH=\$BUILDKITE_BRANCH" - , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}" - ] - # (spec_to_envs spec) - ) - "./buildkite/scripts/build-hardfork-package.sh" - # - [ - Cmd.run "./buildkite/scripts/upload-deb-to-gs.sh ${DebianVersions.lowerName debVersion}" - ] - , label = "Build Mina Hardfork Package for ${DebianVersions.capitalName debVersion}" - , key = generateLedgersJobKey - , target = Size.XLarge - } - , Command.build - Command.Config::{ - commands = Toolchain.runner debVersion [ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}" - ] "./buildkite/scripts/publish-deb.sh" - , label = "Publish Mina for ${DebianVersions.capitalName debVersion} Hardfork" - , depends_on = [{ name = pipelineName, key = generateLedgersJobKey}] - , key = "publish-hardfork-deb-pkg" - , target = Size.Small +let Spec = + { Type = + { codename : DebianVersions.DebVersion + , network : Network.Type + , genesis_timestamp : Optional Text + , config_json_gz_url : Text + , profile : Profiles.Type + , suffix : Text } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - deps = - [ { name = pipelineName - , key = generateLedgersJobKey - } - ] - , service = "mina-daemon" - , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , deb_repo = DebianRepo.Type.Local - , step_key = "daemon-berkeley-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image" - } - , Command.build Command.Config::{ - commands = [ - Cmd.runInDocker Cmd.Docker::{ - image = image - -- an account with this balance seems present in many ledgers? - } "curl ${spec.config_json_gz_url} > config.json.gz && gunzip config.json.gz && sed -e '0,/20.000001/{s/20.000001/20.01/}' -i config.json && ! (mina-verify-packaged-fork-config ${network} config.json /workdir/verification)" - ] - , label = "Assert corrupted packaged artifacts are unverifiable" - , key = "assert-unverify-corrupted-packaged-artifacts" - , target = Size.XLarge - , depends_on = [{ name = pipelineName, key = "daemon-berkeley-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image" }] - , `if` = None B/If - } - , Command.build Command.Config::{ - commands = [ - Cmd.runInDocker Cmd.Docker::{ - image = image - } "curl ${spec.config_json_gz_url} > config.json.gz && gunzip config.json.gz && mina-verify-packaged-fork-config ${network} config.json /workdir/verification" + , default = + { codename = DebianVersions.DebVersion.Bullseye + , network = Network.Type.Devnet + , genesis_timestamp = Some "2024-04-07T11:45:00Z" + , config_json_gz_url = + "https://storage.googleapis.com/o1labs-gitops-infrastructure/devnet/devnet-state-dump-3NK4eDgbkCjKj9fFUXVkrJXsfpfXzJySoAvrFJVCropPW7LLF14F-676026c4d4d2c18a76b357d6422a06f932c3ef4667a8fd88717f68b53fd6b2d7.json.gz" + , profile = Profiles.Type.Standard + , suffix = "hardfork" + } + } + +let spec_to_envs + : Spec.Type -> List Text + = \(spec : Spec.Type) + -> [ "NETWORK_NAME=${Network.lowerName spec.network}" + , "TESTNET_NAME=${Network.lowerName spec.network}-${spec.suffix}" + , "CONFIG_JSON_GZ_URL=${spec.config_json_gz_url}" ] - , label = "Verify packaged artifacts" - , key = "verify-packaged-artifacts" - , target = Size.XLarge - , depends_on = [{ name = pipelineName, key = "daemon-berkeley-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image" }] - , `if` = None B/If - } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - deps = - [ { name = pipelineName - , key = generateLedgersJobKey - } - ] - , service = "mina-archive" - , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , deb_repo = DebianRepo.Type.Local - , step_key = "archive-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image" - } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - deps = - [ { name = pipelineName - , key = generateLedgersJobKey + # Optional/toList + Text + ( Optional/map + Text + Text + ( \(genesis_timestamp : Text) + -> "GENESIS_TIMESTAMP=${genesis_timestamp}" + ) + spec.genesis_timestamp + ) + +let pipeline + : Spec.Type -> Pipeline.Config.Type + = \(spec : Spec.Type) + -> let profile = spec.profile + + let network = Network.lowerName spec.network + + let network_name = "${network}-hardfork" + + let pipelineName = + "MinaArtifactHardfork${DebianVersions.capitalName + spec.codename}${Profiles.toSuffixUppercase + profile}" + + let generateLedgersJobKey = "generate-ledger-tars-from-config" + + let debVersion = spec.codename + + let image = + "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName + debVersion}-${network_name}" + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = [ S.everything ] + , path = "Release" + , name = pipelineName + , tags = + [ PipelineTag.Type.Release + , PipelineTag.Type.Hardfork + , PipelineTag.Type.Long + ] + , mode = PipelineMode.Type.Stable } - ] - , service = "mina-rosetta" - , network = network_name - , deb_repo = DebianRepo.Type.Local - , deb_codename = "${DebianVersions.lowerName debVersion}" - , step_key = "rosetta-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image" - } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + Toolchain.runner + debVersion + ( [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_BRANCH=\$BUILDKITE_BRANCH" + , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + debVersion}" + ] + # spec_to_envs spec + ) + "./buildkite/scripts/build-hardfork-package.sh" + # [ Cmd.run + "./buildkite/scripts/upload-deb-to-gs.sh ${DebianVersions.lowerName + debVersion}" + ] + , label = + "Build Mina Hardfork Package for ${DebianVersions.capitalName + debVersion}" + , key = generateLedgersJobKey + , target = Size.XLarge + } + , Command.build + Command.Config::{ + , commands = + Toolchain.runner + debVersion + [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + debVersion}" + ] + "./buildkite/scripts/publish-deb.sh" + , label = + "Publish Mina for ${DebianVersions.capitalName + debVersion} Hardfork" + , depends_on = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , key = "publish-hardfork-deb-pkg" + , target = Size.Small + } + , DockerImage.generateStep + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = "mina-daemon" + , network = network_name + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + , step_key = + "daemon-berkeley-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}-docker-image" + } + , Command.build + Command.Config::{ + , commands = + [ Cmd.runInDocker + Cmd.Docker::{ image = image } + "curl ${spec.config_json_gz_url} > config.json.gz && gunzip config.json.gz && sed -e '0,/20.000001/{s/20.000001/20.01/}' -i config.json && ! (mina-verify-packaged-fork-config ${network} config.json /workdir/verification)" + ] + , label = + "Assert corrupted packaged artifacts are unverifiable" + , key = "assert-unverify-corrupted-packaged-artifacts" + , target = Size.XLarge + , depends_on = + [ { name = pipelineName + , key = + "daemon-berkeley-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}-docker-image" + } + ] + , if = None B/If + } + , Command.build + Command.Config::{ + , commands = + [ Cmd.runInDocker + Cmd.Docker::{ image = image } + "curl ${spec.config_json_gz_url} > config.json.gz && gunzip config.json.gz && mina-verify-packaged-fork-config ${network} config.json /workdir/verification" + ] + , label = "Verify packaged artifacts" + , key = "verify-packaged-artifacts" + , target = Size.XLarge + , depends_on = + [ { name = pipelineName + , key = + "daemon-berkeley-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}-docker-image" + } + ] + , if = None B/If + } + , DockerImage.generateStep + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = "mina-archive" + , network = network_name + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + , step_key = + "archive-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}-docker-image" + } + , DockerImage.generateStep + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = "mina-rosetta" + , network = network_name + , deb_repo = DebianRepo.Type.Local + , deb_codename = "${DebianVersions.lowerName debVersion}" + , step_key = + "rosetta-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}-docker-image" + } + ] + } -in -{ - pipeline = pipeline, - Spec = Spec -} \ No newline at end of file +in { pipeline = pipeline, Spec = Spec } diff --git a/buildkite/src/Command/Libp2pHelperBuild.dhall b/buildkite/src/Command/Libp2pHelperBuild.dhall index 4354d429f2e..685aa50979c 100644 --- a/buildkite/src/Command/Libp2pHelperBuild.dhall +++ b/buildkite/src/Command/Libp2pHelperBuild.dhall @@ -1,38 +1,41 @@ -let Prelude = ../External/Prelude.dhall - let Command = ./Base.dhall -let Docker = ./Docker/Type.dhall + let Size = ./Size.dhall + let Toolchain = ../Constants/Toolchain.dhall + let BuildFlags = ../Constants/BuildFlags.dhall + let Cmd = ../Lib/Cmds.dhall let DebianVersions = ../Constants/DebianVersions.dhall -let commands = \(debVersion : DebianVersions.DebVersion) -> - [ - Cmd.run "chmod -R 777 src/app/libp2p_helper", - Cmd.run "chmod -R 777 src/libp2p_ipc", - Cmd.runInDocker - Cmd.Docker::{ - image = Toolchain.image debVersion, - extraEnv = [ "GO=/usr/lib/go/bin/go" ] - } - "make libp2p_helper", - Cmd.run "cp src/app/libp2p_helper/result/bin/libp2p_helper . && buildkite/scripts/buildkite-artifact-helper.sh libp2p_helper" - ] - -in - -let cmdConfig = \(debVersion : DebianVersions.DebVersion) -> \(buildFlags: BuildFlags.Type) -> - Command.build - Command.Config::{ - commands = commands debVersion, - label = "Build Libp2p helper for ${DebianVersions.capitalName debVersion} ${BuildFlags.toSuffixUppercase buildFlags}", - key = "libp2p-helper${BuildFlags.toLabelSegment buildFlags}", - target = Size.Small - } - -in - -{ step = cmdConfig } +let commands = + \(debVersion : DebianVersions.DebVersion) + -> [ Cmd.run "chmod -R 777 src/app/libp2p_helper" + , Cmd.run "chmod -R 777 src/libp2p_ipc" + , Cmd.runInDocker + Cmd.Docker::{ + , image = Toolchain.image debVersion + , extraEnv = [ "GO=/usr/lib/go/bin/go" ] + } + "make libp2p_helper" + , Cmd.run + "cp src/app/libp2p_helper/result/bin/libp2p_helper . && buildkite/scripts/buildkite-artifact-helper.sh libp2p_helper" + ] + +let cmdConfig = + \(debVersion : DebianVersions.DebVersion) + -> \(buildFlags : BuildFlags.Type) + -> Command.build + Command.Config::{ + , commands = commands debVersion + , label = + "Build Libp2p helper for ${DebianVersions.capitalName + debVersion} ${BuildFlags.toSuffixUppercase + buildFlags}" + , key = "libp2p-helper${BuildFlags.toLabelSegment buildFlags}" + , target = Size.Small + } + +in { step = cmdConfig } diff --git a/buildkite/src/Command/Mina.dhall b/buildkite/src/Command/Mina.dhall index 4339d638b3f..e732222496d 100644 --- a/buildkite/src/Command/Mina.dhall +++ b/buildkite/src/Command/Mina.dhall @@ -1,12 +1,7 @@ -let Prelude = ../External/Prelude.dhall -let List/map = Prelude.List.map -let B = ../External/Buildkite.dhall -let B/Plugins/Partial = B.definitions/commandStep/properties/plugins/Type -let Map = Prelude.Map - let Cmd = ../Lib/Cmds.dhall + let Docker = ./Docker/Type.dhall -let Summon = ./Summon/Type.dhall + let Base = ./Base.dhall let Size = ./Size.dhall @@ -15,30 +10,26 @@ let dockerImage = (../Constants/ContainerImages.dhall).minaToolchain let fixPermissionsScript = "sudo chown -R opam ." -let Config = { - Type = { - commands : List Cmd.Type, - label : Text, - key : Text - }, - default = {=} -} - - -let build : Config.Type -> Base.Type = \(c : Config.Type) -> - Base.build - Base.Config::{ - commands = [ Cmd.run fixPermissionsScript ] # c.commands, - label = c.label, - key = c.key, - target = Size.Small, - docker = Some Docker::{ image = dockerImage } +let Config = + { Type = { commands : List Cmd.Type, label : Text, key : Text } + , default = {=} + } + +let build + : Config.Type -> Base.Type + = \(c : Config.Type) + -> Base.build + Base.Config::{ + , commands = [ Cmd.run fixPermissionsScript ] # c.commands + , label = c.label + , key = c.key + , target = Size.Small + , docker = Some Docker::{ image = dockerImage } + } + +in { fixPermissionsCommand = + Cmd.runInDocker Cmd.Docker::{ image = dockerImage } fixPermissionsScript + , Config = Config + , build = build + , Type = Base.Type } - -in { - fixPermissionsCommand = Cmd.runInDocker Cmd.Docker::{ image = dockerImage } fixPermissionsScript, - Config = Config, - build = build, - Type = Base.Type -} - diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index c5f966a065a..626bec23419 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -1,209 +1,305 @@ let Prelude = ../External/Prelude.dhall + let List/map = Prelude.List.map + let Command = ./Base.dhall let Cmd = ../Lib/Cmds.dhall -let S = ../Lib/SelectFiles.dhall -let D = S.PathPattern - -let B = ../External/Buildkite.dhall -let B/If = B.definitions/commandStep/properties/if/Type let Pipeline = ../Pipeline/Dsl.dhall + let PipelineTag = ../Pipeline/Tag.dhall + let PipelineMode = ../Pipeline/Mode.dhall let JobSpec = ../Pipeline/JobSpec.dhall -let Summon = ./Summon/Type.dhall let Size = ./Size.dhall + let Libp2p = ./Libp2pHelperBuild.dhall + let DockerImage = ./DockerImage.dhall + let DebianVersions = ../Constants/DebianVersions.dhall + let DebianRepo = ../Constants/DebianRepo.dhall + let Profiles = ../Constants/Profiles.dhall + let BuildFlags = ../Constants/BuildFlags.dhall + let Artifacts = ../Constants/Artifacts.dhall + let Toolchain = ../Constants/Toolchain.dhall -in - -let docker_step : Artifacts.Type -> DebianVersions.DebVersion -> Profiles.Type -> BuildFlags.Type -> DockerImage.ReleaseSpec.Type = - \(artifact : Artifacts.Type) -> - \(debVersion : DebianVersions.DebVersion) -> - \(profile : Profiles.Type) -> - \(buildFlags: BuildFlags.Type) -> - - let step_dep_name = "build" - in - merge { - Daemon = - DockerImage.ReleaseSpec::{ - deps=DebianVersions.dependsOnStep debVersion profile buildFlags step_dep_name, - service="mina-daemon", - network="berkeley", - deb_codename="${DebianVersions.lowerName debVersion}", - deb_profile=profile, - build_flags=buildFlags, - deb_repo = DebianRepo.Type.Local, - step_key="daemon-berkeley-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}${BuildFlags.toLabelSegment buildFlags}-docker-image" - }, - - TestExecutive = - DockerImage.ReleaseSpec::{ - deps=DebianVersions.dependsOnStep debVersion profile buildFlags step_dep_name, service="mina-test-executive", - deb_codename="${DebianVersions.lowerName debVersion}", - deb_repo = DebianRepo.Type.Local, - step_key="test-executive-${DebianVersions.lowerName debVersion}-docker-image" - }, - - BatchTxn = - DockerImage.ReleaseSpec::{ - deps=DebianVersions.dependsOnStep debVersion profile buildFlags step_dep_name, service="mina-batch-txn", - network="berkeley", - deb_codename="${DebianVersions.lowerName debVersion}", - deb_repo = DebianRepo.Type.Local, - step_key="batch-txn-${DebianVersions.lowerName debVersion}-docker-image" - }, - - Archive = - DockerImage.ReleaseSpec::{ - deps=DebianVersions.dependsOnStep debVersion profile buildFlags step_dep_name, service="mina-archive", - deb_codename="${DebianVersions.lowerName debVersion}", - deb_profile=profile, - build_flags=buildFlags, - deb_repo = DebianRepo.Type.Local, - step_key="archive-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}${BuildFlags.toLabelSegment buildFlags}-docker-image" - }, - - ArchiveMigration = - DockerImage.ReleaseSpec::{ - deps=DebianVersions.dependsOnStep debVersion profile buildFlags step_dep_name, service="mina-archive-migration", - deb_codename="${DebianVersions.lowerName debVersion}", - deb_profile=profile, - deb_repo = DebianRepo.Type.Local, - step_key="archive-migration-${DebianVersions.lowerName debVersion}-docker-image" - }, - - Rosetta = - DockerImage.ReleaseSpec::{ - deps=DebianVersions.dependsOnStep debVersion profile buildFlags step_dep_name, service="mina-rosetta", - network="berkeley", - deb_codename="${DebianVersions.lowerName debVersion}", - deb_repo = DebianRepo.Type.Local, - step_key="rosetta-${DebianVersions.lowerName debVersion}${BuildFlags.toLabelSegment buildFlags}-docker-image" - }, - - ZkappTestTransaction = - DockerImage.ReleaseSpec::{ - deps=DebianVersions.dependsOnStep debVersion profile buildFlags step_dep_name, service="mina-zkapp-test-transaction", - deb_codename="${DebianVersions.lowerName debVersion}", - deb_repo = DebianRepo.Type.Local, - step_key="zkapp-test-transaction-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}${BuildFlags.toLabelSegment buildFlags}-docker-image" - }, - - FunctionalTestSuite = - DockerImage.ReleaseSpec::{ - deps=DebianVersions.dependsOnStep debVersion profile buildFlags step_dep_name, - service="mina-test-suite", - deb_codename="${DebianVersions.lowerName debVersion}", - deb_repo = DebianRepo.Type.Local, - step_key="test-suite-${DebianVersions.lowerName debVersion}${Profiles.toLabelSegment profile}-docker-image", - network="berkeley" +let docker_step + : Artifacts.Type + -> DebianVersions.DebVersion + -> Profiles.Type + -> BuildFlags.Type + -> DockerImage.ReleaseSpec.Type + = \(artifact : Artifacts.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(profile : Profiles.Type) + -> \(buildFlags : BuildFlags.Type) + -> let step_dep_name = "build" + + in merge + { Daemon = DockerImage.ReleaseSpec::{ + , deps = + DebianVersions.dependsOnStep + debVersion + profile + buildFlags + step_dep_name + , service = "mina-daemon" + , network = "berkeley" + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_profile = profile + , build_flags = buildFlags + , deb_repo = DebianRepo.Type.Local + , step_key = + "daemon-berkeley-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}${BuildFlags.toLabelSegment + buildFlags}-docker-image" + } + , TestExecutive = DockerImage.ReleaseSpec::{ + , deps = + DebianVersions.dependsOnStep + debVersion + profile + buildFlags + step_dep_name + , service = "mina-test-executive" + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_repo = DebianRepo.Type.Local + , step_key = + "test-executive-${DebianVersions.lowerName + debVersion}-docker-image" + } + , BatchTxn = DockerImage.ReleaseSpec::{ + , deps = + DebianVersions.dependsOnStep + debVersion + profile + buildFlags + step_dep_name + , service = "mina-batch-txn" + , network = "berkeley" + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_repo = DebianRepo.Type.Local + , step_key = + "batch-txn-${DebianVersions.lowerName + debVersion}-docker-image" + } + , Archive = DockerImage.ReleaseSpec::{ + , deps = + DebianVersions.dependsOnStep + debVersion + profile + buildFlags + step_dep_name + , service = "mina-archive" + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_profile = profile + , build_flags = buildFlags + , deb_repo = DebianRepo.Type.Local + , step_key = + "archive-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}${BuildFlags.toLabelSegment + buildFlags}-docker-image" + } + , ArchiveMigration = DockerImage.ReleaseSpec::{ + , deps = + DebianVersions.dependsOnStep + debVersion + profile + buildFlags + step_dep_name + , service = "mina-archive-migration" + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + , step_key = + "archive-migration-${DebianVersions.lowerName + debVersion}-docker-image" + } + , Rosetta = DockerImage.ReleaseSpec::{ + , deps = + DebianVersions.dependsOnStep + debVersion + profile + buildFlags + step_dep_name + , service = "mina-rosetta" + , network = "berkeley" + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_repo = DebianRepo.Type.Local + , step_key = + "rosetta-${DebianVersions.lowerName + debVersion}${BuildFlags.toLabelSegment + buildFlags}-docker-image" + } + , ZkappTestTransaction = DockerImage.ReleaseSpec::{ + , deps = + DebianVersions.dependsOnStep + debVersion + profile + buildFlags + step_dep_name + , service = "mina-zkapp-test-transaction" + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_repo = DebianRepo.Type.Local + , step_key = + "zkapp-test-transaction-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}${BuildFlags.toLabelSegment + buildFlags}-docker-image" + } + , FunctionalTestSuite = DockerImage.ReleaseSpec::{ + , deps = + DebianVersions.dependsOnStep + debVersion + profile + buildFlags + step_dep_name + , service = "mina-test-suite" + , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_repo = DebianRepo.Type.Local + , step_key = + "test-suite-${DebianVersions.lowerName + debVersion}${Profiles.toLabelSegment + profile}-docker-image" + , network = "berkeley" + } + } + artifact + +let MinaBuildSpec = + { Type = + { prefix : Text + , artifacts : List Artifacts.Type + , debVersion : DebianVersions.DebVersion + , profile : Profiles.Type + , buildFlags : BuildFlags.Type + , toolchainSelectMode : Toolchain.SelectionMode + , mode : PipelineMode.Type } - } artifact -in - - -let MinaBuildSpec = { - Type = { - prefix: Text, - artifacts: List Artifacts.Type, - debVersion : DebianVersions.DebVersion, - profile: Profiles.Type, - buildFlags: BuildFlags.Type, - toolchainSelectMode: Toolchain.SelectionMode, - mode: PipelineMode.Type - }, - default = { - prefix = "MinaArtifact", - artifacts = Artifacts.AllButTests, - debVersion = DebianVersions.DebVersion.Bullseye, - profile = Profiles.Type.Standard, - buildFlags = BuildFlags.Type.None, - toolchainSelectMode = Toolchain.SelectionMode.ByDebian, - mode = PipelineMode.Type.PullRequest - } -} - -let build_artifacts = - \(spec: MinaBuildSpec.Type) -> - Command.build - Command.Config::{ - commands = ( - (Toolchain.select spec.toolchainSelectMode spec.debVersion ([ - "DUNE_PROFILE=${Profiles.duneProfile spec.profile}", - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "MINA_BRANCH=$BUILDKITE_BRANCH", - "MINA_COMMIT_SHA1=$BUILDKITE_COMMIT", - "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" - ] # BuildFlags.buildEnvs spec.buildFlags ) - "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames spec.artifacts}") - # - [Cmd.run "./buildkite/scripts/upload-deb-to-gs.sh ${DebianVersions.lowerName spec.debVersion}"]), - label = "Build Mina for ${DebianVersions.capitalName spec.debVersion} ${Profiles.toSuffixUppercase spec.profile} ${BuildFlags.toSuffixUppercase spec.buildFlags}", - key = "build-deb-pkg", - target = Size.XLarge, - retries = [ - Command.Retry::{ - exit_status = Command.ExitStatus.Code +2, - limit = Some 2 - } ] -- libp2p error + , default = + { prefix = "MinaArtifact" + , artifacts = Artifacts.AllButTests + , debVersion = DebianVersions.DebVersion.Bullseye + , profile = Profiles.Type.Standard + , buildFlags = BuildFlags.Type.None + , toolchainSelectMode = Toolchain.SelectionMode.ByDebian + , mode = PipelineMode.Type.PullRequest } + } -let publish_to_debian_repo = - \(spec: MinaBuildSpec.Type) -> - Command.build - Command.Config::{ - commands = Toolchain.select spec.toolchainSelectMode spec.debVersion [ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" - ] "./buildkite/scripts/publish-deb.sh", - label = "Publish Mina for ${DebianVersions.capitalName spec.debVersion} ${Profiles.toSuffixUppercase spec.profile}", - key = "publish-deb-pkg", - depends_on = DebianVersions.dependsOnStep spec.debVersion spec.profile spec.buildFlags "build", - target = Size.Small - } +let build_artifacts = + \(spec : MinaBuildSpec.Type) + -> Command.build + Command.Config::{ + , commands = + Toolchain.select + spec.toolchainSelectMode + spec.debVersion + ( [ "DUNE_PROFILE=${Profiles.duneProfile spec.profile}" + , "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_BRANCH=\$BUILDKITE_BRANCH" + , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + ] + # BuildFlags.buildEnvs spec.buildFlags + ) + "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames + spec.artifacts}" + # [ Cmd.run + "./buildkite/scripts/upload-deb-to-gs.sh ${DebianVersions.lowerName + spec.debVersion}" + ] + , label = + "Build Mina for ${DebianVersions.capitalName + spec.debVersion} ${Profiles.toSuffixUppercase + spec.profile} ${BuildFlags.toSuffixUppercase + spec.buildFlags}" + , key = "build-deb-pkg" + , target = Size.XLarge + , retries = + [ Command.Retry::{ + , exit_status = Command.ExitStatus.Code +2 + , limit = Some 2 + } + ] + } + +let publish_to_debian_repo = + \(spec : MinaBuildSpec.Type) + -> Command.build + Command.Config::{ + , commands = + Toolchain.select + spec.toolchainSelectMode + spec.debVersion + [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + ] + "./buildkite/scripts/publish-deb.sh" + , label = + "Publish Mina for ${DebianVersions.capitalName + spec.debVersion} ${Profiles.toSuffixUppercase + spec.profile}" + , key = "publish-deb-pkg" + , depends_on = + DebianVersions.dependsOnStep + spec.debVersion + spec.profile + spec.buildFlags + "build" + , target = Size.Small + } + +let pipeline + : MinaBuildSpec.Type -> Pipeline.Config.Type + = \(spec : MinaBuildSpec.Type) + -> let steps = + [ Libp2p.step spec.debVersion spec.buildFlags + , build_artifacts spec + , publish_to_debian_repo spec + ] + # List/map + Artifacts.Type + Command.Type + ( \(artifact : Artifacts.Type) + -> DockerImage.generateStep + ( docker_step + artifact + spec.debVersion + spec.profile + spec.buildFlags + ) + ) + spec.artifacts + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion + , path = "Release" + , name = + "${spec.prefix}${DebianVersions.capitalName + spec.debVersion}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] + , mode = spec.mode + } + , steps = steps + } -let pipeline : MinaBuildSpec.Type -> Pipeline.Config.Type = - \(spec: MinaBuildSpec.Type) -> - let steps = [ - Libp2p.step spec.debVersion spec.buildFlags, - (build_artifacts spec), - (publish_to_debian_repo spec) - ] # (List/map - Artifacts.Type - Command.Type - (\(artifact : Artifacts.Type) -> DockerImage.generateStep (docker_step artifact spec.debVersion spec.profile spec.buildFlags) ) - spec.artifacts) - in - - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = DebianVersions.dirtyWhen spec.debVersion, - path = "Release", - name = "${spec.prefix}${DebianVersions.capitalName spec.debVersion}${Profiles.toSuffixUppercase spec.profile}${BuildFlags.toSuffixUppercase spec.buildFlags}", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ], - mode = spec.mode - }, - steps = steps - } - -in -{ - pipeline = pipeline - , MinaBuildSpec = MinaBuildSpec -} +in { pipeline = pipeline, MinaBuildSpec = MinaBuildSpec } diff --git a/buildkite/src/Command/PromotePackage.dhall b/buildkite/src/Command/PromotePackage.dhall index 78a5bbabcd9..b4f0db530df 100644 --- a/buildkite/src/Command/PromotePackage.dhall +++ b/buildkite/src/Command/PromotePackage.dhall @@ -1,242 +1,305 @@ let B = ../External/Buildkite.dhall + let B/If = B.definitions/commandStep/properties/if/Type let Prelude = ../External/Prelude.dhall -let Optional/toList = Prelude.Optional.toList -let Optional/map = Prelude.Optional.map + let List/map = Prelude.List.map + let Package = ../Constants/DebianPackage.dhall + let Network = ../Constants/Network.dhall + let PipelineMode = ../Pipeline/Mode.dhall + let PipelineTag = ../Pipeline/Tag.dhall + let Pipeline = ../Pipeline/Dsl.dhall + let JobSpec = ../Pipeline/JobSpec.dhall + let DebianChannel = ../Constants/DebianChannel.dhall + let Profiles = ../Constants/Profiles.dhall + let Artifact = ../Constants/Artifacts.dhall + let DebianVersions = ../Constants/DebianVersions.dhall + let Toolchain = ../Constants/Toolchain.dhall + let Command = ./Base.dhall -let Docker = ./Docker/Type.dhall -let Size = ./Size.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let Size = ./Size.dhall let Cmd = ../Lib/Cmds.dhall -let PromoteDebianSpec = { - Type = { - deps: List Command.TaggedKey.Type, - package: Package.Type, - version: Text, - new_version: Text, - architecture: Text, - network: Network.Type, - codename: DebianVersions.DebVersion, - from_channel: DebianChannel.Type, - to_channel: DebianChannel.Type, - profile: Profiles.Type, - remove_profile_from_name: Bool, - step_key: Text, - `if`: Optional B/If - }, - default = { - deps = [] : List Command.TaggedKey.Type, - package = Package.Type.LogProc, - version = "", - new_version = "", - architecture = "amd64", - network = Network.Type.Berkeley, - codename = DebianVersions.DebVersion.Bullseye, - from_channel = DebianChannel.Type.Unstable, - to_channel = DebianChannel.Type.Nightly, - profile = Profiles.Type.Standard, - remove_profile_from_name = False, - step_key = "promote-debian-package", - `if` = None B/If - } -} - -let PromoteDockerSpec = { - Type = { - deps: List Command.TaggedKey.Type, - name: Artifact.Type, - version: Text, - profile: Profiles.Type, - codename: DebianVersions.DebVersion, - new_tag: Text, - network: Network.Type, - step_key: Text, - `if`: Optional B/If, - publish: Bool, - remove_profile_from_name: Bool - }, - default = { - deps = [] : List Command.TaggedKey.Type, - name = Artifact.Type.Daemon, - version = "", - new_tag = "", - step_key = "promote-docker", - profile = Profiles.Type.Standard, - network = Network.Type.Berkeley, - codename = DebianVersions.DebVersion.Bullseye, - `if` = None B/If, - publish = False, - remove_profile_from_name = False - } -} - - -let promoteDebianStep = \(spec : PromoteDebianSpec.Type) -> - let package_name : Text = Package.debianName spec.package spec.profile spec.network - let new_name = if spec.remove_profile_from_name then "--new-name ${Package.debianName spec.package Profiles.Type.Standard spec.network}" else "" - in - Command.build - Command.Config::{ - commands = Toolchain.runner DebianVersions.DebVersion.Bullseye [ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY" - ] "./buildkite/scripts/promote-deb.sh --package ${package_name} --version ${spec.version} --new-version ${spec.new_version} --architecture ${spec.architecture} --codename ${DebianVersions.lowerName spec.codename} --from-component ${DebianChannel.lowerName spec.from_channel} --to-component ${DebianChannel.lowerName spec.to_channel} ${new_name}", - label = "Debian: ${spec.step_key}", - key = spec.step_key, - target = Size.XLarge, - depends_on = spec.deps, - `if` = spec.`if` +let PromoteDebianSpec = + { Type = + { deps : List Command.TaggedKey.Type + , package : Package.Type + , version : Text + , new_version : Text + , architecture : Text + , network : Network.Type + , codename : DebianVersions.DebVersion + , from_channel : DebianChannel.Type + , to_channel : DebianChannel.Type + , profile : Profiles.Type + , remove_profile_from_name : Bool + , step_key : Text + , if : Optional B/If + } + , default = + { deps = [] : List Command.TaggedKey.Type + , package = Package.Type.LogProc + , version = "" + , new_version = "" + , architecture = "amd64" + , network = Network.Type.Berkeley + , codename = DebianVersions.DebVersion.Bullseye + , from_channel = DebianChannel.Type.Unstable + , to_channel = DebianChannel.Type.Nightly + , profile = Profiles.Type.Standard + , remove_profile_from_name = False + , step_key = "promote-debian-package" + , if = None B/If + } } -let promoteDebianVerificationStep = \(spec : PromoteDebianSpec.Type) -> - let name = if spec.remove_profile_from_name then "${Package.debianName spec.package Profiles.Type.Standard spec.network}" else (Package.debianName spec.package spec.profile spec.network) - in - Command.build - Command.Config::{ - commands = [ - Cmd.run "./scripts/debian/verify.sh --package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName spec.codename} --channel ${DebianChannel.lowerName spec.to_channel}" - ], - label = "Debian: ${spec.step_key}", - key = spec.step_key, - target = Size.Small, - depends_on = spec.deps, - `if` = spec.`if` +let PromoteDockerSpec = + { Type = + { deps : List Command.TaggedKey.Type + , name : Artifact.Type + , version : Text + , profile : Profiles.Type + , codename : DebianVersions.DebVersion + , new_tag : Text + , network : Network.Type + , step_key : Text + , if : Optional B/If + , publish : Bool + , remove_profile_from_name : Bool + } + , default = + { deps = [] : List Command.TaggedKey.Type + , name = Artifact.Type.Daemon + , version = "" + , new_tag = "" + , step_key = "promote-docker" + , profile = Profiles.Type.Standard + , network = Network.Type.Berkeley + , codename = DebianVersions.DebVersion.Bullseye + , if = None B/If + , publish = False + , remove_profile_from_name = False + } } -let promoteDockerStep = \(spec : PromoteDockerSpec.Type) -> - let old_tag = Artifact.dockerTag spec.name spec.version spec.codename spec.profile spec.network False - let new_tag = Artifact.dockerTag spec.name spec.new_tag spec.codename spec.profile spec.network spec.remove_profile_from_name - let publish = if spec.publish then "-p" else "" - in - Command.build - Command.Config::{ - commands = [ - Cmd.run "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName spec.name} --version ${old_tag} --tag ${new_tag} ${publish}" - ], - label = "Docker: ${spec.step_key}", - key = spec.step_key, - target = Size.XLarge, - depends_on = spec.deps, - `if` = spec.`if` - } +let promoteDebianStep = + \(spec : PromoteDebianSpec.Type) + -> let package_name + : Text + = Package.debianName spec.package spec.profile spec.network -let promoteDockerVerificationStep = \(spec : PromoteDockerSpec.Type) -> - let new_tag = Artifact.dockerTag spec.name spec.new_tag spec.codename spec.profile spec.network spec.remove_profile_from_name - let repo = if spec.publish then "docker.io/minaprotocol" else "gcr.io/o1labs-192920" - in - Command.build - Command.Config::{ - commands = [ - Cmd.run "docker pull ${repo}/${Artifact.dockerName spec.name}:${new_tag}" - ], - label = "Docker: ${spec.step_key}", - key = spec.step_key, - target = Size.Small, - depends_on = spec.deps, - `if` = spec.`if` - } + let new_name = + if spec.remove_profile_from_name + then "--new-name ${Package.debianName + spec.package + Profiles.Type.Standard + spec.network}" -let promotePipeline : List PromoteDebianSpec.Type -> - List PromoteDockerSpec.Type -> - DebianVersions.DebVersion -> - PipelineMode.Type -> - Pipeline.Config.Type = - \(debians_spec : List PromoteDebianSpec.Type) -> - \(dockers_spec : List PromoteDockerSpec.Type) -> - \(debVersion: DebianVersions.DebVersion) -> - \(mode: PipelineMode.Type) -> - - let steps = - (List/map - PromoteDebianSpec.Type - Command.Type - (\(spec: PromoteDebianSpec.Type ) -> promoteDebianStep spec ) - debians_spec - ) - # - (List/map - PromoteDockerSpec.Type - Command.Type - (\(spec: PromoteDockerSpec.Type ) -> promoteDockerStep spec ) - dockers_spec - ) - in - --- tags are empty on purpose not to allow run job from automatically triggered pipelines - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = DebianVersions.dirtyWhen debVersion, - path = "Release", - name = "PromotePackage", - tags = []: List PipelineTag.Type, - mode = mode - }, - steps = steps - } + else "" -let verifyPipeline : List PromoteDebianSpec.Type -> - List PromoteDockerSpec.Type -> - DebianVersions.DebVersion -> - PipelineMode.Type -> - Pipeline.Config.Type = - \(debians_spec : List PromoteDebianSpec.Type) -> - \(dockers_spec : List PromoteDockerSpec.Type) -> - \(debVersion: DebianVersions.DebVersion) -> - \(mode: PipelineMode.Type) -> - - let steps = - (List/map - PromoteDebianSpec.Type - Command.Type - (\(spec: PromoteDebianSpec.Type ) -> promoteDebianVerificationStep spec ) - debians_spec - ) - # - (List/map - PromoteDockerSpec.Type - Command.Type - (\(spec: PromoteDockerSpec.Type ) -> promoteDockerVerificationStep spec ) - dockers_spec - ) - in - --- tags are empty on purpose not to allow run job from automatically triggered pipelines - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = DebianVersions.dirtyWhen debVersion, - path = "Release", - name = "VerifyPackage", - tags = []: List PipelineTag.Type, - mode = mode - }, - steps = steps - } + in Command.build + Command.Config::{ + , commands = + Toolchain.runner + DebianVersions.DebVersion.Bullseye + [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" ] + "./buildkite/scripts/promote-deb.sh --package ${package_name} --version ${spec.version} --new-version ${spec.new_version} --architecture ${spec.architecture} --codename ${DebianVersions.lowerName + spec.codename} --from-component ${DebianChannel.lowerName + spec.from_channel} --to-component ${DebianChannel.lowerName + spec.to_channel} ${new_name}" + , label = "Debian: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , depends_on = spec.deps + , if = spec.if + } + +let promoteDebianVerificationStep = + \(spec : PromoteDebianSpec.Type) + -> let name = + if spec.remove_profile_from_name + + then "${Package.debianName + spec.package + Profiles.Type.Standard + spec.network}" + + else Package.debianName spec.package spec.profile spec.network + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./scripts/debian/verify.sh --package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName + spec.codename} --channel ${DebianChannel.lowerName + spec.to_channel}" + ] + , label = "Debian: ${spec.step_key}" + , key = spec.step_key + , target = Size.Small + , depends_on = spec.deps + , if = spec.if + } + +let promoteDockerStep = + \(spec : PromoteDockerSpec.Type) + -> let old_tag = + Artifact.dockerTag + spec.name + spec.version + spec.codename + spec.profile + spec.network + False + + let new_tag = + Artifact.dockerTag + spec.name + spec.new_tag + spec.codename + spec.profile + spec.network + spec.remove_profile_from_name + + let publish = if spec.publish then "-p" else "" + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName + spec.name} --version ${old_tag} --tag ${new_tag} ${publish}" + ] + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , depends_on = spec.deps + , if = spec.if + } + +let promoteDockerVerificationStep = + \(spec : PromoteDockerSpec.Type) + -> let new_tag = + Artifact.dockerTag + spec.name + spec.new_tag + spec.codename + spec.profile + spec.network + spec.remove_profile_from_name -in - -{ promoteDebianStep = promoteDebianStep - , promoteDockerStep = promoteDockerStep - , promoteDebianVerificationStep = promoteDebianVerificationStep - , promoteDockerVerificationStep = promoteDockerVerificationStep - , promotePipeline = promotePipeline - , verifyPipeline = verifyPipeline - , PromoteDebianSpec = PromoteDebianSpec - , PromoteDockerSpec = PromoteDockerSpec -} \ No newline at end of file + let repo = + if spec.publish + + then "docker.io/minaprotocol" + + else "gcr.io/o1labs-192920" + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "docker pull ${repo}/${Artifact.dockerName + spec.name}:${new_tag}" + ] + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.Small + , depends_on = spec.deps + , if = spec.if + } + +let promotePipeline + : List PromoteDebianSpec.Type + -> List PromoteDockerSpec.Type + -> DebianVersions.DebVersion + -> PipelineMode.Type + -> Pipeline.Config.Type + = \(debians_spec : List PromoteDebianSpec.Type) + -> \(dockers_spec : List PromoteDockerSpec.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(mode : PipelineMode.Type) + -> let steps = + List/map + PromoteDebianSpec.Type + Command.Type + (\(spec : PromoteDebianSpec.Type) -> promoteDebianStep spec) + debians_spec + # List/map + PromoteDockerSpec.Type + Command.Type + (\(spec : PromoteDockerSpec.Type) -> promoteDockerStep spec) + dockers_spec + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen debVersion + , path = "Release" + , name = "PromotePackage" + , tags = [] : List PipelineTag.Type + , mode = mode + } + , steps = steps + } + +let verifyPipeline + : List PromoteDebianSpec.Type + -> List PromoteDockerSpec.Type + -> DebianVersions.DebVersion + -> PipelineMode.Type + -> Pipeline.Config.Type + = \(debians_spec : List PromoteDebianSpec.Type) + -> \(dockers_spec : List PromoteDockerSpec.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(mode : PipelineMode.Type) + -> let steps = + List/map + PromoteDebianSpec.Type + Command.Type + ( \(spec : PromoteDebianSpec.Type) + -> promoteDebianVerificationStep spec + ) + debians_spec + # List/map + PromoteDockerSpec.Type + Command.Type + ( \(spec : PromoteDockerSpec.Type) + -> promoteDockerVerificationStep spec + ) + dockers_spec + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen debVersion + , path = "Release" + , name = "VerifyPackage" + , tags = [] : List PipelineTag.Type + , mode = mode + } + , steps = steps + } + +in { promoteDebianStep = promoteDebianStep + , promoteDockerStep = promoteDockerStep + , promoteDebianVerificationStep = promoteDebianVerificationStep + , promoteDockerVerificationStep = promoteDockerVerificationStep + , promotePipeline = promotePipeline + , verifyPipeline = verifyPipeline + , PromoteDebianSpec = PromoteDebianSpec + , PromoteDockerSpec = PromoteDockerSpec + } diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index ce45f13b8bd..02c02978836 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -1,29 +1,25 @@ -let B = ../External/Buildkite.dhall - -let Prelude = ../External/Prelude.dhall let Artifacts = ../Constants/Artifacts.dhall + let Command = ./Base.dhall -let Docker = ./Docker/Type.dhall -let Size = ./Size.dhall -let RunWithPostgres = ./RunWithPostgres.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let Size = ./Size.dhall -let Cmd = ../Lib/Cmds.dhall in +let RunWithPostgres = ./RunWithPostgres.dhall -{ step = \(dependsOn : List Command.TaggedKey.Type) -> - Command.build - Command.Config::{ - commands = [ - RunWithPostgres.runInDockerWithPostgresConn - ([] : List Text) - "./src/test/archive/sample_db/archive_db.sql" - Artifacts.Type.Archive - "./buildkite/scripts/replayer-test.sh" - ], - label = "Archive: Replayer test", - key = "replayer-test", - target = Size.Large, - depends_on = dependsOn - } -} +in { step = + \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + [ RunWithPostgres.runInDockerWithPostgresConn + ([] : List Text) + "./src/test/archive/sample_db/archive_db.sql" + Artifacts.Type.Archive + "./buildkite/scripts/replayer-test.sh" + ] + , label = "Archive: Replayer test" + , key = "replayer-test" + , target = Size.Large + , depends_on = dependsOn + } + } diff --git a/buildkite/src/Command/RunInToolchain.dhall b/buildkite/src/Command/RunInToolchain.dhall index 60c2b7444a1..6d4616b605d 100644 --- a/buildkite/src/Command/RunInToolchain.dhall +++ b/buildkite/src/Command/RunInToolchain.dhall @@ -1,54 +1,59 @@ -let Prelude = ../External/Prelude.dhall let Cmd = ../Lib/Cmds.dhall -let Mina = ../Command/Mina.dhall -let S = ../Lib/SelectFiles.dhall -let ContainerImages = ../Constants/ContainerImages.dhall - -let r = Cmd.run - -let runInToolchainImage : Text -> List Text -> Text -> List Cmd.Type = - \(image: Text) -> - \(environment : List Text) -> - \(innerScript : Text) -> - [ Mina.fixPermissionsCommand ] # [ - Cmd.runInDocker - (Cmd.Docker::{ image = image, extraEnv = environment }) - (innerScript) - ] - -in - -let runInToolchainBookworm : List Text -> Text -> List Cmd.Type = - \(environment : List Text) -> - \(innerScript : Text) -> - runInToolchainImage ContainerImages.minaToolchainBookworm environment innerScript - -in - -let runInToolchainBullseye : List Text -> Text -> List Cmd.Type = - \(environment : List Text) -> - \(innerScript : Text) -> - runInToolchainImage ContainerImages.minaToolchainBullseye environment innerScript - -in - -let runInToolchainBuster : List Text -> Text -> List Cmd.Type = - \(environment : List Text) -> - \(innerScript : Text) -> - runInToolchainImage ContainerImages.minaToolchainBuster environment innerScript - -let runInToolchain : List Text -> Text -> List Cmd.Type = - \(environment : List Text) -> - \(innerScript : Text) -> - runInToolchainImage ContainerImages.minaToolchain environment innerScript +let Mina = ../Command/Mina.dhall -in +let ContainerImages = ../Constants/ContainerImages.dhall -{ - runInToolchain = runInToolchain - , runInToolchainImage = runInToolchainImage - , runInToolchainBookworm = runInToolchainBookworm - , runInToolchainBullseye = runInToolchainBullseye - , runInToolchainBuster = runInToolchainBuster -} +let runInToolchainImage + : Text -> List Text -> Text -> List Cmd.Type + = \(image : Text) + -> \(environment : List Text) + -> \(innerScript : Text) + -> [ Mina.fixPermissionsCommand ] + # [ Cmd.runInDocker + Cmd.Docker::{ image = image, extraEnv = environment } + innerScript + ] + +let runInToolchainBookworm + : List Text -> Text -> List Cmd.Type + = \(environment : List Text) + -> \(innerScript : Text) + -> runInToolchainImage + ContainerImages.minaToolchainBookworm + environment + innerScript + +let runInToolchainBullseye + : List Text -> Text -> List Cmd.Type + = \(environment : List Text) + -> \(innerScript : Text) + -> runInToolchainImage + ContainerImages.minaToolchainBullseye + environment + innerScript + +let runInToolchainBuster + : List Text -> Text -> List Cmd.Type + = \(environment : List Text) + -> \(innerScript : Text) + -> runInToolchainImage + ContainerImages.minaToolchainBuster + environment + innerScript + +let runInToolchain + : List Text -> Text -> List Cmd.Type + = \(environment : List Text) + -> \(innerScript : Text) + -> runInToolchainImage + ContainerImages.minaToolchain + environment + innerScript + +in { runInToolchain = runInToolchain + , runInToolchainImage = runInToolchainImage + , runInToolchainBookworm = runInToolchainBookworm + , runInToolchainBullseye = runInToolchainBullseye + , runInToolchainBuster = runInToolchainBuster + } diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index f309df20a66..f4547181231 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -1,58 +1,61 @@ let Prelude = ../External/Prelude.dhall + let P = Prelude + let Text/concatMap = P.Text.concatMap let Cmd = ../Lib/Cmds.dhall -let Mina = ../Command/Mina.dhall -let S = ../Lib/SelectFiles.dhall + let ContainerImages = ../Constants/ContainerImages.dhall -let Artifacts= ../Constants/Artifacts.dhall - -let r = Cmd.run - -let runInDockerWithPostgresConn : List Text -> Text -> Artifacts.Type -> Text -> Cmd.Type = - \(environment : List Text) -> - \(initScript: Text ) -> - \(docker: Artifacts.Type) -> - \(innerScript : Text) -> - -- port is used only in pg_conn variable. By using --network host we are disabling - -- any control over postgres port - let port="5432" - let user="postgres" - let password="postgres" - let postgresDockerName="postgres" - let dockerVersion= ContainerImages.postgres - let dbName="archive" - let pg_conn="postgres://${user}:${password}@localhost:${port}/${dbName}" - - let envVars = - Text/concatMap - Text - (\(var : Text) -> " --env ${var}") - ([ - "PG_PORT=${port}", - "POSTGRES_USER=${user}", - "POSTGRES_PASSWORD=${password}", - "POSTGRES_DB=${dbName}", - "PG_CONN=${pg_conn}" - ] # environment) - in - - let outerDir : Text = - "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" - in - - Cmd.chain [ - "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true", - "source buildkite/scripts/export-git-env-vars.sh", - "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}", - "sleep 5", - "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}", - "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName docker}:\\\$MINA_DOCKER_TAG ${innerScript}" - ] - -in - -{ - runInDockerWithPostgresConn = runInDockerWithPostgresConn -} \ No newline at end of file + +let Artifacts = ../Constants/Artifacts.dhall + +let runInDockerWithPostgresConn + : List Text -> Text -> Artifacts.Type -> Text -> Cmd.Type + = \(environment : List Text) + -> \(initScript : Text) + -> \(docker : Artifacts.Type) + -> \(innerScript : Text) + -> let port = "5432" + + let user = "postgres" + + let password = "postgres" + + let postgresDockerName = "postgres" + + let dockerVersion = ContainerImages.postgres + + let dbName = "archive" + + let pg_conn = + "postgres://${user}:${password}@localhost:${port}/${dbName}" + + let envVars = + Text/concatMap + Text + (\(var : Text) -> " --env ${var}") + ( [ "PG_PORT=${port}" + , "POSTGRES_USER=${user}" + , "POSTGRES_PASSWORD=${password}" + , "POSTGRES_DB=${dbName}" + , "PG_CONN=${pg_conn}" + ] + # environment + ) + + let outerDir + : Text + = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" + + in Cmd.chain + [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true" + , "source buildkite/scripts/export-git-env-vars.sh" + , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}" + , "sleep 5" + , "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}" + , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName + docker}:\\\$MINA_DOCKER_TAG ${innerScript}" + ] + +in { runInDockerWithPostgresConn = runInDockerWithPostgresConn } diff --git a/buildkite/src/Command/Size.dhall b/buildkite/src/Command/Size.dhall index 014fcaba9fb..4d565bcbd56 100644 --- a/buildkite/src/Command/Size.dhall +++ b/buildkite/src/Command/Size.dhall @@ -1 +1 @@ - +< XLarge | Large | Medium | Small | Integration | QA | Hardfork > diff --git a/buildkite/src/Command/Summon/Type.dhall b/buildkite/src/Command/Summon/Type.dhall index 01cc4972120..20bfb3d8c80 100644 --- a/buildkite/src/Command/Summon/Type.dhall +++ b/buildkite/src/Command/Summon/Type.dhall @@ -3,17 +3,16 @@ -- See https://github.com/angaza/summon-buildkite-plugin for options -- if you'd like to extend this definition for example -{ - Type = { - `secrets-file`: Text, - provider: Text, - environment: Text, - substitutions: List Text - }, - default = { - `secrets-file` = "./secrets.yml", - provider = "summon-aws-secrets", - environment = "", - substitutions = [] : List Text - } +{ Type = + { secrets-file : Text + , provider : Text + , environment : Text + , substitutions : List Text + } +, default = + { secrets-file = "./secrets.yml" + , provider = "summon-aws-secrets" + , environment = "" + , substitutions = [] : List Text + } } diff --git a/buildkite/src/Command/TestExecutive.dhall b/buildkite/src/Command/TestExecutive.dhall index 62c8e66bc6f..6fa3bb0340e 100644 --- a/buildkite/src/Command/TestExecutive.dhall +++ b/buildkite/src/Command/TestExecutive.dhall @@ -1,42 +1,41 @@ -let Prelude = ../External/Prelude.dhall - let Command = ./Base.dhall -let Docker = ./Docker/Type.dhall -let Size = ./Size.dhall -let RunInToolchain = ../Command/RunInToolchain.dhall +let Size = ./Size.dhall let Cmd = ../Lib/Cmds.dhall -let SelectFiles = ../Lib/SelectFiles.dhall - -in -{ - executeCloud = \(testName : Text) -> \(dependsOn : List Command.TaggedKey.Type) -> - Command.build - Command.Config::{ - commands = - [ - -- Execute test based on BUILD image - Cmd.run "MINA_DEB_CODENAME=bullseye ; source ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/run-test-executive-cloud.sh ${testName}" - ], - artifact_paths = [SelectFiles.exactly "." "${testName}.cloud.test.log"], - label = "${testName} integration test", - key = "integration-test-${testName}", - target = Size.Integration, - depends_on = dependsOn - }, +let SelectFiles = ../Lib/SelectFiles.dhall - executeLocal = \(testName : Text) -> \(dependsOn : List Command.TaggedKey.Type) -> - Command.build - Command.Config::{ - commands = [ - Cmd.run "MINA_DEB_CODENAME=bullseye ; source ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/run-test-executive-local.sh ${testName}" - ], - artifact_paths = [SelectFiles.exactly "." "${testName}.local.test.log"], - label = "${testName} integration test local", - key = "integration-test-${testName}-local", - target = Size.Integration, - depends_on = dependsOn - } -} +in { executeCloud = + \(testName : Text) + -> \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + [ Cmd.run + "MINA_DEB_CODENAME=bullseye ; source ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/run-test-executive-cloud.sh ${testName}" + ] + , artifact_paths = + [ SelectFiles.exactly "." "${testName}.cloud.test.log" ] + , label = "${testName} integration test" + , key = "integration-test-${testName}" + , target = Size.Integration + , depends_on = dependsOn + } + , executeLocal = + \(testName : Text) + -> \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + [ Cmd.run + "MINA_DEB_CODENAME=bullseye ; source ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/run-test-executive-local.sh ${testName}" + ] + , artifact_paths = + [ SelectFiles.exactly "." "${testName}.local.test.log" ] + , label = "${testName} integration test local" + , key = "integration-test-${testName}-local" + , target = Size.Integration + , depends_on = dependsOn + } + } diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index 3a92eab939c..715d94d48e5 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -1,106 +1,145 @@ let Prelude = ../External/Prelude.dhall + let Text/concatSep = Prelude.Text.concatSep + let Profiles = ./Profiles.dhall + let DebianVersions = ./DebianVersions.dhall -let Network = ./Network.dhall +let Network = ./Network.dhall -let Artifact : Type = < Daemon | Archive | ArchiveMigration | TestExecutive | BatchTxn | Rosetta | ZkappTestTransaction | FunctionalTestSuite > +let Artifact + : Type + = < Daemon + | Archive + | ArchiveMigration + | TestExecutive + | BatchTxn + | Rosetta + | ZkappTestTransaction + | FunctionalTestSuite + > -let AllButTests = [ Artifact.Daemon , Artifact.Archive , Artifact.ArchiveMigration , Artifact.BatchTxn , Artifact.TestExecutive , Artifact.Rosetta , Artifact.ZkappTestTransaction ] +let AllButTests = + [ Artifact.Daemon + , Artifact.Archive + , Artifact.ArchiveMigration + , Artifact.BatchTxn + , Artifact.TestExecutive + , Artifact.Rosetta + , Artifact.ZkappTestTransaction + ] -let Main = [ Artifact.Daemon , Artifact.Archive , Artifact.Rosetta ] +let Main = [ Artifact.Daemon, Artifact.Archive, Artifact.Rosetta ] let All = AllButTests # [ Artifact.FunctionalTestSuite ] -let capitalName = \(artifact : Artifact) -> - merge { - Daemon = "Daemon" - , Archive = "Archive" - , ArchiveMigration = "ArchiveMigration" - , TestExecutive = "TestExecutive" - , BatchTxn = "BatchTxn" - , Rosetta = "Rosetta" - , ZkappTestTransaction = "ZkappTestTransaction" - , FunctionalTestSuite = "FunctionalTestSuite" - } artifact - -let lowerName = \(artifact : Artifact) -> - merge { - Daemon = "daemon" - , Archive = "archive" - , ArchiveMigration = "archive_migration" - , TestExecutive = "test_executive" - , BatchTxn = "batch_txn" - , Rosetta = "rosetta" - , ZkappTestTransaction = "zkapp_test_transaction" - , FunctionalTestSuite = "functional_test_suite" - } artifact - -let dockerName = \(artifact : Artifact) -> - merge { - Daemon = "mina-daemon" - , Archive = "mina-archive" - , TestExecutive = "mina-test-executive" - , ArchiveMigration = "mina-archive-migration" - , BatchTxn = "mina-batch-txn" - , Rosetta = "mina-rosetta" - , ZkappTestTransaction = "mina-zkapp-test-transaction" - , FunctionalTestSuite = "mina-test-suite" - } artifact - -let toDebianName = \(artifact : Artifact) -> - merge { - Daemon = "daemon" - , Archive = "archive" - , ArchiveMigration = "archive_migration" - , TestExecutive = "test_executive" - , BatchTxn = "batch_txn" - , Rosetta = "" - , ZkappTestTransaction = "zkapp_test_transaction" - , FunctionalTestSuite = "functional_test_suite" - } artifact - -let toDebianNames = \(artifacts : List Artifact) -> - let text = Prelude.List.map - Artifact - Text - (\(a: Artifact) -> toDebianName a ) - artifacts - in - Text/concatSep " " text - -let dockerTag = \(artifact: Artifact) - -> \(version: Text) - -> \(codename: DebianVersions.DebVersion) - -> \(profile: Profiles.Type) - -> \(network: Network.Type) - -> \(remove_profile_from_name: Bool) - -> - let version_and_codename = "${version}-${DebianVersions.lowerName codename}" - let profile_part = if remove_profile_from_name then "" else "${Profiles.toLabelSegment profile}" - in - merge { - Daemon ="${version_and_codename}-${Network.lowerName network}${profile_part}" - , Archive = "${version_and_codename}" - , ArchiveMigration = "${version_and_codename}" - , TestExecutive = "${version_and_codename}" - , BatchTxn = "${version_and_codename}" - , Rosetta = "${version_and_codename}" - , ZkappTestTransaction = "${version_and_codename}" - , FunctionalTestSuite = "${version_and_codename}" - } artifact -in - -{ - Type = Artifact - , capitalName = capitalName - , lowerName = lowerName - , toDebianName = toDebianName - , toDebianNames = toDebianNames - , dockerName = dockerName - , dockerTag = dockerTag - , All = All - , AllButTests = AllButTests - , Main = Main -} \ No newline at end of file +let capitalName = + \(artifact : Artifact) + -> merge + { Daemon = "Daemon" + , Archive = "Archive" + , ArchiveMigration = "ArchiveMigration" + , TestExecutive = "TestExecutive" + , BatchTxn = "BatchTxn" + , Rosetta = "Rosetta" + , ZkappTestTransaction = "ZkappTestTransaction" + , FunctionalTestSuite = "FunctionalTestSuite" + } + artifact + +let lowerName = + \(artifact : Artifact) + -> merge + { Daemon = "daemon" + , Archive = "archive" + , ArchiveMigration = "archive_migration" + , TestExecutive = "test_executive" + , BatchTxn = "batch_txn" + , Rosetta = "rosetta" + , ZkappTestTransaction = "zkapp_test_transaction" + , FunctionalTestSuite = "functional_test_suite" + } + artifact + +let dockerName = + \(artifact : Artifact) + -> merge + { Daemon = "mina-daemon" + , Archive = "mina-archive" + , TestExecutive = "mina-test-executive" + , ArchiveMigration = "mina-archive-migration" + , BatchTxn = "mina-batch-txn" + , Rosetta = "mina-rosetta" + , ZkappTestTransaction = "mina-zkapp-test-transaction" + , FunctionalTestSuite = "mina-test-suite" + } + artifact + +let toDebianName = + \(artifact : Artifact) + -> merge + { Daemon = "daemon" + , Archive = "archive" + , ArchiveMigration = "archive_migration" + , TestExecutive = "test_executive" + , BatchTxn = "batch_txn" + , Rosetta = "" + , ZkappTestTransaction = "zkapp_test_transaction" + , FunctionalTestSuite = "functional_test_suite" + } + artifact + +let toDebianNames = + \(artifacts : List Artifact) + -> let text = + Prelude.List.map + Artifact + Text + (\(a : Artifact) -> toDebianName a) + artifacts + + in Text/concatSep " " text + +let dockerTag = + \(artifact : Artifact) + -> \(version : Text) + -> \(codename : DebianVersions.DebVersion) + -> \(profile : Profiles.Type) + -> \(network : Network.Type) + -> \(remove_profile_from_name : Bool) + -> let version_and_codename = + "${version}-${DebianVersions.lowerName codename}" + + let profile_part = + if remove_profile_from_name + + then "" + + else "${Profiles.toLabelSegment profile}" + + in merge + { Daemon = + "${version_and_codename}-${Network.lowerName + network}${profile_part}" + , Archive = "${version_and_codename}" + , ArchiveMigration = "${version_and_codename}" + , TestExecutive = "${version_and_codename}" + , BatchTxn = "${version_and_codename}" + , Rosetta = "${version_and_codename}" + , ZkappTestTransaction = "${version_and_codename}" + , FunctionalTestSuite = "${version_and_codename}" + } + artifact + +in { Type = Artifact + , capitalName = capitalName + , lowerName = lowerName + , toDebianName = toDebianName + , toDebianNames = toDebianNames + , dockerName = dockerName + , dockerTag = dockerTag + , All = All + , AllButTests = AllButTests + , Main = Main + } diff --git a/buildkite/src/Constants/BuildFlags.dhall b/buildkite/src/Constants/BuildFlags.dhall index b0727fc6d56..1ce9afb36ae 100644 --- a/buildkite/src/Constants/BuildFlags.dhall +++ b/buildkite/src/Constants/BuildFlags.dhall @@ -1,51 +1,40 @@ -let Prelude = ../External/Prelude.dhall - -let BuildFlags : Type = < None | Instrumented > - -let capitalName = \(buildFlags : BuildFlags) -> - merge { - None = "None" - , Instrumented = "Instrumented" - } buildFlags - -let lowerName = \(buildFlags : BuildFlags) -> - merge { - None = "none" - , Instrumented = "instrumented" - } buildFlags - -let buildEnvs = \(buildFlags : BuildFlags) -> - merge { - None = ([] : List Text ) - , Instrumented = ["DUNE_INSTRUMENT_WITH=bisect_ppx"] - } buildFlags - -let toSuffixUppercase = \(buildFlags : BuildFlags) -> - merge { - None = "" - , Instrumented = "Instrumented" - } buildFlags - -let toSuffixLowercase = \(buildFlags : BuildFlags) -> - merge { - None = "" - , Instrumented = "instrumented" - } buildFlags - -let toLabelSegment = \(buildFlags : BuildFlags) -> - merge { - None = "" - , Instrumented = "-instrumented" - } buildFlags - -in - -{ - Type = BuildFlags - , capitalName = capitalName - , lowerName = lowerName - , buildEnvs = buildEnvs - , toSuffixUppercase = toSuffixUppercase - , toSuffixLowercase = toSuffixLowercase - , toLabelSegment = toLabelSegment -} \ No newline at end of file +let BuildFlags + : Type + = < None | Instrumented > + +let capitalName = + \(buildFlags : BuildFlags) + -> merge { None = "None", Instrumented = "Instrumented" } buildFlags + +let lowerName = + \(buildFlags : BuildFlags) + -> merge { None = "none", Instrumented = "instrumented" } buildFlags + +let buildEnvs = + \(buildFlags : BuildFlags) + -> merge + { None = [] : List Text + , Instrumented = [ "DUNE_INSTRUMENT_WITH=bisect_ppx" ] + } + buildFlags + +let toSuffixUppercase = + \(buildFlags : BuildFlags) + -> merge { None = "", Instrumented = "Instrumented" } buildFlags + +let toSuffixLowercase = + \(buildFlags : BuildFlags) + -> merge { None = "", Instrumented = "instrumented" } buildFlags + +let toLabelSegment = + \(buildFlags : BuildFlags) + -> merge { None = "", Instrumented = "-instrumented" } buildFlags + +in { Type = BuildFlags + , capitalName = capitalName + , lowerName = lowerName + , buildEnvs = buildEnvs + , toSuffixUppercase = toSuffixUppercase + , toSuffixLowercase = toSuffixLowercase + , toLabelSegment = toLabelSegment + } diff --git a/buildkite/src/Constants/ContainerEnvVars.dhall b/buildkite/src/Constants/ContainerEnvVars.dhall index 034b38815db..bbefb6d3962 100644 --- a/buildkite/src/Constants/ContainerEnvVars.dhall +++ b/buildkite/src/Constants/ContainerEnvVars.dhall @@ -51,4 +51,3 @@ , "NIX_CACHE_GCP_SECRET" , "SLOT_TX_END" ] - diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall index ecc2a4250a5..ff3cd4d0d58 100644 --- a/buildkite/src/Constants/ContainerImages.dhall +++ b/buildkite/src/Constants/ContainerImages.dhall @@ -2,17 +2,22 @@ -- NOTE: minaToolchain is the default image for various jobs, set to minaToolchainBullseye -- NOTE: minaToolchainBullseye is also used for building Ubuntu Focal packages in CI -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI -{ - toolchainBase = "codaprotocol/ci-toolchain-base:v3", - minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:d27c15e3143a99b86155ba57696020c00e2a296b388499b3e6fb364478ddda3a", - minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f", - minaToolchainBookworm = "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f", - minaToolchain = "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f", - delegationBackendToolchain = "gcr.io/o1labs-192920/delegation-backend-production@sha256:12ffd0a9016819c720687f440c7a46b8815f8d3ad06d306d342ee5f8dd4375f5", - elixirToolchain = "elixir:1.10-alpine", - nodeToolchain = "node:14.13.1-stretch-slim", - ubuntu2004 = "ubuntu:20.04", - postgres = "postgres:12.4-alpine", - xrefcheck = "serokell/xrefcheck@sha256:8fbb35a909abc353364f1bd3148614a1160ef3c111c0c4ae84e58fdf16019eeb", - nixos = "gcr.io/o1labs-192920/nix-unstable:1.0.0" +{ toolchainBase = "codaprotocol/ci-toolchain-base:v3" +, minaToolchainBuster = + "gcr.io/o1labs-192920/mina-toolchain@sha256:d27c15e3143a99b86155ba57696020c00e2a296b388499b3e6fb364478ddda3a" +, minaToolchainBullseye = + "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f" +, minaToolchainBookworm = + "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f" +, minaToolchain = + "gcr.io/o1labs-192920/mina-toolchain@sha256:966863de43c72c294e14762ae567404005f99654c54338a9a89b999476a36d1f" +, delegationBackendToolchain = + "gcr.io/o1labs-192920/delegation-backend-production@sha256:12ffd0a9016819c720687f440c7a46b8815f8d3ad06d306d342ee5f8dd4375f5" +, elixirToolchain = "elixir:1.10-alpine" +, nodeToolchain = "node:14.13.1-stretch-slim" +, ubuntu2004 = "ubuntu:20.04" +, postgres = "postgres:12.4-alpine" +, xrefcheck = + "serokell/xrefcheck@sha256:8fbb35a909abc353364f1bd3148614a1160ef3c111c0c4ae84e58fdf16019eeb" +, nixos = "gcr.io/o1labs-192920/nix-unstable:1.0.0" } diff --git a/buildkite/src/Constants/DebianChannel.dhall b/buildkite/src/Constants/DebianChannel.dhall index 5460712aaa9..73e66ffe32a 100644 --- a/buildkite/src/Constants/DebianChannel.dhall +++ b/buildkite/src/Constants/DebianChannel.dhall @@ -1,38 +1,47 @@ -let Prelude = ../External/Prelude.dhall +let Channel + : Type + = < Unstable + | Nightly + | Itn + | Umt + | UmtMainnet + | Devnet + | Alpha + | Beta + | Experimental + | Stable + > -let Channel : Type = < Unstable | Nightly | Itn | Umt | UmtMainnet | Devnet | Alpha | Beta | Experimental | Stable > +let capitalName = + \(channel : Channel) + -> merge + { Unstable = "Unstable" + , Nightly = "Nightly" + , Itn = "Itn" + , Umt = "Umt" + , UmtMainnet = "UmtMainnet" + , Devnet = "Devnet" + , Alpha = "Alpha" + , Beta = "Beta" + , Stable = "Stable" + , Experimental = "Experimental" + } + channel -let capitalName = \(channel : Channel) -> - merge { - Unstable = "Unstable" - , Nightly = "Nightly" - , Itn = "Itn" - , Umt = "Umt" - , UmtMainnet = "UmtMainnet" - , Devnet = "Devnet" - , Alpha = "Alpha" - , Beta = "Beta" - , Stable = "Stable" - , Experimental = "Experimental" - } channel +let lowerName = + \(channel : Channel) + -> merge + { Unstable = "unstable" + , Nightly = "nightly" + , Itn = "itn" + , Umt = "umt" + , UmtMainnet = "umt-mainnet" + , Devnet = "devnet" + , Alpha = "alpha" + , Beta = "beta" + , Stable = "stable" + , Experimental = "experimental" + } + channel -let lowerName = \(channel : Channel) -> - merge { - Unstable = "unstable" - , Nightly = "nightly" - , Itn = "itn" - , Umt = "umt" - , UmtMainnet = "umt-mainnet" - , Devnet = "devnet" - , Alpha = "alpha" - , Beta = "beta" - , Stable = "stable" - , Experimental = "experimental" - } channel - -in -{ - Type = Channel - , capitalName = capitalName - , lowerName = lowerName -} +in { Type = Channel, capitalName = capitalName, lowerName = lowerName } diff --git a/buildkite/src/Constants/DebianPackage.dhall b/buildkite/src/Constants/DebianPackage.dhall index c18799ef9af..8f38cf5d48f 100644 --- a/buildkite/src/Constants/DebianPackage.dhall +++ b/buildkite/src/Constants/DebianPackage.dhall @@ -1,55 +1,83 @@ let Network = ./Network.dhall + let Profile = ./Profiles.dhall -let Package : Type = < Daemon | Archive | ArchiveMigration | TestExecutive | BatchTxn | LogProc | ZkappTestTransaction | FunctionalTestSuite > - -let MainPackages = [ Package.Daemon , Package.Archive , Package.ArchiveMigration , Package.LogProc ] - -let AuxiliaryPackages = [ Package.TestExecutive , Package.BatchTxn , Package.ZkappTestTransaction , Package.FunctionalTestSuite ] - - -let capitalName = \(package : Package) -> - merge { - Daemon = "Daemon" - , Archive = "Archive" - , ArchiveMigration = "ArchiveMigration" - , TestExecutive = "TestExecutive" - , BatchTxn = "BatchTxn" - , LogProc = "Logproc" - , ZkappTestTransaction = "ZkappTestTransaction" - , FunctionalTestSuite = "FunctionalTestSuite" - } package - -let lowerName = \(package : Package) -> - merge { - Daemon = "daemon" - , Archive = "archive" - , ArchiveMigration = "archive_migration" - , TestExecutive = "test_executive" - , BatchTxn = "batch_txn" - , LogProc = "logproc" - , ZkappTestTransaction = "zkapp_test_transaction" - , FunctionalTestSuite = "functional_test_suite" - } package - -let debianName = \(package : Package) -> \(profile : Profile.Type) -> \(network : Network.Type) -> - merge { - Daemon = "mina-${Network.lowerName network}${Profile.toLabelSegment profile}" - , Archive = "mina-archive" - , ArchiveMigration = "mina-archive-migration" - , TestExecutive = "mina-test-executive" - , BatchTxn = "mina-batch-txn" - , LogProc = "mina-logproc" - , ZkappTestTransaction = "mina-zkapp-test-transaction" - , FunctionalTestSuite = "mina-test-suite" - } package - -in -{ - Type = Package - , MainPackages = MainPackages - , AuxiliaryPackages = AuxiliaryPackages - , capitalName = capitalName - , lowerName = lowerName - , debianName = debianName -} \ No newline at end of file +let Package + : Type + = < Daemon + | Archive + | ArchiveMigration + | TestExecutive + | BatchTxn + | LogProc + | ZkappTestTransaction + | FunctionalTestSuite + > + +let MainPackages = + [ Package.Daemon + , Package.Archive + , Package.ArchiveMigration + , Package.LogProc + ] + +let AuxiliaryPackages = + [ Package.TestExecutive + , Package.BatchTxn + , Package.ZkappTestTransaction + , Package.FunctionalTestSuite + ] + +let capitalName = + \(package : Package) + -> merge + { Daemon = "Daemon" + , Archive = "Archive" + , ArchiveMigration = "ArchiveMigration" + , TestExecutive = "TestExecutive" + , BatchTxn = "BatchTxn" + , LogProc = "Logproc" + , ZkappTestTransaction = "ZkappTestTransaction" + , FunctionalTestSuite = "FunctionalTestSuite" + } + package + +let lowerName = + \(package : Package) + -> merge + { Daemon = "daemon" + , Archive = "archive" + , ArchiveMigration = "archive_migration" + , TestExecutive = "test_executive" + , BatchTxn = "batch_txn" + , LogProc = "logproc" + , ZkappTestTransaction = "zkapp_test_transaction" + , FunctionalTestSuite = "functional_test_suite" + } + package + +let debianName = + \(package : Package) + -> \(profile : Profile.Type) + -> \(network : Network.Type) + -> merge + { Daemon = + "mina-${Network.lowerName network}${Profile.toLabelSegment + profile}" + , Archive = "mina-archive" + , ArchiveMigration = "mina-archive-migration" + , TestExecutive = "mina-test-executive" + , BatchTxn = "mina-batch-txn" + , LogProc = "mina-logproc" + , ZkappTestTransaction = "mina-zkapp-test-transaction" + , FunctionalTestSuite = "mina-test-suite" + } + package + +in { Type = Package + , MainPackages = MainPackages + , AuxiliaryPackages = AuxiliaryPackages + , capitalName = capitalName + , lowerName = lowerName + , debianName = debianName + } diff --git a/buildkite/src/Constants/DebianRepo.dhall b/buildkite/src/Constants/DebianRepo.dhall index 01a9b1a6952..98239c16858 100644 --- a/buildkite/src/Constants/DebianRepo.dhall +++ b/buildkite/src/Constants/DebianRepo.dhall @@ -1,15 +1,13 @@ -let Prelude = ../External/Prelude.dhall +let DebianRepo + : Type + = < Local | PackagesO1Test > -let DebianRepo : Type = < Local | PackagesO1Test > +let address = + \(repo : DebianRepo) + -> merge + { Local = "http://localhost:8080" + , PackagesO1Test = "http://packages.o1test.net" + } + repo -let address = \(repo : DebianRepo) -> - merge { - Local = "http://localhost:8080", - PackagesO1Test = "http://packages.o1test.net" - } repo - -in -{ - Type = DebianRepo - , address = address -} +in { Type = DebianRepo, address = address } diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index fe9896ea348..98cd6882689 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -1,100 +1,137 @@ -let Prelude = ../External/Prelude.dhall let Profiles = ./Profiles.dhall + let BuildFlags = ./BuildFlags.dhall + let S = ../Lib/SelectFiles.dhall -let D = S.PathPattern let DebVersion = < Bookworm | Bullseye | Buster | Jammy | Focal > -let capitalName = \(debVersion : DebVersion) -> - merge { - Bookworm = "Bookworm" - , Bullseye = "Bullseye" - , Buster = "Buster" - , Jammy = "Jammy" - , Focal = "Focal" - } debVersion - -let lowerName = \(debVersion : DebVersion) -> - merge { - Bookworm = "bookworm" - , Bullseye = "bullseye" - , Buster = "buster" - , Jammy = "jammy" - , Focal = "focal" - } debVersion - +let capitalName = + \(debVersion : DebVersion) + -> merge + { Bookworm = "Bookworm" + , Bullseye = "Bullseye" + , Buster = "Buster" + , Jammy = "Jammy" + , Focal = "Focal" + } + debVersion +let lowerName = + \(debVersion : DebVersion) + -> merge + { Bookworm = "bookworm" + , Bullseye = "bullseye" + , Buster = "buster" + , Jammy = "jammy" + , Focal = "focal" + } + debVersion -let dependsOnStep = \(debVersion : DebVersion) -> \(profile : Profiles.Type) -> \(buildFlag: BuildFlags.Type) -> \(step : Text )-> - let profileSuffix = Profiles.toSuffixUppercase profile in - let prefix = "MinaArtifact" in - merge { - Bookworm = [{ name = "${prefix}${profileSuffix}${BuildFlags.toSuffixUppercase buildFlag}", key = "${step}-deb-pkg" }] - , Bullseye = [{ name = "${prefix}${capitalName debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase buildFlag}", key = "${step}-deb-pkg" }] - , Buster = [{ name = "${prefix}${capitalName debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase buildFlag}", key = "${step}-deb-pkg" }] - , Jammy = [{ name = "${prefix}${capitalName debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase buildFlag}", key = "${step}-deb-pkg" }] - , Focal = [{ name = "${prefix}${capitalName debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase buildFlag}", key = "${step}-deb-pkg" }] - } debVersion +let dependsOnStep = + \(debVersion : DebVersion) + -> \(profile : Profiles.Type) + -> \(buildFlag : BuildFlags.Type) + -> \(step : Text) + -> let profileSuffix = Profiles.toSuffixUppercase profile -let dependsOn = \(debVersion : DebVersion) -> \(profile : Profiles.Type) -> - dependsOnStep debVersion profile BuildFlags.Type.None "build" + let prefix = "MinaArtifact" --- Most debian builds are only used for public releases --- so they don't need to be triggered by dirtyWhen on every change --- these files representing changing the logic of the job, in which case test every platform -let minimalDirtyWhen = [ - S.exactly "buildkite/src/Constants/DebianVersions" "dhall", - S.exactly "buildkite/src/Constants/ContainerImages" "dhall", - S.exactly "buildkite/src/Command/HardforkPackageGeneration" "dhall", - S.exactly "buildkite/src/Command/MinaArtifact" "dhall", - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact"), - S.strictlyStart (S.contains "dockerfiles/stages"), - S.exactly "scripts/rebuild-deb" "sh", - S.exactly "scripts/release-docker" "sh", - S.exactly "buildkite/scripts/build-artifact" "sh", - S.exactly "buildkite/scripts/build-hardfork-package" "sh", - S.exactly "buildkite/scripts/check-compatibility" "sh", - -- Snark profiler dirtyWhen - S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall", - S.exactly "buildkite/scripts/run-snark-transaction-profiler" "sh", - S.exactly "scripts/snark_transaction_profiler" "py", - S.exactly "buildkite/scripts/version-linter" "sh", - S.exactly "scripts/version-linter" "py" -] + in merge + { Bookworm = + [ { name = + "${prefix}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + , key = "${step}-deb-pkg" + } + ] + , Bullseye = + [ { name = + "${prefix}${capitalName + debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + , key = "${step}-deb-pkg" + } + ] + , Buster = + [ { name = + "${prefix}${capitalName + debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + , key = "${step}-deb-pkg" + } + ] + , Jammy = + [ { name = + "${prefix}${capitalName + debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + , key = "${step}-deb-pkg" + } + ] + , Focal = + [ { name = + "${prefix}${capitalName + debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + , key = "${step}-deb-pkg" + } + ] + } + debVersion --- The default debian version (Bullseye) is used in all downstream CI jobs --- so the jobs must also trigger whenever src changes -let bullseyeDirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.strictlyStart (S.contains "automation"), - S.strictly (S.contains "Makefile"), - S.exactly "buildkite/scripts/connect-to-berkeley" "sh", - S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh", - S.exactly "buildkite/scripts/rosetta-integration-tests" "sh", - S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh", - S.exactly "buildkite/scripts/rosetta-integration-tests-fast" "sh", - S.strictlyStart (S.contains "buildkite/src/Jobs/Test") -] # minimalDirtyWhen +let dependsOn = + \(debVersion : DebVersion) + -> \(profile : Profiles.Type) + -> dependsOnStep debVersion profile BuildFlags.Type.None "build" -in +let minimalDirtyWhen = + [ S.exactly "buildkite/src/Constants/DebianVersions" "dhall" + , S.exactly "buildkite/src/Constants/ContainerImages" "dhall" + , S.exactly "buildkite/src/Command/HardforkPackageGeneration" "dhall" + , S.exactly "buildkite/src/Command/MinaArtifact" "dhall" + , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact") + , S.strictlyStart (S.contains "dockerfiles/stages") + , S.exactly "scripts/rebuild-deb" "sh" + , S.exactly "scripts/release-docker" "sh" + , S.exactly "buildkite/scripts/build-artifact" "sh" + , S.exactly "buildkite/scripts/build-hardfork-package" "sh" + , S.exactly "buildkite/scripts/check-compatibility" "sh" + , S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall" + , S.exactly "buildkite/scripts/run-snark-transaction-profiler" "sh" + , S.exactly "scripts/snark_transaction_profiler" "py" + , S.exactly "buildkite/scripts/version-linter" "sh" + , S.exactly "scripts/version-linter" "py" + ] -let dirtyWhen = \(debVersion : DebVersion) -> - merge { - Bookworm = minimalDirtyWhen - , Bullseye = bullseyeDirtyWhen - , Buster = minimalDirtyWhen - , Jammy = minimalDirtyWhen - , Focal = minimalDirtyWhen - } debVersion +let bullseyeDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.strictlyStart (S.contains "automation") + , S.strictly (S.contains "Makefile") + , S.exactly "buildkite/scripts/connect-to-berkeley" "sh" + , S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh" + , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + , S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh" + , S.exactly "buildkite/scripts/rosetta-integration-tests-fast" "sh" + , S.strictlyStart (S.contains "buildkite/src/Jobs/Test") + ] + # minimalDirtyWhen -in +let dirtyWhen = + \(debVersion : DebVersion) + -> merge + { Bookworm = minimalDirtyWhen + , Bullseye = bullseyeDirtyWhen + , Buster = minimalDirtyWhen + , Jammy = minimalDirtyWhen + , Focal = minimalDirtyWhen + } + debVersion -{ - DebVersion = DebVersion - , capitalName = capitalName - , lowerName = lowerName - , dependsOn = dependsOn - , dependsOnStep = dependsOnStep - , dirtyWhen = dirtyWhen -} +in { DebVersion = DebVersion + , capitalName = capitalName + , lowerName = lowerName + , dependsOn = dependsOn + , dependsOnStep = dependsOnStep + , dirtyWhen = dirtyWhen + } diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index 8febc3ae337..b57d9785ed0 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -1,43 +1,72 @@ -let Prelude = ../External/Prelude.dhall let Profiles = ./Profiles.dhall -let Docker: Type = < Bookworm | Bullseye | Buster | Jammy | Focal > - -let capitalName = \(docker : Docker) -> - merge { - Bookworm = "Bookworm" - , Bullseye = "Bullseye" - , Buster = "Buster" - , Jammy = "Jammy" - , Focal = "Focal" - } docker - -let lowerName = \(docker : Docker) -> - merge { - Bookworm = "bookworm" - , Bullseye = "bullseye" - , Buster = "buster" - , Jammy = "jammy" - , Focal = "focal" - } docker - -let dependsOn = \(docker : Docker) -> \(profile : Profiles.Type) -> \(binary: Text) -> - let profileSuffix = Profiles.toSuffixUppercase profile in - let prefix = "MinaArtifact" in - let suffix = "docker-image" in - merge { - Bookworm = [{ name = "${prefix}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }] - , Bullseye = [{ name = "${prefix}${capitalName docker}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }] - , Buster = [{ name = "${prefix}${capitalName docker}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }] - , Jammy = [{ name = "${prefix}${capitalName docker}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }] - , Focal = [{ name = "${prefix}${capitalName docker}${profileSuffix}", key = "${binary}-${lowerName docker}-${suffix}" }] - } docker - -in - -{ - Type = Docker - , capitalName = capitalName - , lowerName = lowerName - , dependsOn = dependsOn -} \ No newline at end of file +let Docker + : Type + = < Bookworm | Bullseye | Buster | Jammy | Focal > + +let capitalName = + \(docker : Docker) + -> merge + { Bookworm = "Bookworm" + , Bullseye = "Bullseye" + , Buster = "Buster" + , Jammy = "Jammy" + , Focal = "Focal" + } + docker + +let lowerName = + \(docker : Docker) + -> merge + { Bookworm = "bookworm" + , Bullseye = "bullseye" + , Buster = "buster" + , Jammy = "jammy" + , Focal = "focal" + } + docker + +let dependsOn = + \(docker : Docker) + -> \(profile : Profiles.Type) + -> \(binary : Text) + -> let profileSuffix = Profiles.toSuffixUppercase profile + + let prefix = "MinaArtifact" + + let suffix = "docker-image" + + in merge + { Bookworm = + [ { name = "${prefix}${profileSuffix}" + , key = "${binary}-${lowerName docker}-${suffix}" + } + ] + , Bullseye = + [ { name = "${prefix}${capitalName docker}${profileSuffix}" + , key = "${binary}-${lowerName docker}-${suffix}" + } + ] + , Buster = + [ { name = "${prefix}${capitalName docker}${profileSuffix}" + , key = "${binary}-${lowerName docker}-${suffix}" + } + ] + , Jammy = + [ { name = "${prefix}${capitalName docker}${profileSuffix}" + , key = "${binary}-${lowerName docker}-${suffix}" + } + ] + , Focal = + [ { name = "${prefix}${capitalName docker}${profileSuffix}" + , key = "${binary}-${lowerName docker}-${suffix}" + } + ] + } + docker + +in { Type = Docker + , capitalName = capitalName + , lowerName = lowerName + , dependsOn = dependsOn + } diff --git a/buildkite/src/Constants/Network.dhall b/buildkite/src/Constants/Network.dhall index 7bc3c0ad8d1..c32892204c6 100644 --- a/buildkite/src/Constants/Network.dhall +++ b/buildkite/src/Constants/Network.dhall @@ -1,26 +1,17 @@ -let Prelude = ../External/Prelude.dhall -let Profiles = ./Profiles.dhall +let Network + : Type + = < Devnet | Mainnet | Berkeley > -let Network: Type = < Devnet | Mainnet | Berkeley > +let capitalName = + \(network : Network) + -> merge + { Devnet = "Devnet", Mainnet = "Mainnet", Berkeley = "Berkeley" } + network -let capitalName = \(network : Network) -> - merge { - Devnet = "Devnet" - , Mainnet = "Mainnet" - , Berkeley = "Berkeley" - } network +let lowerName = + \(network : Network) + -> merge + { Devnet = "devnet", Mainnet = "mainnet", Berkeley = "berkeley" } + network -let lowerName = \(network : Network) -> - merge { - Devnet = "devnet" - , Mainnet = "mainnet" - , Berkeley = "berkeley" - } network - -in - -{ - Type = Network - , capitalName = capitalName - , lowerName = lowerName -} \ No newline at end of file +in { Type = Network, capitalName = capitalName, lowerName = lowerName } diff --git a/buildkite/src/Constants/Profiles.dhall b/buildkite/src/Constants/Profiles.dhall index a55feba44d6..a113413bd30 100644 --- a/buildkite/src/Constants/Profiles.dhall +++ b/buildkite/src/Constants/Profiles.dhall @@ -1,65 +1,72 @@ -let Prelude = ../External/Prelude.dhall +let Profile + : Type + = < Standard | Mainnet | Lightnet | Hardfork > -let Profile : Type = < Standard | Mainnet | Lightnet | Hardfork > +let capitalName = + \(profile : Profile) + -> merge + { Standard = "Standard" + , Mainnet = "Mainnet" + , Lightnet = "Lightnet" + , Hardfork = "Hardfork" + } + profile -let capitalName = \(profile : Profile) -> - merge { - Standard = "Standard" - , Mainnet = "Mainnet" - , Lightnet = "Lightnet" - , Hardfork = "Hardfork" - } profile +let lowerName = + \(profile : Profile) + -> merge + { Standard = "standard" + , Mainnet = "mainnet" + , Lightnet = "lightnet" + , Hardfork = "hardfork" + } + profile -let lowerName = \(profile : Profile) -> - merge { - Standard = "standard" - , Mainnet = "mainnet" - , Lightnet = "lightnet" - , Hardfork = "hardfork" - } profile +let duneProfile = + \(profile : Profile) + -> merge + { Standard = "devnet" + , Mainnet = "mainnet" + , Lightnet = "lightnet" + , Hardfork = "hardfork" + } + profile -let duneProfile = \(profile : Profile) -> - merge { - Standard = "devnet" - , Mainnet = "mainnet" - , Lightnet = "lightnet" - , Hardfork = "hardfork" - } profile +let toSuffixUppercase = + \(profile : Profile) + -> merge + { Standard = "" + , Mainnet = "Mainnet" + , Lightnet = "Lightnet" + , Hardfork = "Hardfork" + } + profile -let toSuffixUppercase = \(profile : Profile) -> - merge { - Standard = "" - , Mainnet = "Mainnet" - , Lightnet = "Lightnet" - , Hardfork = "Hardfork" - } profile +let toSuffixLowercase = + \(profile : Profile) + -> merge + { Standard = "" + , Mainnet = "mainnet" + , Lightnet = "lightnet" + , Hardfork = "hardfork" + } + profile -let toSuffixLowercase = \(profile : Profile) -> - merge { - Standard = "" - , Mainnet = "mainnet" - , Lightnet = "lightnet" - , Hardfork = "hardfork" - } profile +let toLabelSegment = + \(profile : Profile) + -> merge + { Standard = "" + , Mainnet = "-mainnet" + , Lightnet = "-lightnet" + , Hardfork = "-hardfork" + } + profile -let toLabelSegment = \(profile : Profile) -> - merge { - Standard = "" - , Mainnet = "-mainnet" - , Lightnet = "-lightnet" - , Hardfork = "-hardfork" - } profile - - - -in - -{ - Type = Profile - , capitalName = capitalName - , lowerName = lowerName - , duneProfile = duneProfile - , toSuffixUppercase = toSuffixUppercase - , toSuffixLowercase = toSuffixLowercase - , toLabelSegment = toLabelSegment -} +in { Type = Profile + , capitalName = capitalName + , lowerName = lowerName + , duneProfile = duneProfile + , toSuffixUppercase = toSuffixUppercase + , toSuffixLowercase = toSuffixLowercase + , toLabelSegment = toLabelSegment + } diff --git a/buildkite/src/Constants/Toolchain.dhall b/buildkite/src/Constants/Toolchain.dhall index 7741f12a3ca..b70181349bf 100644 --- a/buildkite/src/Constants/Toolchain.dhall +++ b/buildkite/src/Constants/Toolchain.dhall @@ -1,44 +1,47 @@ let DebianVersions = ./DebianVersions.dhall -let RunInToolchain = ../Command/RunInToolchain.dhall -let ContainerImages = ./ContainerImages.dhall - -let SelectionMode: Type = < ByDebian | Custom : Text > - ---- Bullseye and Focal are so similar that they share a toolchain runner ---- Same with Bookworm and Jammy -let runner = \(debVersion : DebianVersions.DebVersion) -> - merge { - Bookworm = RunInToolchain.runInToolchainBookworm - , Bullseye = RunInToolchain.runInToolchainBullseye - , Buster = RunInToolchain.runInToolchainBuster - , Jammy = RunInToolchain.runInToolchainBookworm - , Focal = RunInToolchain.runInToolchainBullseye - } debVersion - -let select = \(mode: SelectionMode) -> \(debVersion : DebianVersions.DebVersion) -> - merge { - ByDebian = runner debVersion - , Custom = \(image: Text) -> RunInToolchain.runInToolchainImage image - } mode ---- Bullseye and Focal are so similar that they share a toolchain image ---- Same with Bookworm and Jammy -let image = \(debVersion : DebianVersions.DebVersion) -> - merge { - Bookworm = ContainerImages.minaToolchainBookworm - , Bullseye = ContainerImages.minaToolchainBullseye - , Buster = ContainerImages.minaToolchainBuster - , Jammy = ContainerImages.minaToolchainBookworm - , Focal = ContainerImages.minaToolchainBullseye - } debVersion - -in - -{ - SelectionMode = SelectionMode - , select = select - , runner = runner - , image = image -} +let RunInToolchain = ../Command/RunInToolchain.dhall +let ContainerImages = ./ContainerImages.dhall +let SelectionMode + : Type + = < ByDebian | Custom : Text > + +let runner = + \(debVersion : DebianVersions.DebVersion) + -> merge + { Bookworm = RunInToolchain.runInToolchainBookworm + , Bullseye = RunInToolchain.runInToolchainBullseye + , Buster = RunInToolchain.runInToolchainBuster + , Jammy = RunInToolchain.runInToolchainBookworm + , Focal = RunInToolchain.runInToolchainBullseye + } + debVersion + +let select = + \(mode : SelectionMode) + -> \(debVersion : DebianVersions.DebVersion) + -> merge + { ByDebian = runner debVersion + , Custom = + \(image : Text) -> RunInToolchain.runInToolchainImage image + } + mode + +let image = + \(debVersion : DebianVersions.DebVersion) + -> merge + { Bookworm = ContainerImages.minaToolchainBookworm + , Bullseye = ContainerImages.minaToolchainBullseye + , Buster = ContainerImages.minaToolchainBuster + , Jammy = ContainerImages.minaToolchainBookworm + , Focal = ContainerImages.minaToolchainBullseye + } + debVersion + +in { SelectionMode = SelectionMode + , select = select + , runner = runner + , image = image + } diff --git a/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall b/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall index 635808b00d9..31756138e28 100644 --- a/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall +++ b/buildkite/src/Entrypoints/GenerateHardforkPackage.dhall @@ -2,55 +2,53 @@ -- Keep these rules lean! They have to run unconditionally. let SelectFiles = ../Lib/SelectFiles.dhall + let Cmd = ../Lib/Cmds.dhall let Command = ../Command/Base.dhall + let Docker = ../Command/Docker/Type.dhall + let JobSpec = ../Pipeline/JobSpec.dhall + let Pipeline = ../Pipeline/Dsl.dhall -let PipelineMode = ../Pipeline/Mode.dhall -let PipelineFilter = ../Pipeline/Filter.dhall -let PipelineTag = ../Pipeline/Tag.dhall -let Size = ../Command/Size.dhall -let triggerCommand = ../Pipeline/TriggerCommand.dhall +let Size = ../Command/Size.dhall -let B = ../External/Buildkite.dhall -let Prelude = ../External/Prelude.dhall -let List/map = Prelude.List.map let HardforkPackageGeneration = ../Command/HardforkPackageGeneration.dhall -let Package = ../Constants/DebianPackage.dhall -let Profile = ../Constants/Profiles.dhall -let Network = ../Constants/Network.dhall -let DebianVersions = ../Constants/DebianVersions.dhall - -let generate_hardfork_package = \(spec : HardforkPackageGeneration.Spec.Type) -> - - let config : Pipeline.Config.Type = Pipeline.Config::{ - spec = JobSpec::{ - name = "generate", - dirtyWhen = [ SelectFiles.everything ] - }, - steps = [ - Command.build Command.Config::{ - commands = [ - Cmd.run "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall", - Cmd.quietly "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{=}).pipeline' | buildkite-agent pipeline upload", - Cmd.quietly "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{codename = DebianVersions.DebVersion.Buster}).pipeline' | buildkite-agent pipeline upload" - ], - label = "Generate hardfork package", - key = "generate-hardfork-package", - target = Size.Small, - docker = Some Docker::{ - image = (../Constants/ContainerImages.dhall).toolchainBase, - environment = ["BUILDKITE_AGENT_ACCESS_TOKEN"] - } - } - ] - } - in (Pipeline.build config).pipeline - - -in { - generate_hardfork_package = generate_hardfork_package -} \ No newline at end of file + +let generate_hardfork_package = + \(spec : HardforkPackageGeneration.Spec.Type) + -> let config + : Pipeline.Config.Type + = Pipeline.Config::{ + , spec = JobSpec::{ + , name = "generate" + , dirtyWhen = [ SelectFiles.everything ] + } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall" + , Cmd.quietly + "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{=}).pipeline' | buildkite-agent pipeline upload" + , Cmd.quietly + "dhall-to-yaml --quoted <<< '(Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{codename = DebianVersions.DebVersion.Buster}).pipeline' | buildkite-agent pipeline upload" + ] + , label = "Generate hardfork package" + , key = "generate-hardfork-package" + , target = Size.Small + , docker = Some Docker::{ + , image = + (../Constants/ContainerImages.dhall).toolchainBase + , environment = [ "BUILDKITE_AGENT_ACCESS_TOKEN" ] + } + } + ] + } + + in (Pipeline.build config).pipeline + +in { generate_hardfork_package = generate_hardfork_package } diff --git a/buildkite/src/Entrypoints/PromotePackage.dhall b/buildkite/src/Entrypoints/PromotePackage.dhall index bc353f0c5ee..40fabae3eca 100644 --- a/buildkite/src/Entrypoints/PromotePackage.dhall +++ b/buildkite/src/Entrypoints/PromotePackage.dhall @@ -1,212 +1,235 @@ -let B = ../External/Buildkite.dhall let Prelude = ../External/Prelude.dhall + let List/map = Prelude.List.map + let PromotePackage = ../Command/PromotePackage.dhall -let Package = ../Constants/DebianPackage.dhall -let Profile = ../Constants/Profiles.dhall -let Artifact = ../Constants/Artifacts.dhall + +let Package = ../Constants/DebianPackage.dhall + +let Profile = ../Constants/Profiles.dhall + +let Artifact = ../Constants/Artifacts.dhall + let DebianChannel = ../Constants/DebianChannel.dhall + let Network = ../Constants/Network.dhall + let DebianVersions = ../Constants/DebianVersions.dhall + let Pipeline = ../Pipeline/Dsl.dhall + let PipelineMode = ../Pipeline/Mode.dhall -let promote_artifacts = - \(debians: List Package.Type) -> - \(dockers: List Artifact.Type) -> - \(version: Text ) -> - \(new_version: Text ) -> - \(architecture: Text ) -> - \(profile: Profile.Type) -> - \(network: Network.Type) -> - \(codenames: List DebianVersions.DebVersion ) -> - \(from_channel: DebianChannel.Type ) -> - \(to_channel: DebianChannel.Type ) -> - \(tag: Text ) -> - \(remove_profile_from_name: Bool) -> - \(publish: Bool) -> - - let debians_spec = - List/map - Package.Type - (List PromotePackage.PromoteDebianSpec.Type) - (\(debian: Package.Type) -> - List/map - DebianVersions.DebVersion - PromotePackage.PromoteDebianSpec.Type - (\(codename: DebianVersions.DebVersion) -> - PromotePackage.PromoteDebianSpec::{ - profile = profile - , package = debian - , version = version - , new_version = new_version - , architecture = architecture - , network = network - , codename = codename - , from_channel = from_channel - , to_channel = to_channel - , remove_profile_from_name = remove_profile_from_name - , step_key = "promote-debian-${Package.lowerName debian}-${DebianVersions.lowerName codename}-from-${DebianChannel.lowerName from_channel}-to-${DebianChannel.lowerName to_channel}" - } - ) - codenames - - ) - debians - in - - let debians_spec = - Prelude.List.fold - (List PromotePackage.PromoteDebianSpec.Type) - debians_spec - (List PromotePackage.PromoteDebianSpec.Type) - (\(a : List PromotePackage.PromoteDebianSpec.Type) -> \(b : List PromotePackage.PromoteDebianSpec.Type) -> a # b) - ([] : List PromotePackage.PromoteDebianSpec.Type) - - in - - let dockers_spec = - List/map - Artifact.Type - (List PromotePackage.PromoteDockerSpec.Type) - (\(docker: Artifact.Type) -> - List/map - DebianVersions.DebVersion - PromotePackage.PromoteDockerSpec.Type - (\(codename: DebianVersions.DebVersion) -> - PromotePackage.PromoteDockerSpec::{ - profile = profile - , name = docker - , version = version - , codename = codename - , new_tag = new_version - , network = network - , publish = publish - , remove_profile_from_name = remove_profile_from_name - , step_key = "add-tag-to-${Artifact.lowerName docker}-${DebianVersions.lowerName codename}-docker" - } - ) - codenames - ) - dockers - - in - - let dockers_spec = - Prelude.List.fold - (List PromotePackage.PromoteDockerSpec.Type) - dockers_spec - (List PromotePackage.PromoteDockerSpec.Type) - (\(a : List PromotePackage.PromoteDockerSpec.Type) -> \(b : List PromotePackage.PromoteDockerSpec.Type) -> a # b) - ([] : List PromotePackage.PromoteDockerSpec.Type) - - in - - let pipelineType = Pipeline.build - ( - PromotePackage.promotePipeline - (debians_spec) - (dockers_spec) - (DebianVersions.DebVersion.Bullseye) - (PipelineMode.Type.Stable) - ) - in pipelineType.pipeline - -let verify_artifacts = - \(debians: List Package.Type) -> - \(dockers: List Artifact.Type) -> - \(new_version: Text ) -> - \(profile: Profile.Type) -> - \(network: Network.Type) -> - \(codenames: List DebianVersions.DebVersion ) -> - \(to_channel: DebianChannel.Type ) -> - \(tag: Text ) -> - \(remove_profile_from_name: Bool) -> - \(publish: Bool) -> - - let debians_spec = - List/map - Package.Type - (List PromotePackage.PromoteDebianSpec.Type) - (\(debian: Package.Type) -> - List/map - DebianVersions.DebVersion - PromotePackage.PromoteDebianSpec.Type - (\(codename: DebianVersions.DebVersion) -> - PromotePackage.PromoteDebianSpec::{ - profile = profile - , package = debian - , new_version = new_version - , network = network - , codename = codename - , to_channel = to_channel - , remove_profile_from_name = remove_profile_from_name - , step_key = "verify-promote-debian-${Package.lowerName debian}-${DebianVersions.lowerName codename}-${DebianChannel.lowerName to_channel}" - } - ) - codenames - - ) - debians - in - - let debians_spec = - Prelude.List.fold - (List PromotePackage.PromoteDebianSpec.Type) - debians_spec - (List PromotePackage.PromoteDebianSpec.Type) - (\(a : List PromotePackage.PromoteDebianSpec.Type) -> \(b : List PromotePackage.PromoteDebianSpec.Type) -> a # b) - ([] : List PromotePackage.PromoteDebianSpec.Type) - - in - - let dockers_spec = - List/map - Artifact.Type - (List PromotePackage.PromoteDockerSpec.Type) - (\(docker: Artifact.Type) -> - List/map - DebianVersions.DebVersion - PromotePackage.PromoteDockerSpec.Type - (\(codename: DebianVersions.DebVersion) -> - PromotePackage.PromoteDockerSpec::{ - profile = profile - , name = docker - , codename = codename - , new_tag = new_version - , network = network - , publish = publish - , remove_profile_from_name = remove_profile_from_name - , step_key = "verify-tag-${Artifact.lowerName docker}-${DebianVersions.lowerName codename}-docker" - } - ) - codenames - ) - dockers - - in - - let dockers_spec = - Prelude.List.fold - (List PromotePackage.PromoteDockerSpec.Type) - dockers_spec - (List PromotePackage.PromoteDockerSpec.Type) - (\(a : List PromotePackage.PromoteDockerSpec.Type) -> \(b : List PromotePackage.PromoteDockerSpec.Type) -> a # b) - ([] : List PromotePackage.PromoteDockerSpec.Type) - - in - - let pipelineType = Pipeline.build - ( - PromotePackage.verifyPipeline - (debians_spec) - (dockers_spec) - (DebianVersions.DebVersion.Bullseye) - (PipelineMode.Type.Stable) - ) - in pipelineType.pipeline - -in { - promote_artifacts = promote_artifacts, - verify_artifacts = verify_artifacts -} \ No newline at end of file +let promote_artifacts = + \(debians : List Package.Type) + -> \(dockers : List Artifact.Type) + -> \(version : Text) + -> \(new_version : Text) + -> \(architecture : Text) + -> \(profile : Profile.Type) + -> \(network : Network.Type) + -> \(codenames : List DebianVersions.DebVersion) + -> \(from_channel : DebianChannel.Type) + -> \(to_channel : DebianChannel.Type) + -> \(tag : Text) + -> \(remove_profile_from_name : Bool) + -> \(publish : Bool) + -> let debians_spec = + List/map + Package.Type + (List PromotePackage.PromoteDebianSpec.Type) + ( \(debian : Package.Type) + -> List/map + DebianVersions.DebVersion + PromotePackage.PromoteDebianSpec.Type + ( \(codename : DebianVersions.DebVersion) + -> PromotePackage.PromoteDebianSpec::{ + , profile = profile + , package = debian + , version = version + , new_version = new_version + , architecture = architecture + , network = network + , codename = codename + , from_channel = from_channel + , to_channel = to_channel + , remove_profile_from_name = + remove_profile_from_name + , step_key = + "promote-debian-${Package.lowerName + debian}-${DebianVersions.lowerName + codename}-from-${DebianChannel.lowerName + from_channel}-to-${DebianChannel.lowerName + to_channel}" + } + ) + codenames + ) + debians + + let debians_spec = + Prelude.List.fold + (List PromotePackage.PromoteDebianSpec.Type) + debians_spec + (List PromotePackage.PromoteDebianSpec.Type) + ( \(a : List PromotePackage.PromoteDebianSpec.Type) + -> \(b : List PromotePackage.PromoteDebianSpec.Type) + -> a # b + ) + ([] : List PromotePackage.PromoteDebianSpec.Type) + + let dockers_spec = + List/map + Artifact.Type + (List PromotePackage.PromoteDockerSpec.Type) + ( \(docker : Artifact.Type) + -> List/map + DebianVersions.DebVersion + PromotePackage.PromoteDockerSpec.Type + ( \(codename : DebianVersions.DebVersion) + -> PromotePackage.PromoteDockerSpec::{ + , profile = profile + , name = docker + , version = version + , codename = codename + , new_tag = new_version + , network = network + , publish = publish + , remove_profile_from_name = + remove_profile_from_name + , step_key = + "add-tag-to-${Artifact.lowerName + docker}-${DebianVersions.lowerName + codename}-docker" + } + ) + codenames + ) + dockers + + let dockers_spec = + Prelude.List.fold + (List PromotePackage.PromoteDockerSpec.Type) + dockers_spec + (List PromotePackage.PromoteDockerSpec.Type) + ( \(a : List PromotePackage.PromoteDockerSpec.Type) + -> \(b : List PromotePackage.PromoteDockerSpec.Type) + -> a # b + ) + ([] : List PromotePackage.PromoteDockerSpec.Type) + + let pipelineType = + Pipeline.build + ( PromotePackage.promotePipeline + debians_spec + dockers_spec + DebianVersions.DebVersion.Bullseye + PipelineMode.Type.Stable + ) + + in pipelineType.pipeline + +let verify_artifacts = + \(debians : List Package.Type) + -> \(dockers : List Artifact.Type) + -> \(new_version : Text) + -> \(profile : Profile.Type) + -> \(network : Network.Type) + -> \(codenames : List DebianVersions.DebVersion) + -> \(to_channel : DebianChannel.Type) + -> \(tag : Text) + -> \(remove_profile_from_name : Bool) + -> \(publish : Bool) + -> let debians_spec = + List/map + Package.Type + (List PromotePackage.PromoteDebianSpec.Type) + ( \(debian : Package.Type) + -> List/map + DebianVersions.DebVersion + PromotePackage.PromoteDebianSpec.Type + ( \(codename : DebianVersions.DebVersion) + -> PromotePackage.PromoteDebianSpec::{ + , profile = profile + , package = debian + , new_version = new_version + , network = network + , codename = codename + , to_channel = to_channel + , remove_profile_from_name = + remove_profile_from_name + , step_key = + "verify-promote-debian-${Package.lowerName + debian}-${DebianVersions.lowerName + codename}-${DebianChannel.lowerName + to_channel}" + } + ) + codenames + ) + debians + + let debians_spec = + Prelude.List.fold + (List PromotePackage.PromoteDebianSpec.Type) + debians_spec + (List PromotePackage.PromoteDebianSpec.Type) + ( \(a : List PromotePackage.PromoteDebianSpec.Type) + -> \(b : List PromotePackage.PromoteDebianSpec.Type) + -> a # b + ) + ([] : List PromotePackage.PromoteDebianSpec.Type) + + let dockers_spec = + List/map + Artifact.Type + (List PromotePackage.PromoteDockerSpec.Type) + ( \(docker : Artifact.Type) + -> List/map + DebianVersions.DebVersion + PromotePackage.PromoteDockerSpec.Type + ( \(codename : DebianVersions.DebVersion) + -> PromotePackage.PromoteDockerSpec::{ + , profile = profile + , name = docker + , codename = codename + , new_tag = new_version + , network = network + , publish = publish + , remove_profile_from_name = + remove_profile_from_name + , step_key = + "verify-tag-${Artifact.lowerName + docker}-${DebianVersions.lowerName + codename}-docker" + } + ) + codenames + ) + dockers + + let dockers_spec = + Prelude.List.fold + (List PromotePackage.PromoteDockerSpec.Type) + dockers_spec + (List PromotePackage.PromoteDockerSpec.Type) + ( \(a : List PromotePackage.PromoteDockerSpec.Type) + -> \(b : List PromotePackage.PromoteDockerSpec.Type) + -> a # b + ) + ([] : List PromotePackage.PromoteDockerSpec.Type) + + let pipelineType = + Pipeline.build + ( PromotePackage.verifyPipeline + debians_spec + dockers_spec + DebianVersions.DebVersion.Bullseye + PipelineMode.Type.Stable + ) + + in pipelineType.pipeline + +in { promote_artifacts = promote_artifacts + , verify_artifacts = verify_artifacts + } diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall index cbeccbfe42c..6a326a9588f 100644 --- a/buildkite/src/Jobs/Lint/Fast.dhall +++ b/buildkite/src/Jobs/Lint/Fast.dhall @@ -1,10 +1,7 @@ -let Prelude = ../../External/Prelude.dhall - -let B = ../../External/Buildkite.dhall - let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall @@ -15,8 +12,6 @@ let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall - let Size = ../../Command/Size.dhall let commands = @@ -28,13 +23,13 @@ let commands = in Pipeline.build Pipeline.Config::{ - spec = JobSpec::{ - , dirtyWhen = [ - S.strictly (S.contains "Makefile"), - S.strictlyStart (S.contains "src/"), - S.strictlyStart (S.contains "rfcs/"), - S.exactly_noext "CODEOWNERS", - S.exactly "scripts/check-snarky-submodule" "sh" + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictly (S.contains "Makefile") + , S.strictlyStart (S.contains "src/") + , S.strictlyStart (S.contains "rfcs/") + , S.exactly_noext "CODEOWNERS" + , S.exactly "scripts/check-snarky-submodule" "sh" ] , path = "Lint" , name = "Fast" diff --git a/buildkite/src/Jobs/Lint/HelmChart.dhall b/buildkite/src/Jobs/Lint/HelmChart.dhall index f0f93b132f1..8c7ea461349 100644 --- a/buildkite/src/Jobs/Lint/HelmChart.dhall +++ b/buildkite/src/Jobs/Lint/HelmChart.dhall @@ -1,40 +1,42 @@ -let Prelude = ../../External/Prelude.dhall - let S = ../../Lib/SelectFiles.dhall + let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -in - -Pipeline.build - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ - S.contains "helm/", - S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/HelmChart"), - -- trigger on HelmRelease job change due to dependency - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/HelmRelease"), - S.exactly "buildkite/scripts/helm-ci" "sh" - ], - path = "Lint", - name = "HelmChart", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ Cmd.run "HELM_LINT=true buildkite/scripts/helm-ci.sh" ] - , label = "Helm chart lint steps" - , key = "lint-helm-chart" - , target = Size.Small - , docker = None Docker.Type +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.contains "helm/" + , S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/HelmChart") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Release/HelmRelease") + , S.exactly "buildkite/scripts/helm-ci" "sh" + ] + , path = "Lint" + , name = "HelmChart" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run "HELM_LINT=true buildkite/scripts/helm-ci.sh" ] + , label = "Helm chart lint steps" + , key = "lint-helm-chart" + , target = Size.Small + , docker = None Docker.Type + } + ] + } diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index 83ec98870c6..57c2ecc0099 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -1,113 +1,125 @@ -let Prelude = ../../External/Prelude.dhall let B = ../../External/Buildkite.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall + let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type -in - -Pipeline.build - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ SelectFiles.everything ], - path = "Lint", - name = "Merge", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ Cmd.run "buildkite/scripts/merges-cleanly.sh compatible"] - , label = "Check merges cleanly into compatible" - , key = "clean-merge-compatible" - , target = Size.Small - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).toolchainBase +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = [ SelectFiles.everything ] + , path = "Lint" + , name = "Merge" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] + } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run "buildkite/scripts/merges-cleanly.sh compatible" ] + , label = "Check merges cleanly into compatible" + , key = "clean-merge-compatible" + , target = Size.Small + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } - }, - Command.build - Command.Config::{ - commands = [ Cmd.run "buildkite/scripts/merges-cleanly.sh develop"] - , label = "Check merges cleanly into develop" - , key = "clean-merge-develop" - , target = Size.Small - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).toolchainBase + , Command.build + Command.Config::{ + , commands = + [ Cmd.run "buildkite/scripts/merges-cleanly.sh develop" ] + , label = "Check merges cleanly into develop" + , key = "clean-merge-develop" + , target = Size.Small + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } - }, - Command.build - Command.Config::{ - commands = [ Cmd.run "buildkite/scripts/merges-cleanly.sh master"] - , label = "Check merges cleanly into master" - , key = "clean-merge-master" - , target = Size.Small - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).toolchainBase + , Command.build + Command.Config::{ + , commands = + [ Cmd.run "buildkite/scripts/merges-cleanly.sh master" ] + , label = "Check merges cleanly into master" + , key = "clean-merge-master" + , target = Size.Small + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } - }, - Command.build - Command.Config::{ - commands = [ Cmd.run "scripts/merged-to-proof-systems.sh compatible"] - , label = "[proof-systems] Check merges cleanly into proof-systems compatible branch" - , key = "merged-to-proof-systems-compatible" - , soft_fail = Some (B/SoftFail.Boolean True) - , target = Size.Small - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).toolchainBase + , Command.build + Command.Config::{ + , commands = + [ Cmd.run "scripts/merged-to-proof-systems.sh compatible" ] + , label = + "[proof-systems] Check merges cleanly into proof-systems compatible branch" + , key = "merged-to-proof-systems-compatible" + , soft_fail = Some (B/SoftFail.Boolean True) + , target = Size.Small + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } - }, - Command.build - Command.Config::{ - commands = [ Cmd.run "scripts/merged-to-proof-systems.sh berkeley"] - , label = "[proof-systems] Check merges cleanly into proof-systems berkeley branch" - , key = "merged-to-proof-systems-berkeley" - , soft_fail = Some (B/SoftFail.Boolean True) - , target = Size.Small - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).toolchainBase + , Command.build + Command.Config::{ + , commands = + [ Cmd.run "scripts/merged-to-proof-systems.sh berkeley" ] + , label = + "[proof-systems] Check merges cleanly into proof-systems berkeley branch" + , key = "merged-to-proof-systems-berkeley" + , soft_fail = Some (B/SoftFail.Boolean True) + , target = Size.Small + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } - }, - Command.build - Command.Config::{ - commands = [ Cmd.run "scripts/merged-to-proof-systems.sh develop"] - , label = "[proof-systems] Check merges cleanly into proof-systems develop branch" - , key = "merged-to-proof-systems-develop" - , soft_fail = Some (B/SoftFail.Boolean True) - , target = Size.Small - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).toolchainBase + , Command.build + Command.Config::{ + , commands = + [ Cmd.run "scripts/merged-to-proof-systems.sh develop" ] + , label = + "[proof-systems] Check merges cleanly into proof-systems develop branch" + , key = "merged-to-proof-systems-develop" + , soft_fail = Some (B/SoftFail.Boolean True) + , target = Size.Small + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } - }, - Command.build - Command.Config::{ - commands = [ Cmd.run "scripts/merged-to-proof-systems.sh master"] - , label = "[proof-systems] Check merges cleanly into proof-systems master branch" - , key = "merged-to-proof-systems-master" - , soft_fail = Some (B/SoftFail.Boolean True) - , target = Size.Small - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).toolchainBase + , Command.build + Command.Config::{ + , commands = [ Cmd.run "scripts/merged-to-proof-systems.sh master" ] + , label = + "[proof-systems] Check merges cleanly into proof-systems master branch" + , key = "merged-to-proof-systems-master" + , soft_fail = Some (B/SoftFail.Boolean True) + , target = Size.Small + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } - } - , Command.build - Command.Config::{ - commands = [ Cmd.run "true" ] : List Cmd.Type - , label = "pr" - , key = "pr" - , target = Size.Small - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).toolchainBase + , Command.build + Command.Config::{ + , commands = [ Cmd.run "true" ] : List Cmd.Type + , label = "pr" + , key = "pr" + , target = Size.Small + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } - } - ] - } + ] + } diff --git a/buildkite/src/Jobs/Lint/OCaml.dhall b/buildkite/src/Jobs/Lint/OCaml.dhall index 3dec98290af..c0496b692e9 100644 --- a/buildkite/src/Jobs/Lint/OCaml.dhall +++ b/buildkite/src/Jobs/Lint/OCaml.dhall @@ -1,12 +1,9 @@ -let Prelude = ../../External/Prelude.dhall - let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall -let Cmd = ../../Lib/Cmds.dhall - let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall @@ -25,7 +22,8 @@ in Pipeline.build let dirtyDhallDirCompiles = assert - : S.compile [ dirtyDhallDir ] ≡ "^buildkite/src/Jobs/Lint/OCaml" + : S.compile [ dirtyDhallDir ] + === "^buildkite/src/Jobs/Lint/OCaml" in JobSpec::{ , dirtyWhen = diff --git a/buildkite/src/Jobs/Lint/Rust.dhall b/buildkite/src/Jobs/Lint/Rust.dhall index c099d225646..c79b2ddcafb 100644 --- a/buildkite/src/Jobs/Lint/Rust.dhall +++ b/buildkite/src/Jobs/Lint/Rust.dhall @@ -1,37 +1,41 @@ -let Prelude = ../../External/Prelude.dhall - let S = ../../Lib/SelectFiles.dhall -let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall -in - -Pipeline.build - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/Rust") ], - path = "Lint", - name = "Rust", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] - }, - steps = [ - Command.build - Command.Config::{ - commands = RunInToolchain.runInToolchain ([] : List Text) "cd src/app/trace-tool ; PATH=/home/opam/.cargo/bin:$PATH cargo check" - , label = "Rust lint steps; trace-tool" - , key = "lint-trace-tool" - , target = Size.Small - , docker = None Docker.Type +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.contains "src/app/trace-tool" + , S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/Rust") + ] + , path = "Lint" + , name = "Rust" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + ([] : List Text) + "cd src/app/trace-tool ; PATH=/home/opam/.cargo/bin:\$PATH cargo check" + , label = "Rust lint steps; trace-tool" + , key = "lint-trace-tool" + , target = Size.Small + , docker = None Docker.Type + } + ] + } diff --git a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall index c60772c198b..9d87e7ee373 100644 --- a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall @@ -1,50 +1,49 @@ -let Prelude = ../../External/Prelude.dhall -let B = ../../External/Buildkite.dhall - -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - let S = ../../Lib/SelectFiles.dhall + let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -in - -Pipeline.build - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ - S.exactly "automation/terraform/monitoring/o1-testnet-alerts" "tf", - S.strictlyStart (S.contains "automation/terraform/modules/testnet-alerts"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/TestnetAlerts"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") - ], - path = "Lint", - name = "TestnetAlerts", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ - --- destroy state prior to start to ensure reset - Cmd.run "cd automation/terraform/monitoring && terraform init && terraform destroy -auto-approve", - Cmd.run ( - "terraform apply -auto-approve -target module.o1testnet_alerts.null_resource.alert_rules_lint" ++ - " -target module.o1testnet_alerts.null_resource.alert_rules_check" - ) +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.exactly "automation/terraform/monitoring/o1-testnet-alerts" "tf" + , S.strictlyStart + (S.contains "automation/terraform/modules/testnet-alerts") + , S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/TestnetAlerts") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ] - , label = "Lint Testnet alert rules" - , key = "lint-testnet-alerts" - , target = Size.Small - , docker = None Docker.Type + , path = "Lint" + , name = "TestnetAlerts" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run + "cd automation/terraform/monitoring && terraform init && terraform destroy -auto-approve" + , Cmd.run + ( "terraform apply -auto-approve -target module.o1testnet_alerts.null_resource.alert_rules_lint" + ++ " -target module.o1testnet_alerts.null_resource.alert_rules_check" + ) + ] + , label = "Lint Testnet alert rules" + , key = "lint-testnet-alerts" + , target = Size.Small + , docker = None Docker.Type + } + ] + } diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall index a2469a29ef0..c9b632ee054 100644 --- a/buildkite/src/Jobs/Lint/ValidationService.dhall +++ b/buildkite/src/Jobs/Lint/ValidationService.dhall @@ -1,70 +1,101 @@ let Prelude = ../../External/Prelude.dhall + let Text/concatSep = Prelude.Text.concatSep + let List/map = Prelude.List.map let S = ../../Lib/SelectFiles.dhall + let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let Cmd = ../../Lib/Cmds.dhall + let Command = ../../Command/Base.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall + let Size = ../../Command/Size.dhall + let Docker = ../../Command/Docker/Type.dhall + let ValidationService = ../../Projects/ValidationService.dhall let B = ../../External/Buildkite.dhall + let B/Skip = B.definitions/commandStep/properties/skip/Type let commands = - let sigPath = "mix_cache.sig" - let archivePath = "\"mix-cache-\\\$(sha256sum ${sigPath} | cut -d\" \" -f1).tar.gz\"" - let unpackageScript = "tar xzf ${archivePath} -C ${ValidationService.rootPath}" - - in [ - Cmd.runInDocker ValidationService.docker "elixir --version | tail -n1 > ${sigPath}", - Cmd.run "cat ${sigPath}", - Cmd.cacheThrough ValidationService.docker archivePath Cmd.CacheSetupCmd::{ - package = Cmd.run "tar czf ${archivePath} -C ${ValidationService.rootPath} priv/plts", - create = - let scripts = - List/map - Cmd.Type - Text - (\(cmd : Cmd.Type) -> Cmd.format cmd) - ValidationService.initCommands - in - Cmd.run - (Text/concatSep " && " - (scripts # [ "mix check --force" ])) - }, - Cmd.runInDocker ValidationService.docker ( - unpackageScript ++ - " && echo \"TODO do whatever with the cached assets\"" - ) - ] - -in Pipeline.build Pipeline.Config::{ - spec = - let dirtyDhallDir = S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/ValidationService") - - in JobSpec::{ - dirtyWhen = [ - dirtyDhallDir, - S.strictlyStart (S.contains ValidationService.rootPath) - ], - path = "Lint", - name = "ValidationService", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] - }, - steps = [ - Command.build Command.Config::{ - commands = commands, - label = "Validation service lint steps; employs various forms static analysis on the elixir codebase", - key = "lint", - target = Size.Small, - skip = Some (B/Skip.String "https://github.com/MinaProtocol/mina/issues/6285"), - docker = None Docker.Type - } - ] -} + let sigPath = "mix_cache.sig" + + let archivePath = + "\"mix-cache-\\\$(sha256sum ${sigPath} | cut -d\" \" -f1).tar.gz\"" + + let unpackageScript = + "tar xzf ${archivePath} -C ${ValidationService.rootPath}" + + in [ Cmd.runInDocker + ValidationService.docker + "elixir --version | tail -n1 > ${sigPath}" + , Cmd.run "cat ${sigPath}" + , Cmd.cacheThrough + ValidationService.docker + archivePath + Cmd.CacheSetupCmd::{ + , package = + Cmd.run + "tar czf ${archivePath} -C ${ValidationService.rootPath} priv/plts" + , create = + let scripts = + List/map + Cmd.Type + Text + (\(cmd : Cmd.Type) -> Cmd.format cmd) + ValidationService.initCommands + + in Cmd.run + ( Text/concatSep + " && " + (scripts # [ "mix check --force" ]) + ) + } + , Cmd.runInDocker + ValidationService.docker + ( unpackageScript + ++ " && echo \"TODO do whatever with the cached assets\"" + ) + ] + +in Pipeline.build + Pipeline.Config::{ + , spec = + let dirtyDhallDir = + S.strictlyStart + (S.contains "buildkite/src/Jobs/Lint/ValidationService") + + in JobSpec::{ + , dirtyWhen = + [ dirtyDhallDir + , S.strictlyStart (S.contains ValidationService.rootPath) + ] + , path = "Lint" + , name = "ValidationService" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] + } + , steps = + [ Command.build + Command.Config::{ + , commands = commands + , label = + "Validation service lint steps; employs various forms static analysis on the elixir codebase" + , key = "lint" + , target = Size.Small + , skip = Some + ( B/Skip.String + "https://github.com/MinaProtocol/mina/issues/6285" + ) + , docker = None Docker.Type + } + ] + } diff --git a/buildkite/src/Jobs/Lint/Xrefcheck.dhall b/buildkite/src/Jobs/Lint/Xrefcheck.dhall index 28a9268420c..20092e9f4ee 100644 --- a/buildkite/src/Jobs/Lint/Xrefcheck.dhall +++ b/buildkite/src/Jobs/Lint/Xrefcheck.dhall @@ -1,45 +1,46 @@ -let Prelude = ../../External/Prelude.dhall let B = ../../External/Buildkite.dhall let SelectFiles = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Cmd = ../../Lib/Cmds.dhall + let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type -in - -Pipeline.build - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ - SelectFiles.strictly (SelectFiles::{ exts = Some ["md"] }), - SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml") - ], - path = "Lint", - name = "Xrefcheck", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [] : List Cmd.Type - , label = "Verifies references in markdown" - , key = "xrefcheck" - , target = Size.Small - , soft_fail = Some (B/SoftFail.Boolean True) - , docker = Some Docker::{ - image = (../../Constants/ContainerImages.dhall).xrefcheck, - shell = None (List Text) - } +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ SelectFiles.strictly SelectFiles::{ exts = Some [ "md" ] } + , SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml") + ] + , path = "Lint" + , name = "Xrefcheck" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = [] : List Cmd.Type + , label = "Verifies references in markdown" + , key = "xrefcheck" + , target = Size.Small + , soft_fail = Some (B/SoftFail.Boolean True) + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).xrefcheck + , shell = None (List Text) + } + } + ] + } diff --git a/buildkite/src/Jobs/Release/HelmRelease.dhall b/buildkite/src/Jobs/Release/HelmRelease.dhall index 8e843171ab7..19cd1eaf143 100644 --- a/buildkite/src/Jobs/Release/HelmRelease.dhall +++ b/buildkite/src/Jobs/Release/HelmRelease.dhall @@ -1,41 +1,45 @@ -let Prelude = ../../External/Prelude.dhall - let S = ../../Lib/SelectFiles.dhall + let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -in - -Pipeline.build - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ - S.strictlyEnd (S.contains "Chart.yaml"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/HelmRelease"), - S.exactly "buildkite/scripts/helm-ci" "sh" - ], - path = "Release", - name = "HelmRelease", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ Cmd.run "HELM_RELEASE=true AUTO_DEPLOY=true buildkite/scripts/helm-ci.sh" ] - , label = "Helm chart release" - , key = "release-helm-chart" - , target = Size.Medium - , docker = None Docker.Type - , artifact_paths = [ S.contains "updates/*" ] - , depends_on = [ { name = "HelmChart", key = "lint-helm-chart" } ] +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyEnd (S.contains "Chart.yaml") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Release/HelmRelease") + , S.exactly "buildkite/scripts/helm-ci" "sh" + ] + , path = "Release" + , name = "HelmRelease" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run + "HELM_RELEASE=true AUTO_DEPLOY=true buildkite/scripts/helm-ci.sh" + ] + , label = "Helm chart release" + , key = "release-helm-chart" + , target = Size.Medium + , docker = None Docker.Type + , artifact_paths = [ S.contains "updates/*" ] + , depends_on = [ { name = "HelmChart", key = "lint-helm-chart" } ] + } + ] + } diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall index 62be772c8d6..7c7a98b6a1b 100644 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall @@ -1,44 +1,42 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall -let Command = ../../Command/Base.dhall -let Size = ../../Command/Size.dhall let DockerImage = ../../Command/DockerImage.dhall + let Profiles = ../../Constants/Profiles.dhall + let DebianVersions = ../../Constants/DebianVersions.dhall + let DebianRepo = ../../Constants/DebianRepo.dhall -let spec = DockerImage.ReleaseSpec::{ - service="itn-orchestrator", - step_key="itn-orchestrator-docker-image", - network="berkeley", - deb_repo = DebianRepo.Type.Local, - deps = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Standard -} - -in - -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/ItnOrchestratorArtifact"), - S.strictlyStart (S.contains "src/app/itn_orchestrator") - ], - path = "Release", - name = "ItnOrchestratorArtifact", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] - }, - steps = [ - DockerImage.generateStep spec - ] - } +let spec = + DockerImage.ReleaseSpec::{ + , service = "itn-orchestrator" + , step_key = "itn-orchestrator-docker-image" + , network = "berkeley" + , deb_repo = DebianRepo.Type.Local + , deps = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Profiles.Type.Standard + } + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart + (S.contains "buildkite/src/Jobs/Release/ItnOrchestratorArtifact") + , S.strictlyStart (S.contains "src/app/itn_orchestrator") + ] + , path = "Release" + , name = "ItnOrchestratorArtifact" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] + } + , steps = [ DockerImage.generateStep spec ] + } diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall index a55d5fa8099..55afc509024 100644 --- a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall +++ b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall @@ -1,49 +1,50 @@ -let Prelude = ../../External/Prelude.dhall - let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Size = ../../Command/Size.dhall -let DockerImage = ../../Command/DockerImage.dhall +let DockerImage = ../../Command/DockerImage.dhall -let spec = DockerImage.ReleaseSpec::{ - service="leaderboard", - step_key="leaderboard-docker-image" -} - -in - -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/LeaderboardArtifact"), - S.strictlyStart (S.contains "frontend/leaderboard") - ], - path = "Release", - name = "LeaderboardArtifact", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.run "echo export MINA_VERSION=$(cat frontend/leaderboard/package.json | jq '.version') > LEADERBOARD_DEPLOY_ENV" - ], - label = "Setup Leaderboard docker image deploy environment", - key = "setup-deploy-env", - target = Size.Small, - artifact_paths = [ S.contains "frontend/leaderboard/package.json" ] - }, - DockerImage.generateStep spec - ] - } +let spec = + DockerImage.ReleaseSpec::{ + , service = "leaderboard" + , step_key = "leaderboard-docker-image" + } + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart + (S.contains "buildkite/src/Jobs/Release/LeaderboardArtifact") + , S.strictlyStart (S.contains "frontend/leaderboard") + ] + , path = "Release" + , name = "LeaderboardArtifact" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] + } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run + "echo export MINA_VERSION=\$(cat frontend/leaderboard/package.json | jq '.version') > LEADERBOARD_DEPLOY_ENV" + ] + , label = "Setup Leaderboard docker image deploy environment" + , key = "setup-deploy-env" + , target = Size.Small + , artifact_paths = + [ S.contains "frontend/leaderboard/package.json" ] + } + , DockerImage.generateStep spec + ] + } diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall index 27a4198c75a..cf2a027030b 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall @@ -1,18 +1,21 @@ let ArtifactPipelines = ../../Command/MinaArtifact.dhall -let DebianVersions = ../../Constants/DebianVersions.dhall -let Profiles = ../../Constants/Profiles.dhall let Artifacts = ../../Constants/Artifacts.dhall -let Toolchain = ../../Constants/Toolchain.dhall -let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall -in +let Pipeline = ../../Pipeline/Dsl.dhall -Pipeline.build - (ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - artifacts = [ Artifacts.Type.Daemon , Artifacts.Type.Archive, Artifacts.Type.ArchiveMigration, Artifacts.Type.BatchTxn , Artifacts.Type.TestExecutive , - Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction, Artifacts.Type.FunctionalTestSuite ] - } - ) \ No newline at end of file +in Pipeline.build + ( ArtifactPipelines.pipeline + ArtifactPipelines.MinaBuildSpec::{ + , artifacts = + [ Artifacts.Type.Daemon + , Artifacts.Type.Archive + , Artifacts.Type.ArchiveMigration + , Artifacts.Type.BatchTxn + , Artifacts.Type.TestExecutive + , Artifacts.Type.Rosetta + , Artifacts.Type.ZkappTestTransaction + , Artifacts.Type.FunctionalTestSuite + ] + } + ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall index e51d7318963..7181e5fd0a5 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall @@ -1,19 +1,15 @@ let ArtifactPipelines = ../../Command/MinaArtifact.dhall -let DebianVersions = ../../Constants/DebianVersions.dhall -let Profiles = ../../Constants/Profiles.dhall let Artifacts = ../../Constants/Artifacts.dhall -let Toolchain = ../../Constants/Toolchain.dhall + let BuildFlags = ../../Constants/BuildFlags.dhall -let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall -in +let Pipeline = ../../Pipeline/Dsl.dhall -Pipeline.build - (ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - artifacts = [ Artifacts.Type.Daemon, Artifacts.Type.Archive ], - buildFlags = BuildFlags.Type.Instrumented - } - ) \ No newline at end of file +in Pipeline.build + ( ArtifactPipelines.pipeline + ArtifactPipelines.MinaBuildSpec::{ + , artifacts = [ Artifacts.Type.Daemon, Artifacts.Type.Archive ] + , buildFlags = BuildFlags.Type.Instrumented + } + ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall index 323a4dac6f1..68ce331394c 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall @@ -1,18 +1,15 @@ let ArtifactPipelines = ../../Command/MinaArtifact.dhall -let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall + let Artifacts = ../../Constants/Artifacts.dhall -let Toolchain = ../../Constants/Toolchain.dhall -let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall -in +let Pipeline = ../../Pipeline/Dsl.dhall -Pipeline.build - (ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - artifacts = [ Artifacts.Type.Daemon ], - profile = Profiles.Type.Lightnet - } - ) \ No newline at end of file +in Pipeline.build + ( ArtifactPipelines.pipeline + ArtifactPipelines.MinaBuildSpec::{ + , artifacts = [ Artifacts.Type.Daemon ] + , profile = Profiles.Type.Lightnet + } + ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall b/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall index faf0878c7ab..d18c2fc3cfc 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBuster.dhall @@ -1,18 +1,23 @@ let ArtifactPipelines = ../../Command/MinaArtifact.dhall let DebianVersions = ../../Constants/DebianVersions.dhall -let Profiles = ../../Constants/Profiles.dhall + let Artifacts = ../../Constants/Artifacts.dhall -let Toolchain = ../../Constants/Toolchain.dhall -let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall -in +let Pipeline = ../../Pipeline/Dsl.dhall -Pipeline.build - (ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - artifacts = [ Artifacts.Type.Daemon , Artifacts.Type.Archive, Artifacts.Type.ArchiveMigration , Artifacts.Type.BatchTxn , Artifacts.Type.TestExecutive , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction ], - debVersion = DebianVersions.DebVersion.Buster - } - ) \ No newline at end of file +in Pipeline.build + ( ArtifactPipelines.pipeline + ArtifactPipelines.MinaBuildSpec::{ + , artifacts = + [ Artifacts.Type.Daemon + , Artifacts.Type.Archive + , Artifacts.Type.ArchiveMigration + , Artifacts.Type.BatchTxn + , Artifacts.Type.TestExecutive + , Artifacts.Type.Rosetta + , Artifacts.Type.ZkappTestTransaction + ] + , debVersion = DebianVersions.DebVersion.Buster + } + ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall index 577dfc0952a..7face04e5d4 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall @@ -1,18 +1,23 @@ let ArtifactPipelines = ../../Command/MinaArtifact.dhall let DebianVersions = ../../Constants/DebianVersions.dhall -let Profiles = ../../Constants/Profiles.dhall + let Artifacts = ../../Constants/Artifacts.dhall -let Toolchain = ../../Constants/Toolchain.dhall -let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall -in +let Pipeline = ../../Pipeline/Dsl.dhall -Pipeline.build - (ArtifactPipelines.pipeline - ArtifactPipelines.MinaBuildSpec::{ - artifacts = [ Artifacts.Type.Daemon , Artifacts.Type.Archive , Artifacts.Type.ArchiveMigration, Artifacts.Type.BatchTxn , Artifacts.Type.TestExecutive , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction ], - debVersion = DebianVersions.DebVersion.Focal - } - ) \ No newline at end of file +in Pipeline.build + ( ArtifactPipelines.pipeline + ArtifactPipelines.MinaBuildSpec::{ + , artifacts = + [ Artifacts.Type.Daemon + , Artifacts.Type.Archive + , Artifacts.Type.ArchiveMigration + , Artifacts.Type.BatchTxn + , Artifacts.Type.TestExecutive + , Artifacts.Type.Rosetta + , Artifacts.Type.ZkappTestTransaction + ] + , debVersion = DebianVersions.DebVersion.Focal + } + ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactHardforkBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactHardforkBullseye.dhall index 8c9571942ac..d956c42d42d 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactHardforkBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactHardforkBullseye.dhall @@ -1,7 +1,6 @@ let HardforkPackageGeneration = ../../Command/HardforkPackageGeneration.dhall -let DebianVersions = ../../Constants/DebianVersions.dhall - let Pipeline = ../../Pipeline/Dsl.dhall -in Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{=} ) +in Pipeline.build + (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{=}) diff --git a/buildkite/src/Jobs/Release/MinaArtifactHardforkBuster.dhall b/buildkite/src/Jobs/Release/MinaArtifactHardforkBuster.dhall index db8b2195f2f..b92435c718c 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactHardforkBuster.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactHardforkBuster.dhall @@ -4,6 +4,9 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Pipeline = ../../Pipeline/Dsl.dhall -in Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{ - codename = DebianVersions.DebVersion.Buster -}) +in Pipeline.build + ( HardforkPackageGeneration.pipeline + HardforkPackageGeneration.Spec::{ + , codename = DebianVersions.DebVersion.Buster + } + ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactHardforkFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactHardforkFocal.dhall index 9df2f625980..b60e28822ad 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactHardforkFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactHardforkFocal.dhall @@ -4,6 +4,9 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Pipeline = ../../Pipeline/Dsl.dhall -in Pipeline.build (HardforkPackageGeneration.pipeline HardforkPackageGeneration.Spec::{ - codename = DebianVersions.DebVersion.Focal -}) \ No newline at end of file +in Pipeline.build + ( HardforkPackageGeneration.pipeline + HardforkPackageGeneration.Spec::{ + , codename = DebianVersions.DebVersion.Focal + } + ) diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall index c01b4999a74..8603e4001fd 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -1,50 +1,38 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall -let Command = ../../Command/Base.dhall -let Size = ../../Command/Size.dhall let DockerImage = ../../Command/DockerImage.dhall -let DockerLogin = ../../Command/DockerLogin/Type.dhall - - -in -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "dockerfiles/stages/1-"), - S.strictlyStart (S.contains "dockerfiles/stages/2-"), - S.strictlyStart (S.contains "dockerfiles/stages/3-"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"), - S.strictly (S.contains "opam.export"), - -- Rust version has changed - S.strictlyEnd (S.contains "rust-toolchain.toml") - ], - path = "Release", - name = "MinaToolchainArtifactBullseye", - tags = [ PipelineTag.Type.Toolchain ] - }, - steps = [ - - -- mina-toolchain Debian 11 "Bullseye" Toolchain - let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - service="mina-toolchain", - deb_codename="bullseye", - extra_args="--no-cache", - step_key="toolchain-bullseye-docker-image" +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "dockerfiles/stages/1-") + , S.strictlyStart (S.contains "dockerfiles/stages/2-") + , S.strictlyStart (S.contains "dockerfiles/stages/3-") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact") + , S.strictly (S.contains "opam.export") + , S.strictlyEnd (S.contains "rust-toolchain.toml") + ] + , path = "Release" + , name = "MinaToolchainArtifactBullseye" + , tags = [ PipelineTag.Type.Toolchain ] + } + , steps = + [ let toolchainBullseyeSpec = + DockerImage.ReleaseSpec::{ + , service = "mina-toolchain" + , deb_codename = "bullseye" + , extra_args = "--no-cache" + , step_key = "toolchain-bullseye-docker-image" + } + + in DockerImage.generateStep toolchainBullseyeSpec + ] } - - in - - DockerImage.generateStep toolchainBullseyeSpec - - ] - } \ No newline at end of file diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall index 8abe19cf794..3057316b1cc 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBuster.dhall @@ -1,52 +1,41 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let PipelineMode = ../../Pipeline/Mode.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall -let Command = ../../Command/Base.dhall -let Size = ../../Command/Size.dhall let DockerImage = ../../Command/DockerImage.dhall -let DockerLogin = ../../Command/DockerLogin/Type.dhall - -in - -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "dockerfiles/stages/1-"), - S.strictlyStart (S.contains "dockerfiles/stages/2-"), - S.strictlyStart (S.contains "dockerfiles/stages/3-"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"), - S.strictly (S.contains "opam.export"), - -- Rust version has changed - S.strictlyEnd (S.contains "rust-toolchain.toml") - ], - path = "Release", - name = "MinaToolchainArtifactBuster", - tags = [ PipelineTag.Type.Toolchain ], - mode = PipelineMode.Type.Stable - }, - steps = [ - - -- mina-toolchain Debian 10 "Buster" Toolchain - let toolchainBusterSpec = DockerImage.ReleaseSpec::{ - service="mina-toolchain", - deb_codename="buster", - extra_args="--no-cache", - step_key="toolchain-buster-docker-image" +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "dockerfiles/stages/1-") + , S.strictlyStart (S.contains "dockerfiles/stages/2-") + , S.strictlyStart (S.contains "dockerfiles/stages/3-") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact") + , S.strictly (S.contains "opam.export") + , S.strictlyEnd (S.contains "rust-toolchain.toml") + ] + , path = "Release" + , name = "MinaToolchainArtifactBuster" + , tags = [ PipelineTag.Type.Toolchain ] + , mode = PipelineMode.Type.Stable + } + , steps = + [ let toolchainBusterSpec = + DockerImage.ReleaseSpec::{ + , service = "mina-toolchain" + , deb_codename = "buster" + , extra_args = "--no-cache" + , step_key = "toolchain-buster-docker-image" + } + + in DockerImage.generateStep toolchainBusterSpec + ] } - - in - - DockerImage.generateStep toolchainBusterSpec - - ] - } \ No newline at end of file diff --git a/buildkite/src/Jobs/Release/TestnetAlerts.dhall b/buildkite/src/Jobs/Release/TestnetAlerts.dhall index dbe400524eb..83d8b40b045 100644 --- a/buildkite/src/Jobs/Release/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Release/TestnetAlerts.dhall @@ -1,48 +1,49 @@ -let Prelude = ../../External/Prelude.dhall -let B = ../../External/Buildkite.dhall - -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - let S = ../../Lib/SelectFiles.dhall + let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -in - -Pipeline.build - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "automation/terraform/modules/testnet-alerts"), - S.exactly "automation/terraform/monitoring/o1-testnet-alerts" "tf", - S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") - ], - path = "Release", - name = "TestnetAlerts", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ - --- destroy state prior to start to ensure reset - Cmd.run "cd automation/terraform/monitoring && terraform init && terraform destroy -auto-approve", - Cmd.run "terraform apply -auto-approve -target module.o1testnet_alerts.docker_container.sync_alert_rules" +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart + (S.contains "automation/terraform/modules/testnet-alerts") + , S.exactly "automation/terraform/monitoring/o1-testnet-alerts" "tf" + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Release/TestnetAlerts") ] - , label = "Deploy Testnet alert rules" - , key = "deploy-testnet-alerts" - , target = Size.Medium - , depends_on = [ { name = "TestnetAlerts", key = "lint-testnet-alerts" } ] - , docker = None Docker.Type - , `if` = Some "build.env('DEPLOY_ALERTS') == 'true'" + , path = "Release" + , name = "TestnetAlerts" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run + "cd automation/terraform/monitoring && terraform init && terraform destroy -auto-approve" + , Cmd.run + "terraform apply -auto-approve -target module.o1testnet_alerts.docker_container.sync_alert_rules" + ] + , label = "Deploy Testnet alert rules" + , key = "deploy-testnet-alerts" + , target = Size.Medium + , depends_on = + [ { name = "TestnetAlerts", key = "lint-testnet-alerts" } ] + , docker = None Docker.Type + , if = Some "build.env('DEPLOY_ALERTS') == 'true'" + } + ] + } diff --git a/buildkite/src/Jobs/Release/TraceTool.dhall b/buildkite/src/Jobs/Release/TraceTool.dhall index af915364a0d..7426bbbbde3 100644 --- a/buildkite/src/Jobs/Release/TraceTool.dhall +++ b/buildkite/src/Jobs/Release/TraceTool.dhall @@ -1,39 +1,43 @@ -let Prelude = ../../External/Prelude.dhall - let S = ../../Lib/SelectFiles.dhall -let Cmd = ../../Lib/Cmds.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall +let Size = ../../Command/Size.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall -in - -Pipeline.build - Pipeline.Config::{ - spec = JobSpec::{ - dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TraceTool") ], - path = "Release", - name = "TraceTool", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] - }, - steps = [ - Command.build - Command.Config::{ - commands = RunInToolchain.runInToolchain ([] : List Text) "cd src/app/trace-tool && PATH=/home/opam/.cargo/bin:$PATH cargo build --release" - , label = "Build trace-tool" - , key = "build-trace-tool" - , target = Size.Small - , docker = None Docker.Type - , artifact_paths = [ S.contains "src/app/trace-tool/target/release/trace-tool" ] +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.contains "src/app/trace-tool" + , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TraceTool") + ] + , path = "Release" + , name = "TraceTool" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + ([] : List Text) + "cd src/app/trace-tool && PATH=/home/opam/.cargo/bin:\$PATH cargo build --release" + , label = "Build trace-tool" + , key = "build-trace-tool" + , target = Size.Small + , docker = None Docker.Type + , artifact_paths = + [ S.contains "src/app/trace-tool/target/release/trace-tool" ] + } + ] + } diff --git a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall index 96e1207f1d9..68cf0be8442 100644 --- a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall +++ b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall @@ -1,59 +1,68 @@ let Prelude = ../../External/Prelude.dhall -let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall + let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall + let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let WithCargo = ../../Command/WithCargo.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -in let user = "admin" + let password = "codarules" + let db = "archiver" + let command_key = "archive-unit-tests" -in -Pipeline.build - Pipeline.Config:: - { spec = - JobSpec:: - { dirtyWhen = +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = [ S.strictlyStart (S.contains "src") - , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/ArchiveNodeUnitTest") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Test/ArchiveNodeUnitTest") ] , path = "Test" , name = "ArchiveNodeUnitTest" , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } - , steps = - let outerDir : Text = - "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" - in - [ Command.build - Command.Config:: - { commands = - RunInToolchain.runInToolchain - [ "POSTGRES_PASSWORD=${password}" - , "POSTGRES_USER=${user}" - , "POSTGRES_DB=${db}" - , "GO=/usr/lib/go/bin/go" - , "DUNE_INSTRUMENT_WITH=bisect_ppx" - , "COVERALLS_TOKEN" - ] - (Prelude.Text.concatSep " && " - [ "bash buildkite/scripts/setup-database-for-archive-node.sh ${user} ${password} ${db}" - , "PGPASSWORD=${password} psql -h localhost -p 5432 -U ${user} -d ${db} -a -f src/app/archive/create_schema.sql" - , WithCargo.withCargo "eval \\\$(opam config env) && dune runtest src/app/archive && buildkite/scripts/upload-partial-coverage-data.sh ${command_key} dev" - ]) + , steps = + [ Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + [ "POSTGRES_PASSWORD=${password}" + , "POSTGRES_USER=${user}" + , "POSTGRES_DB=${db}" + , "GO=/usr/lib/go/bin/go" + , "DUNE_INSTRUMENT_WITH=bisect_ppx" + , "COVERALLS_TOKEN" + ] + ( Prelude.Text.concatSep + " && " + [ "bash buildkite/scripts/setup-database-for-archive-node.sh ${user} ${password} ${db}" + , "PGPASSWORD=${password} psql -h localhost -p 5432 -U ${user} -d ${db} -a -f src/app/archive/create_schema.sql" + , WithCargo.withCargo + "eval \\\$(opam config env) && dune runtest src/app/archive && buildkite/scripts/upload-partial-coverage-data.sh ${command_key} dev" + ] + ) , label = "Archive node unit tests" , key = command_key , target = Size.Large , docker = None Docker.Type , artifact_paths = [ S.contains "test_output/artifacts/*" ] } - ] - } + ] + } diff --git a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall b/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall index fbe72a1751c..b44d2b6ea96 100644 --- a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall +++ b/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall @@ -1,47 +1,48 @@ let JobSpec = ../../Pipeline/JobSpec.dhall + let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall + let PipelineTag = ../../Pipeline/Tag.dhall -let Prelude = ../../External/Prelude.dhall let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Command = ../../Command/Base.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall -let dependsOn = [ - { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" } -] - -in Pipeline.build Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/scripts/check-compatibility" "sh", - S.exactly "buildkite/src/Jobs/Test/BerkeleyCompatibility" "dhall" - ], - path = "Test", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ], - name = "BerkeleyCompatibility" - }, - steps = [ - Command.build Command.Config::{ - commands = [ - Cmd.run "buildkite/scripts/check-compatibility.sh berkeley" - ], - label = "Test: berkeley compatibilty test", - key = "berkeley-compatibilty-test", - target = Size.XLarge, - docker = None Docker.Type, - depends_on = dependsOn, - timeout_in_minutes = Some +60 - } - ] -} +let Size = ../../Command/Size.dhall +let dependsOn = + [ { name = "MinaArtifactBullseye" + , key = "daemon-berkeley-bullseye-docker-image" + } + ] +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/scripts/check-compatibility" "sh" + , S.exactly "buildkite/src/Jobs/Test/BerkeleyCompatibility" "dhall" + ] + , path = "Test" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + , name = "BerkeleyCompatibility" + } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run "buildkite/scripts/check-compatibility.sh berkeley" ] + , label = "Test: berkeley compatibilty test" + , key = "berkeley-compatibilty-test" + , target = Size.XLarge + , docker = None Docker.Type + , depends_on = dependsOn + , timeout_in_minutes = Some +60 + } + ] + } diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index 166fe90939e..f1596f7cd45 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -1,28 +1,33 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall + let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall + let DebianVersions = ../../Constants/DebianVersions.dhall + let Profiles = ../../Constants/Profiles.dhall -let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Standard - -in Pipeline.build Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/scripts/check-graphql-schema" "sh", - S.strictly (S.contains "Makefile") - ], - path = "Test", - name = "CheckGraphQLSchema", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - CheckGraphQLSchema.step dependsOn - ] -} +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Profiles.Type.Standard + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/scripts/check-graphql-schema" "sh" + , S.strictly (S.contains "Makefile") + ] + , path = "Test" + , name = "CheckGraphQLSchema" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = [ CheckGraphQLSchema.step dependsOn ] + } diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall index 0ba1ae287d2..2396f299df0 100644 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ b/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall @@ -1,30 +1,46 @@ let S = ../../Lib/SelectFiles.dhall + let B = ../../External/Buildkite.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + let JobSpec = ../../Pipeline/JobSpec.dhall + let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall + let Profiles = ../../Constants/Profiles.dhall + let Dockers = ../../Constants/DockerVersions.dhall -let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "daemon-berkeley" - -in Pipeline.build Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/scripts/connect-to-testnet" "sh", - S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall", - S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" - ], - path = "Test", - name = "ConnectToBerkeley", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - ConnectToTestnet.step dependsOn "berkeley" "40s" "2m" (B/SoftFail.Boolean True) - ] -} +let dependsOn = + Dockers.dependsOn + Dockers.Type.Bullseye + Profiles.Type.Standard + "daemon-berkeley" + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/scripts/connect-to-testnet" "sh" + , S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall" + , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" + ] + , path = "Test" + , name = "ConnectToBerkeley" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = + [ ConnectToTestnet.step + dependsOn + "berkeley" + "40s" + "2m" + (B/SoftFail.Boolean True) + ] + } diff --git a/buildkite/src/Jobs/Test/CoverageTearDown.dhall b/buildkite/src/Jobs/Test/CoverageTearDown.dhall index aa10f2a5ff8..8a891806be9 100644 --- a/buildkite/src/Jobs/Test/CoverageTearDown.dhall +++ b/buildkite/src/Jobs/Test/CoverageTearDown.dhall @@ -1,33 +1,35 @@ -let Prelude = ../../External/Prelude.dhall let S = ../../Lib/SelectFiles.dhall -let Cmd = ../../Lib/Cmds.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let Command = ../../Command/Base.dhall -let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall + let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall + let PipelineTag = ../../Pipeline/Tag.dhall -in Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ S.everything ], - path = "Test", - tags = [ PipelineTag.Type.TearDown ], - name = "CoverageTearDown" - } - , steps = [ - Command.build - Command.Config::{ - commands = - RunInToolchain.runInToolchainBullseye ["COVERALLS_TOKEN"] - "buildkite/scripts/finish-coverage-data-upload.sh", - label = "Finish coverage data gathering", - key = "finish-coverage-data", - target = Size.Small +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = [ S.everything ] + , path = "Test" + , tags = [ PipelineTag.Type.TearDown ] + , name = "CoverageTearDown" } - ] - } \ No newline at end of file + , steps = + [ Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchainBullseye + [ "COVERALLS_TOKEN" ] + "buildkite/scripts/finish-coverage-data-upload.sh" + , label = "Finish coverage data gathering" + , key = "finish-coverage-data" + , target = Size.Small + } + ] + } diff --git a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall index 5e6f025c382..edf423131f1 100644 --- a/buildkite/src/Jobs/Test/DaemonUnitTest.dhall +++ b/buildkite/src/Jobs/Test/DaemonUnitTest.dhall @@ -1,55 +1,56 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -let buildTestCmd : Text -> Text -> Size -> Command.Type = \(profile : Text) -> \(path : Text) -> \(cmd_target : Size) -> - let command_key = "unit-test-${profile}" - in - Command.build - Command.Config::{ - commands = RunInToolchain.runInToolchain ["DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN"] "buildkite/scripts/unit-test.sh ${profile} ${path} && buildkite/scripts/upload-partial-coverage-data.sh ${command_key} dev", - label = "${profile} unit-tests", - key = command_key, - target = cmd_target, - docker = None Docker.Type, - artifact_paths = [ S.contains "core_dumps/*" ] - } - -in - -Pipeline.build - Pipeline.Config::{ - spec = - let unitDirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.strictly (S.contains "Makefile"), - S.exactly "buildkite/src/Jobs/Test/DaemonUnitTest" "dhall", - S.exactly "buildkite/src/Constants/ContainerImages" "dhall", - S.exactly "scripts/link-coredumps" "sh", - S.exactly "buildkite/scripts/unit-test" "sh" - ] - - in - - JobSpec::{ - dirtyWhen = unitDirtyWhen, - path = "Test", - name = "DaemonUnitTest", - tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ] - }, - steps = [ - buildTestCmd "dev" "src/lib" Size.XLarge - ] - } +let buildTestCmd + : Text -> Text -> Size -> Command.Type + = \(profile : Text) + -> \(path : Text) + -> \(cmd_target : Size) + -> let command_key = "unit-test-${profile}" + + in Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + [ "DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN" ] + "buildkite/scripts/unit-test.sh ${profile} ${path} && buildkite/scripts/upload-partial-coverage-data.sh ${command_key} dev" + , label = "${profile} unit-tests" + , key = command_key + , target = cmd_target + , docker = None Docker.Type + , artifact_paths = [ S.contains "core_dumps/*" ] + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let unitDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.strictly (S.contains "Makefile") + , S.exactly "buildkite/src/Jobs/Test/DaemonUnitTest" "dhall" + , S.exactly "buildkite/src/Constants/ContainerImages" "dhall" + , S.exactly "scripts/link-coredumps" "sh" + , S.exactly "buildkite/scripts/unit-test" "sh" + ] + + in JobSpec::{ + , dirtyWhen = unitDirtyWhen + , path = "Test" + , name = "DaemonUnitTest" + , tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ] + } + , steps = [ buildTestCmd "dev" "src/lib" Size.XLarge ] + } diff --git a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall index ba907ed0d5e..99132224a5b 100644 --- a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall +++ b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall @@ -1,47 +1,48 @@ let JobSpec = ../../Pipeline/JobSpec.dhall + let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineMode = ../../Pipeline/Mode.dhall + let PipelineTag = ../../Pipeline/Tag.dhall -let Prelude = ../../External/Prelude.dhall let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Command = ../../Command/Base.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall -let dependsOn = [ - { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" } -] - -in Pipeline.build Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/scripts/check-compatibility" "sh", - S.exactly "buildkite/src/Jobs/Test/DevelopCompatibility" "dhall" - ], - path = "Test", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ], - name = "DevelopCompatibility" - }, - steps = [ - Command.build Command.Config::{ - commands = [ - Cmd.run "buildkite/scripts/check-compatibility.sh develop" - ], - label = "Test: develop compatibilty test", - key = "develop-compatibilty-test", - target = Size.XLarge, - docker = None Docker.Type, - depends_on = dependsOn, - timeout_in_minutes = Some +60 - } - ] -} +let Size = ../../Command/Size.dhall +let dependsOn = + [ { name = "MinaArtifactBullseye" + , key = "daemon-berkeley-bullseye-docker-image" + } + ] +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/scripts/check-compatibility" "sh" + , S.exactly "buildkite/src/Jobs/Test/DevelopCompatibility" "dhall" + ] + , path = "Test" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + , name = "DevelopCompatibility" + } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run "buildkite/scripts/check-compatibility.sh develop" ] + , label = "Test: develop compatibilty test" + , key = "develop-compatibilty-test" + , target = Size.XLarge + , docker = None Docker.Type + , depends_on = dependsOn + , timeout_in_minutes = Some +60 + } + ] + } diff --git a/buildkite/src/Jobs/Test/EmergencyHfTest.dhall b/buildkite/src/Jobs/Test/EmergencyHfTest.dhall index 876afed84de..9479e0e1b70 100755 --- a/buildkite/src/Jobs/Test/EmergencyHfTest.dhall +++ b/buildkite/src/Jobs/Test/EmergencyHfTest.dhall @@ -1,18 +1,14 @@ let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall -let JobSpec = ../../Pipeline/JobSpec.dhall +let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall -let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall - -let ReplayerTest = ../../Command/ReplayerTest.dhall -let Profiles = ../../Constants/Profiles.dhall -let Dockers = ../../Constants/DockerVersions.dhall +let Size = ../../Command/Size.dhall let Cmd = ../../Lib/Cmds.dhall @@ -27,15 +23,16 @@ in Pipeline.build , name = "EmergencyHfTest" , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } - , steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.run "PSQL=\"docker exec replayer-postgres psql\" ./scripts/archive/emergency_hf/test/runner.sh " - ], - label = "Emergency HF test", - key = "emergency-hf-test", - target = Size.Large - } - ] + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run + "PSQL=\"docker exec replayer-postgres psql\" ./scripts/archive/emergency_hf/test/runner.sh " + ] + , label = "Emergency HF test" + , key = "emergency-hf-test" + , target = Size.Large + } + ] } diff --git a/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall b/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall index 826f38be20d..67137cc5fa9 100644 --- a/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall +++ b/buildkite/src/Jobs/Test/FuzzyZkappTest.dhall @@ -1,56 +1,72 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineMode = ../../Pipeline/Mode.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall + let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -let buildTestCmd : Text -> Text -> Natural -> Natural -> Size -> Command.Type = \(profile : Text) -> \(path : Text) -> \(timeout : Natural) -> \(individual_test_timeout : Natural) -> \(cmd_target : Size) -> - let timeout = Natural/show timeout in - let individual_test_timeout = Natural/show individual_test_timeout in - let key = "fuzzy-zkapp-unit-test-${profile}" in - Command.build - Command.Config::{ - commands = RunInToolchain.runInToolchain ["DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN"] - "buildkite/scripts/fuzzy-zkapp-test.sh ${profile} ${path} ${timeout} ${individual_test_timeout} && buildkite/scripts/upload-partial-coverage-data.sh ${key} dev", - label = "Fuzzy zkapp unit tests", - key = key, - target = cmd_target, - docker = None Docker.Type, - artifact_paths = [ S.contains "core_dumps/*" ], - flake_retry_limit = Some 0 - } -in - -Pipeline.build - Pipeline.Config::{ - spec = - let unitDirtyWhen = [ - S.strictlyStart (S.contains "src/lib"), - S.strictlyStart (S.contains "src/lib/transaction_snark/test/zkapp_fuzzy"), - S.exactly "buildkite/src/Jobs/Test/FuzzyZkappTest" "dhall", - S.exactly "buildkite/scripts/fuzzy-zkapp-test" "sh" - ] - - in - - JobSpec::{ - dirtyWhen = unitDirtyWhen, - path = "Test", - name = "FuzzyZkappTest", - tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ], - mode = PipelineMode.Type.Stable - }, - steps = [ - buildTestCmd "dev" "src/lib/transaction_snark/test/zkapp_fuzzy/zkapp_fuzzy.exe" 4200 150 Size.Small - ] - } +let buildTestCmd + : Text -> Text -> Natural -> Natural -> Size -> Command.Type + = \(profile : Text) + -> \(path : Text) + -> \(timeout : Natural) + -> \(individual_test_timeout : Natural) + -> \(cmd_target : Size) + -> let timeout = Natural/show timeout + + let individual_test_timeout = Natural/show individual_test_timeout + + let key = "fuzzy-zkapp-unit-test-${profile}" + + in Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + [ "DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN" ] + "buildkite/scripts/fuzzy-zkapp-test.sh ${profile} ${path} ${timeout} ${individual_test_timeout} && buildkite/scripts/upload-partial-coverage-data.sh ${key} dev" + , label = "Fuzzy zkapp unit tests" + , key = key + , target = cmd_target + , docker = None Docker.Type + , artifact_paths = [ S.contains "core_dumps/*" ] + , flake_retry_limit = Some 0 + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let unitDirtyWhen = + [ S.strictlyStart (S.contains "src/lib") + , S.strictlyStart + (S.contains "src/lib/transaction_snark/test/zkapp_fuzzy") + , S.exactly "buildkite/src/Jobs/Test/FuzzyZkappTest" "dhall" + , S.exactly "buildkite/scripts/fuzzy-zkapp-test" "sh" + ] + + in JobSpec::{ + , dirtyWhen = unitDirtyWhen + , path = "Test" + , name = "FuzzyZkappTest" + , tags = [ PipelineTag.Type.VeryLong, PipelineTag.Type.Test ] + , mode = PipelineMode.Type.Stable + } + , steps = + [ buildTestCmd + "dev" + "src/lib/transaction_snark/test/zkapp_fuzzy/zkapp_fuzzy.exe" + 4200 + 150 + Size.Small + ] + } diff --git a/buildkite/src/Jobs/Test/HFTest.dhall b/buildkite/src/Jobs/Test/HFTest.dhall index 6b989b996e6..fa98c3b781c 100644 --- a/buildkite/src/Jobs/Test/HFTest.dhall +++ b/buildkite/src/Jobs/Test/HFTest.dhall @@ -1,52 +1,56 @@ let ContainerImages = ../../Constants/ContainerImages.dhall let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let PipelineMode = ../../Pipeline/Mode.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -in - -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/HFTest" "dhall", - S.strictlyStart (S.contains "scripts/hardfork"), - S.strictlyStart (S.contains "nix"), - S.exactly "flake" "nix", - S.exactly "flake" "lock", - S.exactly "default" "nix" - ], - path = "Test", - name = "HFTest", - mode = PipelineMode.Type.Stable, - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.runInDocker Cmd.Docker::{ - image = ContainerImages.nixos, - privileged = True - } "./scripts/hardfork/build-and-test.sh $BUILDKITE_BRANCH" - ], - label = "hard fork test", - key = "hard-fork-test", - target = Size.Small, - docker = None Docker.Type, - timeout_in_minutes = Some +420 +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/HFTest" "dhall" + , S.strictlyStart (S.contains "scripts/hardfork") + , S.strictlyStart (S.contains "nix") + , S.exactly "flake" "nix" + , S.exactly "flake" "lock" + , S.exactly "default" "nix" + ] + , path = "Test" + , name = "HFTest" + , mode = PipelineMode.Type.Stable + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.runInDocker + Cmd.Docker::{ + , image = ContainerImages.nixos + , privileged = True + } + "./scripts/hardfork/build-and-test.sh \$BUILDKITE_BRANCH" + ] + , label = "hard fork test" + , key = "hard-fork-test" + , target = Size.Small + , docker = None Docker.Type + , timeout_in_minutes = Some +420 + } + ] + } diff --git a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall index ce5a974ee22..822d4f4941a 100644 --- a/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall +++ b/buildkite/src/Jobs/Test/Libp2pUnitTest.dhall @@ -1,64 +1,67 @@ let ContainerImages = ../../Constants/ContainerImages.dhall let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall -let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall -in +let Docker = ../../Command/Docker/Type.dhall -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src/app/libp2p_helper"), - S.strictlyStart (S.contains "src/libp2p_ipc"), - S.exactly "Makefile" "", - S.exactly "buildkite/src/Jobs/Test/Libp2pUnitTest" "dhall" - ], - path = "Test", - name = "Libp2pUnitTest", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.run "chmod -R 777 src/libp2p_ipc", - Cmd.runInDocker - Cmd.Docker:: - { image=ContainerImages.minaToolchain - , extraEnv = [ "GO=/usr/lib/go/bin/go" ] - } "make -C src/app/libp2p_helper test" - ], - label = "libp2p unit-tests", - key = "libp2p-unit-tests", - target = Size.Large, - docker = None Docker.Type - }, - Command.build - Command.Config::{ - commands = [ - Cmd.run "chmod -R 777 src/libp2p_ipc", - Cmd.runInDocker - Cmd.Docker:: - { image=ContainerImages.minaToolchain - , extraEnv = [ "GO=/usr/lib/go/bin/go" ] - } "make -C src/app/libp2p_helper test-bs-qc" - ], - label = "libp2p bitswap QuickCheck", - key = "libp2p-bs-qc", - target = Size.Large, - docker = None Docker.Type, - timeout_in_minutes = Some +45 +let Size = ../../Command/Size.dhall +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src/app/libp2p_helper") + , S.strictlyStart (S.contains "src/libp2p_ipc") + , S.exactly "Makefile" "" + , S.exactly "buildkite/src/Jobs/Test/Libp2pUnitTest" "dhall" + ] + , path = "Test" + , name = "Libp2pUnitTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run "chmod -R 777 src/libp2p_ipc" + , Cmd.runInDocker + Cmd.Docker::{ + , image = ContainerImages.minaToolchain + , extraEnv = [ "GO=/usr/lib/go/bin/go" ] + } + "make -C src/app/libp2p_helper test" + ] + , label = "libp2p unit-tests" + , key = "libp2p-unit-tests" + , target = Size.Large + , docker = None Docker.Type + } + , Command.build + Command.Config::{ + , commands = + [ Cmd.run "chmod -R 777 src/libp2p_ipc" + , Cmd.runInDocker + Cmd.Docker::{ + , image = ContainerImages.minaToolchain + , extraEnv = [ "GO=/usr/lib/go/bin/go" ] + } + "make -C src/app/libp2p_helper test-bs-qc" + ] + , label = "libp2p bitswap QuickCheck" + , key = "libp2p-bs-qc" + , target = Size.Large + , docker = None Docker.Type + , timeout_in_minutes = Some +45 + } + ] + } diff --git a/buildkite/src/Jobs/Test/NixBuildTest.dhall b/buildkite/src/Jobs/Test/NixBuildTest.dhall index 1fa9df268cc..2bd10248a95 100644 --- a/buildkite/src/Jobs/Test/NixBuildTest.dhall +++ b/buildkite/src/Jobs/Test/NixBuildTest.dhall @@ -1,49 +1,52 @@ let ContainerImages = ../../Constants/ContainerImages.dhall let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -in - -Pipeline.build - Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/NixBuildTest" "dhall", - S.exactly "buildkite/scripts/test-nix" "sh", - S.strictlyStart (S.contains "nix"), - S.exactly "flake" "nix", - S.exactly "flake" "lock", - S.exactly "default" "nix" - ], - path = "Test", - name = "NixBuildTest", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] - }, - steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.runInDocker Cmd.Docker::{ - image = ContainerImages.nixos, - privileged = True - } "./buildkite/scripts/test-nix.sh $BUILDKITE_BRANCH" - ], - label = "nix build tests", - key = "nix-build-tests", - target = Size.Small, - docker = None Docker.Type +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/NixBuildTest" "dhall" + , S.exactly "buildkite/scripts/test-nix" "sh" + , S.strictlyStart (S.contains "nix") + , S.exactly "flake" "nix" + , S.exactly "flake" "lock" + , S.exactly "default" "nix" + ] + , path = "Test" + , name = "NixBuildTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } - ] - } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.runInDocker + Cmd.Docker::{ + , image = ContainerImages.nixos + , privileged = True + } + "./buildkite/scripts/test-nix.sh \$BUILDKITE_BRANCH" + ] + , label = "nix build tests" + , key = "nix-build-tests" + , target = Size.Small + , docker = None Docker.Type + } + ] + } diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 26ca788b177..d1fcc44368c 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -1,14 +1,19 @@ let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let ReplayerTest = ../../Command/ReplayerTest.dhall + let Profiles = ../../Constants/Profiles.dhall + let Dockers = ../../Constants/DockerVersions.dhall -let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" +let dependsOn = + Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index b629e86b304..ed93bbba9a6 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -1,52 +1,57 @@ -let Prelude = ../../External/Prelude.dhall -let B = ../../External/Buildkite.dhall - let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall + let Size = ../../Command/Size.dhall -let Libp2p = ../../Command/Libp2pHelperBuild.dhall -let DockerImage = ../../Command/DockerImage.dhall + let Profiles = ../../Constants/Profiles.dhall + let Dockers = ../../Constants/DockerVersions.dhall -let dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall", - S.exactly "buildkite/scripts/rosetta-integration-tests" "sh", - S.exactly "buildkite/scripts/rosetta-integration-tests-fast" "sh" -] - -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - -in - -Pipeline.build - Pipeline.Config:: - { spec = - JobSpec::{ - dirtyWhen = dirtyWhen, - path = "Test", - name = "RosettaIntegrationTests", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - } - , steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"), - Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}"} "buildkite/scripts/rosetta-integration-tests-fast.sh" - ], - label = "Rosetta integration tests Bullseye" - , key = "rosetta-integration-tests-bullseye" - , target = Size.Small - , depends_on = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "rosetta" +let dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" + , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + , S.exactly "buildkite/scripts/rosetta-integration-tests-fast" "sh" + ] + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = dirtyWhen + , path = "Test" + , name = "RosettaIntegrationTests" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] } - ] - } \ No newline at end of file + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run + "export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}" + , Cmd.runInDocker + Cmd.Docker::{ + , image = + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}" + } + "buildkite/scripts/rosetta-integration-tests-fast.sh" + ] + , label = "Rosetta integration tests Bullseye" + , key = "rosetta-integration-tests-bullseye" + , target = Size.Small + , depends_on = + Dockers.dependsOn + Dockers.Type.Bullseye + Profiles.Type.Standard + "rosetta" + } + ] + } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index 59f6451e1bd..5759c90f0ab 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -1,51 +1,53 @@ -let Prelude = ../../External/Prelude.dhall -let B = ../../External/Buildkite.dhall - let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineMode = ../../Pipeline/Mode.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall + let Size = ../../Command/Size.dhall -let Libp2p = ../../Command/Libp2pHelperBuild.dhall -let DockerImage = ../../Command/DockerImage.dhall -let DebianVersions = ../../Constants/DebianVersions.dhall - -let dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall", - S.exactly "buildkite/scripts/rosetta-integration-tests" "sh", - S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh" -] - -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type - -in - -Pipeline.build - Pipeline.Config:: - { spec = - JobSpec::{ - dirtyWhen = dirtyWhen, - path = "Test", - name = "RosettaIntegrationTestsLong", - mode = PipelineMode.Type.Stable - } - , steps = [ - Command.build - Command.Config::{ - commands = [ - Cmd.run ("export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"), - Cmd.runInDocker Cmd.Docker::{image="gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}"} "buildkite/scripts/rosetta-integration-tests-full.sh" - ], - label = "Rosetta integration tests Bullseye Long" - , key = "rosetta-integration-tests-bullseye-long" - , target = Size.Small - , depends_on = [ { name = "MinaArtifactBullseye", key = "rosetta-bullseye-docker-image" } ] + +let dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" + , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + , S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh" + ] + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = dirtyWhen + , path = "Test" + , name = "RosettaIntegrationTestsLong" + , mode = PipelineMode.Type.Stable } - ] - } \ No newline at end of file + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run + "export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}" + , Cmd.runInDocker + Cmd.Docker::{ + , image = + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}" + } + "buildkite/scripts/rosetta-integration-tests-full.sh" + ] + , label = "Rosetta integration tests Bullseye Long" + , key = "rosetta-integration-tests-bullseye-long" + , target = Size.Small + , depends_on = + [ { name = "MinaArtifactBullseye" + , key = "rosetta-bullseye-docker-image" + } + ] + } + ] + } diff --git a/buildkite/src/Jobs/Test/RosettaUnitTest.dhall b/buildkite/src/Jobs/Test/RosettaUnitTest.dhall index 5a428dffe1f..62e6306a09c 100644 --- a/buildkite/src/Jobs/Test/RosettaUnitTest.dhall +++ b/buildkite/src/Jobs/Test/RosettaUnitTest.dhall @@ -1,50 +1,53 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -let buildTestCmd : Text -> Text -> Size -> Command.Type = \(profile : Text) -> \(path : Text) -> \(cmd_target : Size) -> - let key = "rosetta-unit-test-${profile}" in - Command.build - Command.Config::{ - commands = RunInToolchain.runInToolchain ["DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN"] "buildkite/scripts/unit-test.sh ${profile} ${path} && buildkite/scripts/upload-partial-coverage-data.sh ${key} dev", - label = "Rosetta unit tests", - key = key, - target = cmd_target, - docker = None Docker.Type, - artifact_paths = [ S.contains "core_dumps/*" ] - } - -in - -Pipeline.build - Pipeline.Config::{ - spec = - let unitDirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/RosettaUnitTest" "dhall", - S.exactly "buildkite/scripts/unit-test" "sh" - ] - - in - - JobSpec::{ - dirtyWhen = unitDirtyWhen, - path = "Test", - name = "RosettaUnitTest", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - buildTestCmd "dev" "src/app/rosetta" Size.Small - ] - } +let buildTestCmd + : Text -> Text -> Size -> Command.Type + = \(profile : Text) + -> \(path : Text) + -> \(cmd_target : Size) + -> let key = "rosetta-unit-test-${profile}" + + in Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + [ "DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN" ] + "buildkite/scripts/unit-test.sh ${profile} ${path} && buildkite/scripts/upload-partial-coverage-data.sh ${key} dev" + , label = "Rosetta unit tests" + , key = key + , target = cmd_target + , docker = None Docker.Type + , artifact_paths = [ S.contains "core_dumps/*" ] + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let unitDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/RosettaUnitTest" "dhall" + , S.exactly "buildkite/scripts/unit-test" "sh" + ] + + in JobSpec::{ + , dirtyWhen = unitDirtyWhen + , path = "Test" + , name = "RosettaUnitTest" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = [ buildTestCmd "dev" "src/app/rosetta" Size.Small ] + } diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index f83c74d5a5c..6204ebc1013 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -1,61 +1,63 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let DebianVersions = ../../Constants/DebianVersions.dhall + let Profiles = ../../Constants/Profiles.dhall let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall +let Size = ../../Command/Size.dhall -let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Standard - -in - -let buildTestCmd : Size -> List Command.TaggedKey.Type -> Command.Type = \(cmd_target : Size) -> \(dependsOn : List Command.TaggedKey.Type) -> - Command.build - Command.Config::{ - commands = - RunInToolchain.runInToolchain ([] : List Text) "buildkite/scripts/run-snark-transaction-profiler.sh" - , - label = "Snark Transaction Profiler", - key = "snark-transaction-profiler", - target = cmd_target, - docker = None Docker.Type, - artifact_paths = [ S.contains "core_dumps/*" ], - depends_on = dependsOn - } - -in - -Pipeline.build - Pipeline.Config::{ - spec = - let lintDirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall", - S.exactly "buildkite/scripts/run-snark-transaction-profiler" "sh", - S.exactly "scripts/snark_transaction_profiler" "py" - ] - - in - - JobSpec::{ - dirtyWhen = lintDirtyWhen, - path = "Test", - name = "RunSnarkProfiler", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - buildTestCmd Size.Small dependsOn - ] - } +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Profiles.Type.Standard + +let buildTestCmd + : Size -> List Command.TaggedKey.Type -> Command.Type + = \(cmd_target : Size) + -> \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + ([] : List Text) + "buildkite/scripts/run-snark-transaction-profiler.sh" + , label = "Snark Transaction Profiler" + , key = "snark-transaction-profiler" + , target = cmd_target + , docker = None Docker.Type + , artifact_paths = [ S.contains "core_dumps/*" ] + , depends_on = dependsOn + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let lintDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/RunSnarkProfiler" "dhall" + , S.exactly + "buildkite/scripts/run-snark-transaction-profiler" + "sh" + , S.exactly "scripts/snark_transaction_profiler" "py" + ] + + in JobSpec::{ + , dirtyWhen = lintDirtyWhen + , path = "Test" + , name = "RunSnarkProfiler" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = [ buildTestCmd Size.Small dependsOn ] + } diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index b30e18cafb2..f0d36a78031 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -1,75 +1,64 @@ -let Prelude = ../../External/Prelude.dhall +let PipelineTag = ../../Pipeline/Tag.dhall -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall -let PipelineTag = ../../Pipeline/Tag.dhall let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall -let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall - - -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall -let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern - -let Pipeline = ../../Pipeline/Dsl.dhall -let JobSpec = ../../Pipeline/JobSpec.dhall -let Command = ../../Command/Base.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall + let DebianVersions = ../../Constants/DebianVersions.dhall + let Profiles = ../../Constants/Profiles.dhall + let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall +let Size = ../../Command/Size.dhall let dependsOn = - DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Lightnet - # DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Standard - - -let buildTestCmd : Size -> Command.Type = \(cmd_target : Size) -> - let key = "single-node-tests" in - Command.build - Command.Config::{ - commands = RunInToolchain.runInToolchain ["DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN"] "buildkite/scripts/single-node-tests.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}", - label = "single-node-tests", - key = key, - target = cmd_target, - docker = None Docker.Type, - depends_on = dependsOn - } - -in - -Pipeline.build - Pipeline.Config::{ - spec = - let unitDirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.strictly (S.contains "Makefile"), - S.exactly "buildkite/src/Jobs/Test/SingleNodeTest" "dhall", - S.exactly "buildkite/scripts/single-node-tests" "sh" - ] - - in - - JobSpec::{ - dirtyWhen = unitDirtyWhen, - path = "Test", - name = "SingleNodeTest", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - buildTestCmd Size.XLarge - ] - } + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Profiles.Type.Lightnet + # DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Profiles.Type.Standard + +let buildTestCmd + : Size -> Command.Type + = \(cmd_target : Size) + -> let key = "single-node-tests" + + in Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + [ "DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN" ] + "buildkite/scripts/single-node-tests.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" + , label = "single-node-tests" + , key = key + , target = cmd_target + , docker = None Docker.Type + , depends_on = dependsOn + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let unitDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.strictly (S.contains "Makefile") + , S.exactly "buildkite/src/Jobs/Test/SingleNodeTest" "dhall" + , S.exactly "buildkite/scripts/single-node-tests" "sh" + ] + + in JobSpec::{ + , dirtyWhen = unitDirtyWhen + , path = "Test" + , name = "SingleNodeTest" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = [ buildTestCmd Size.XLarge ] + } diff --git a/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall b/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall index 1599007ca2b..1a8db62038f 100644 --- a/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall +++ b/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall @@ -1,54 +1,53 @@ let Command = ../../Command/Base.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -let Prelude = ../../External/Prelude.dhall let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let B = ../../External/Buildkite.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall -let buildTestCmd : Size -> Command.Type = \(cmd_target : Size) -> - Command.build - Command.Config::{ - commands = [ - Cmd.run "buildkite/scripts/terraform-test.sh" - ], - label = "Terraform: Test", - key = "terraform-network-test", - target = cmd_target, - docker = None Docker.Type, - soft_fail = Some (B/SoftFail.Boolean True) - } - -in - -Pipeline.build - Pipeline.Config::{ - spec = - let unitDirtyWhen = [ - S.strictlyStart (S.contains "src/automation/terraform"), - S.strictlyStart (S.contains "src/helm"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TerraformNetworkTest"), - S.strictlyStart (S.contains "buildkite/scripts/terraform-test.sh") - ] - - in - - JobSpec::{ - dirtyWhen = unitDirtyWhen, - path = "Test", - name = "TerraformNetworkTest", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] - }, - steps = [ - buildTestCmd Size.Large - ] - } \ No newline at end of file +let buildTestCmd + : Size -> Command.Type + = \(cmd_target : Size) + -> Command.build + Command.Config::{ + , commands = [ Cmd.run "buildkite/scripts/terraform-test.sh" ] + , label = "Terraform: Test" + , key = "terraform-network-test" + , target = cmd_target + , docker = None Docker.Type + , soft_fail = Some (B/SoftFail.Boolean True) + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let unitDirtyWhen = + [ S.strictlyStart (S.contains "src/automation/terraform") + , S.strictlyStart (S.contains "src/helm") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Test/TerraformNetworkTest") + , S.strictlyStart + (S.contains "buildkite/scripts/terraform-test.sh") + ] + + in JobSpec::{ + , dirtyWhen = unitDirtyWhen + , path = "Test" + , name = "TerraformNetworkTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] + } + , steps = [ buildTestCmd Size.Large ] + } diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 2e998eaf650..ef79e77c88d 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -1,49 +1,67 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall + let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineMode = ../../Pipeline/Mode.dhall + let PipelineTag = ../../Pipeline/Tag.dhall let TestExecutive = ../../Command/TestExecutive.dhall + let Profiles = ../../Constants/Profiles.dhall + let Dockers = ../../Constants/DockerVersions.dhall -let dependsOn = - Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "daemon-berkeley" - # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" - -in Pipeline.build Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.strictlyStart (S.contains "dockerfiles"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Command/TestExecutive"), - S.strictlyStart (S.contains "automation/terraform/modules/o1-integration"), - S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet"), - S.strictlyStart (S.contains "automation/buildkite/script/run-test-executive-cloud"), - S.strictlyStart (S.contains "automation/buildkite/script/run-test-executive-local") - ], - path = "Test", - name = "TestnetIntegrationTests", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ], - mode = PipelineMode.Type.Stable - }, - steps = [ - TestExecutive.executeCloud "peers-reliability" dependsOn, - TestExecutive.executeCloud "chain-reliability" dependsOn, - TestExecutive.executeLocal "payment" dependsOn, - TestExecutive.executeCloud "gossip-consis" dependsOn, - TestExecutive.executeCloud "block-prod-prio" dependsOn, - TestExecutive.executeCloud "medium-bootstrap" dependsOn, - TestExecutive.executeCloud "block-reward" dependsOn, - TestExecutive.executeCloud "zkapps" dependsOn, - TestExecutive.executeCloud "zkapps-timing" dependsOn, - TestExecutive.executeCloud "zkapps-nonce" dependsOn, - TestExecutive.executeCloud "verification-key" dependsOn, - TestExecutive.executeCloud "slot-end" dependsOn, - TestExecutive.executeCloud "epoch-ledger" dependsOn - ] -} +let dependsOn = + Dockers.dependsOn + Dockers.Type.Bullseye + Profiles.Type.Standard + "daemon-berkeley" + # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.strictlyStart (S.contains "dockerfiles") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Command/TestExecutive") + , S.strictlyStart + (S.contains "automation/terraform/modules/o1-integration") + , S.strictlyStart + (S.contains "automation/terraform/modules/kubernetes/testnet") + , S.strictlyStart + ( S.contains + "automation/buildkite/script/run-test-executive-cloud" + ) + , S.strictlyStart + ( S.contains + "automation/buildkite/script/run-test-executive-local" + ) + ] + , path = "Test" + , name = "TestnetIntegrationTests" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + , mode = PipelineMode.Type.Stable + } + , steps = + [ TestExecutive.executeCloud "peers-reliability" dependsOn + , TestExecutive.executeCloud "chain-reliability" dependsOn + , TestExecutive.executeLocal "payment" dependsOn + , TestExecutive.executeCloud "gossip-consis" dependsOn + , TestExecutive.executeCloud "block-prod-prio" dependsOn + , TestExecutive.executeCloud "medium-bootstrap" dependsOn + , TestExecutive.executeCloud "block-reward" dependsOn + , TestExecutive.executeCloud "zkapps" dependsOn + , TestExecutive.executeCloud "zkapps-timing" dependsOn + , TestExecutive.executeCloud "zkapps-nonce" dependsOn + , TestExecutive.executeCloud "verification-key" dependsOn + , TestExecutive.executeCloud "slot-end" dependsOn + , TestExecutive.executeCloud "epoch-ledger" dependsOn + ] + } diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index edfa0dfddc7..897dc9a01c0 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -1,33 +1,45 @@ let S = ../../Lib/SelectFiles.dhall let JobSpec = ../../Pipeline/JobSpec.dhall + let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineMode = ../../Pipeline/Mode.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let TestExecutive = ../../Command/TestExecutive.dhall + let Profiles = ../../Constants/Profiles.dhall + let Dockers = ../../Constants/DockerVersions.dhall -let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "daemon-berkeley" - # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" - -in Pipeline.build Pipeline.Config::{ - spec = - JobSpec::{ - dirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.strictlyStart (S.contains "dockerfiles"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest"), - S.strictlyStart (S.contains "buildkite/src/Jobs/Command/TestExecutive"), - S.strictlyStart (S.contains "automation/terraform/modules/o1-integration"), - S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet") - ], - path = "Test", - name = "TestnetIntegrationTestsLong", - mode = PipelineMode.Type.Stable, - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - TestExecutive.executeCloud "hard-fork" dependsOn - ] -} +let dependsOn = + Dockers.dependsOn + Dockers.Type.Bullseye + Profiles.Type.Standard + "daemon-berkeley" + # Dockers.dependsOn Dockers.Type.Bullseye Profiles.Type.Standard "archive" + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.strictlyStart (S.contains "dockerfiles") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest") + , S.strictlyStart + (S.contains "buildkite/src/Jobs/Command/TestExecutive") + , S.strictlyStart + (S.contains "automation/terraform/modules/o1-integration") + , S.strictlyStart + (S.contains "automation/terraform/modules/kubernetes/testnet") + ] + , path = "Test" + , name = "TestnetIntegrationTestsLong" + , mode = PipelineMode.Type.Stable + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = [ TestExecutive.executeCloud "hard-fork" dependsOn ] + } diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index 04bf224f400..83d7e2f6be8 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -1,60 +1,62 @@ -let Prelude = ../../External/Prelude.dhall - let Cmd = ../../Lib/Cmds.dhall + let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall -let Size = ../../Command/Size.dhall +let Size = ../../Command/Size.dhall -let dependsOn = [ - { name = "MinaArtifactBullseye", key = "build-deb-pkg" } -] - -in - -let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> Command.Type = \(release_branch : Text) -> \(cmd_target : Size) -> \(dependsOn : List Command.TaggedKey.Type) -> - Command.build - Command.Config::{ - commands = - RunInToolchain.runInToolchain ([] : List Text) "buildkite/scripts/dump-mina-type-shapes.sh" - # [ Cmd.run "gsutil cp $(git log -n 1 --format=%h --abbrev=7 --no-merges)-type_shape.txt $MINA_TYPE_SHAPE gs://mina-type-shapes"] - # RunInToolchain.runInToolchain ([] : List Text) "buildkite/scripts/version-linter.sh ${release_branch}" - , - label = "Versioned type linter", - key = "version-linter", - target = cmd_target, - docker = None Docker.Type, - depends_on = dependsOn, - artifact_paths = [ S.contains "core_dumps/*" ] - } -in - -Pipeline.build - Pipeline.Config::{ - spec = - let lintDirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/VersionLint" "dhall", - S.exactly "buildkite/scripts/version-linter" "sh" - ] - - in - - JobSpec::{ - dirtyWhen = lintDirtyWhen, - path = "Test", - name = "VersionLint", - tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - }, - steps = [ - buildTestCmd "develop" Size.Small dependsOn - ] - } \ No newline at end of file +let dependsOn = [ { name = "MinaArtifactBullseye", key = "build-deb-pkg" } ] + +let buildTestCmd + : Text -> Size -> List Command.TaggedKey.Type -> Command.Type + = \(release_branch : Text) + -> \(cmd_target : Size) + -> \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + ([] : List Text) + "buildkite/scripts/dump-mina-type-shapes.sh" + # [ Cmd.run + "gsutil cp \$(git log -n 1 --format=%h --abbrev=7 --no-merges)-type_shape.txt \$MINA_TYPE_SHAPE gs://mina-type-shapes" + ] + # RunInToolchain.runInToolchain + ([] : List Text) + "buildkite/scripts/version-linter.sh ${release_branch}" + , label = "Versioned type linter" + , key = "version-linter" + , target = cmd_target + , docker = None Docker.Type + , depends_on = dependsOn + , artifact_paths = [ S.contains "core_dumps/*" ] + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let lintDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/VersionLint" "dhall" + , S.exactly "buildkite/scripts/version-linter" "sh" + ] + + in JobSpec::{ + , dirtyWhen = lintDirtyWhen + , path = "Test" + , name = "VersionLint" + , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] + } + , steps = [ buildTestCmd "develop" Size.Small dependsOn ] + } diff --git a/buildkite/src/Jobs/Test/ZkappMetrics.dhall b/buildkite/src/Jobs/Test/ZkappMetrics.dhall index 31ec5072d7b..1a4a883b552 100644 --- a/buildkite/src/Jobs/Test/ZkappMetrics.dhall +++ b/buildkite/src/Jobs/Test/ZkappMetrics.dhall @@ -1,21 +1,23 @@ -let Prelude = ../../External/Prelude.dhall let S = ../../Lib/SelectFiles.dhall -let Cmd = ../../Lib/Cmds.dhall + let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall -let WithCargo = ../../Command/WithCargo.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall -in -Pipeline.build - Pipeline.Config:: - { spec = - JobSpec:: - { dirtyWhen = +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = [ S.strictlyStart (S.contains "buildkite/src/Jobs/Test/ZkappMetrics") , S.strictlyStart (S.contains "src") ] @@ -23,15 +25,17 @@ Pipeline.build , name = "ZkappMetrics" , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] } - , steps = - [ Command.build - Command.Config:: - { commands = RunInToolchain.runInToolchain ([] : List Text) "./buildkite/scripts/zkapp_metrics.sh" + , steps = + [ Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + ([] : List Text) + "./buildkite/scripts/zkapp_metrics.sh" , label = "Zkapp Metrics" , key = "zkapp-metrics" , target = Size.Medium , docker = None Docker.Type } - ] - } - + ] + } diff --git a/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall b/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall index 28d9de50273..1d546c9eacc 100644 --- a/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall +++ b/buildkite/src/Jobs/Test/ZkappTestToolUnitTest.dhall @@ -1,50 +1,56 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -let buildTestCmd : Text -> Text -> Size -> Command.Type = \(profile : Text) -> \(path : Text) -> \(cmd_target : Size) -> - let key = "zkapp-tool-unit-test-${profile}" in - Command.build - Command.Config::{ - commands = RunInToolchain.runInToolchain ["DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN"] "buildkite/scripts/unit-test.sh ${profile} ${path} && buildkite/scripts/upload-partial-coverage-data.sh ${key} dev", - label = "Zkapps test transaction tool unit tests", - key = key, - target = cmd_target, - docker = None Docker.Type, - artifact_paths = [ S.contains "core_dumps/*" ] - } - -in - -Pipeline.build - Pipeline.Config::{ - spec = - let unitDirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/ZkappTestToolUnitTest" "dhall", - S.exactly "buildkite/scripts/unit-test" "sh" - ] - - in - - JobSpec::{ - dirtyWhen = unitDirtyWhen, - path = "Test", - name = "ZkappTestToolUnitTest", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] - }, - steps = [ - buildTestCmd "dev" "src/app/zkapp_test_transaction" Size.Small - ] - } +let buildTestCmd + : Text -> Text -> Size -> Command.Type + = \(profile : Text) + -> \(path : Text) + -> \(cmd_target : Size) + -> let key = "zkapp-tool-unit-test-${profile}" + + in Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + [ "DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN" ] + "buildkite/scripts/unit-test.sh ${profile} ${path} && buildkite/scripts/upload-partial-coverage-data.sh ${key} dev" + , label = "Zkapps test transaction tool unit tests" + , key = key + , target = cmd_target + , docker = None Docker.Type + , artifact_paths = [ S.contains "core_dumps/*" ] + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let unitDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly + "buildkite/src/Jobs/Test/ZkappTestToolUnitTest" + "dhall" + , S.exactly "buildkite/scripts/unit-test" "sh" + ] + + in JobSpec::{ + , dirtyWhen = unitDirtyWhen + , path = "Test" + , name = "ZkappTestToolUnitTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] + } + , steps = + [ buildTestCmd "dev" "src/app/zkapp_test_transaction" Size.Small ] + } diff --git a/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall b/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall index f5476fe2b55..0ccea3808b1 100644 --- a/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall +++ b/buildkite/src/Jobs/Test/ZkappsExamplesTest.dhall @@ -1,52 +1,53 @@ -let Prelude = ../../External/Prelude.dhall - -let Cmd = ../../Lib/Cmds.dhall let S = ../../Lib/SelectFiles.dhall -let D = S.PathPattern let Pipeline = ../../Pipeline/Dsl.dhall + let PipelineTag = ../../Pipeline/Tag.dhall + let JobSpec = ../../Pipeline/JobSpec.dhall let Command = ../../Command/Base.dhall + let RunInToolchain = ../../Command/RunInToolchain.dhall + let Docker = ../../Command/Docker/Type.dhall + let Size = ../../Command/Size.dhall -let buildTestCmd : Text -> Size -> Command.Type = \(profile : Text) -> \(cmd_target : Size) -> - let command_key = "zkapps-examples-unit-test-${profile}" - in - Command.build - Command.Config::{ - commands = RunInToolchain.runInToolchain ["DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN"] "buildkite/scripts/zkapps-examples-unit-tests.sh ${profile} && buildkite/scripts/upload-partial-coverage-data.sh ${command_key} dev", - label = "${profile} zkApps examples tests", - key = command_key, - target = cmd_target, - docker = None Docker.Type, - artifact_paths = [ S.contains "core_dumps/*" ] - } - -in - -Pipeline.build - Pipeline.Config::{ - spec = - let unitDirtyWhen = [ - S.strictlyStart (S.contains "src"), - S.exactly "buildkite/src/Jobs/Test/ZkappsExamplesTest" "dhall", - S.exactly "scripts/link-coredumps" "sh", - S.exactly "buildkite/scripts/zkapps-examples-unit-tests" "sh" - ] - - in - - JobSpec::{ - dirtyWhen = unitDirtyWhen, - path = "Test", - name = "ZkappsExamplesTest", - tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] - }, - steps = [ - buildTestCmd "dev" Size.XLarge - ] - } +let buildTestCmd + : Text -> Size -> Command.Type + = \(profile : Text) + -> \(cmd_target : Size) + -> let command_key = "zkapps-examples-unit-test-${profile}" + + in Command.build + Command.Config::{ + , commands = + RunInToolchain.runInToolchain + [ "DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN" ] + "buildkite/scripts/zkapps-examples-unit-tests.sh ${profile} && buildkite/scripts/upload-partial-coverage-data.sh ${command_key} dev" + , label = "${profile} zkApps examples tests" + , key = command_key + , target = cmd_target + , docker = None Docker.Type + , artifact_paths = [ S.contains "core_dumps/*" ] + } + +in Pipeline.build + Pipeline.Config::{ + , spec = + let unitDirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "buildkite/src/Jobs/Test/ZkappsExamplesTest" "dhall" + , S.exactly "scripts/link-coredumps" "sh" + , S.exactly "buildkite/scripts/zkapps-examples-unit-tests" "sh" + ] + + in JobSpec::{ + , dirtyWhen = unitDirtyWhen + , path = "Test" + , name = "ZkappsExamplesTest" + , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ] + } + , steps = [ buildTestCmd "dev" Size.XLarge ] + } diff --git a/buildkite/src/Lib/Cmds.dhall b/buildkite/src/Lib/Cmds.dhall index 1e500fcd0bd..49e2d8fc432 100644 --- a/buildkite/src/Lib/Cmds.dhall +++ b/buildkite/src/Lib/Cmds.dhall @@ -1,149 +1,161 @@ -- A DSL for manipulating commands and their execution environments let Prelude = ../External/Prelude.dhall + let P = Prelude -let List/map = P.List.map -let List/null = P.List.null -let Optional/toList = P.Optional.toList -let Optional/default = P.Optional.default + let Optional/map = P.Optional.map -let List/concatMap = P.List.concatMap -let List/concat = P.List.concat + let Text/concatSep = P.Text.concatSep + let Text/concatMap = P.Text.concatMap --- abstract out defaultEnv so tests are less verbose -let module = \(environment : List Text) -> - - let Docker = { - Type = { - image : Text, - extraEnv : List Text, - privileged : Bool - }, - default = { - extraEnv = ([] : List Text), - privileged = False - } - } - - let Cmd = { line: Text, readable: Optional Text } - let run : Text -> Cmd = - \(script: Text) -> { line = script, readable = Some script } - - let chain : List Text -> Cmd = - \(chainOfCommands: List Text) -> - run (Text/concatSep " && " chainOfCommands) - - let quietly : Text -> Cmd = - \(script: Text) -> { line = script, readable = None Text } - let true : Cmd = quietly "true" - let false : Cmd = quietly "false" - - let inDocker : Docker.Type -> Cmd -> Cmd = - \(docker : Docker.Type) -> - \(inner : Cmd) -> - let envVars = - Text/concatMap - Text - (\(var : Text) -> " --env ${var}") - (docker.extraEnv # environment) - let outerDir : Text = - "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" - let sharedDir : Text = "/var/buildkite/shared" - in - { line = "docker run -it --rm --entrypoint /bin/sh --init --volume ${sharedDir}:/shared --volume ${outerDir}:/workdir --workdir /workdir${envVars}${if docker.privileged then " --privileged" else ""} ${docker.image} -c '${inner.line}'" - , readable = Optional/map Text Text (\(readable : Text) -> "Docker@${docker.image} ( ${readable} )") inner.readable - } - - let runInDocker : Docker.Type -> Text -> Cmd = - \(docker : Docker.Type) -> - \(script : Text) -> - inDocker docker (run script) - - let CacheSetupCmd = { - Type = { - -- run your command to create data (only on miss) - create : Cmd, - -- package data before an upload to gcloud (only on miss) - package : Cmd - }, - default = {=} - } - - let format : Cmd -> Text = - \(cmd : Cmd) -> cmd.line - - -- Loads through cache, innards with docker, buildkite-agent interactions outside, continues in docker after hit or miss with continuation - let cacheThrough : Docker.Type -> Text -> CacheSetupCmd.Type -> Cmd = - \(docker : Docker.Type) -> - \(cachePath : Text) -> - \(cmd : CacheSetupCmd.Type) -> - let missScript = - ( format cmd.create ) ++ " && " ++ - ( format cmd.package ) - let missCmd = - runInDocker docker missScript - in - { line = "./buildkite/scripts/cache-through.sh ${cachePath} \"${format missCmd}\"" - , readable = - Optional/map - Text - Text - (\(readable : Text) -> "Cache@${cachePath} ( onMiss = ${readable} )") - missCmd.readable - } - in - - { Type = Cmd - , Docker = Docker - , CacheSetupCmd = CacheSetupCmd - , quietly = quietly - , run = run - , chain = chain - , true = true - , false = false - , runInDocker = runInDocker - , inDocker = inDocker - , cacheThrough = cacheThrough - , format = format - } +let module = + \(environment : List Text) + -> let Docker = + { Type = + { image : Text, extraEnv : List Text, privileged : Bool } + , default = { extraEnv = [] : List Text, privileged = False } + } + + let Cmd = { line : Text, readable : Optional Text } + + let run + : Text -> Cmd + = \(script : Text) -> { line = script, readable = Some script } + + let chain + : List Text -> Cmd + = \(chainOfCommands : List Text) + -> run (Text/concatSep " && " chainOfCommands) + + let quietly + : Text -> Cmd + = \(script : Text) -> { line = script, readable = None Text } + + let true + : Cmd + = quietly "true" + + let false + : Cmd + = quietly "false" + + let inDocker + : Docker.Type -> Cmd -> Cmd + = \(docker : Docker.Type) + -> \(inner : Cmd) + -> let envVars = + Text/concatMap + Text + (\(var : Text) -> " --env ${var}") + (docker.extraEnv # environment) + + let outerDir + : Text + = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" + + let sharedDir + : Text + = "/var/buildkite/shared" + + in { line = + "docker run -it --rm --entrypoint /bin/sh --init --volume ${sharedDir}:/shared --volume ${outerDir}:/workdir --workdir /workdir${envVars}${ if docker.privileged + + then " --privileged" + + else ""} ${docker.image} -c '${inner.line}'" + , readable = + Optional/map + Text + Text + ( \(readable : Text) + -> "Docker@${docker.image} ( ${readable} )" + ) + inner.readable + } + + let runInDocker + : Docker.Type -> Text -> Cmd + = \(docker : Docker.Type) + -> \(script : Text) + -> inDocker docker (run script) + + let CacheSetupCmd = + { Type = { create : Cmd, package : Cmd }, default = {=} } + + let format + : Cmd -> Text + = \(cmd : Cmd) -> cmd.line + + let cacheThrough + : Docker.Type -> Text -> CacheSetupCmd.Type -> Cmd + = \(docker : Docker.Type) + -> \(cachePath : Text) + -> \(cmd : CacheSetupCmd.Type) + -> let missScript = + format cmd.create ++ " && " ++ format cmd.package + + let missCmd = runInDocker docker missScript + + in { line = + "./buildkite/scripts/cache-through.sh ${cachePath} \"${format + missCmd}\"" + , readable = + Optional/map + Text + Text + ( \(readable : Text) + -> "Cache@${cachePath} ( onMiss = ${readable} )" + ) + missCmd.readable + } + + in { Type = Cmd + , Docker = Docker + , CacheSetupCmd = CacheSetupCmd + , quietly = quietly + , run = run + , chain = chain + , true = true + , false = false + , runInDocker = runInDocker + , inDocker = inDocker + , cacheThrough = cacheThrough + , format = format + } let tests = - let M = module ["TEST"] in - - let dockerExample = assert : - { line = -"docker run -it --rm --entrypoint /bin/sh --init --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello'" - , readable = - Some "Docker@foo/bar:tag ( echo hello )" - } - === - M.inDocker - M.Docker::{ - image = "foo/bar:tag", - extraEnv = [ "ENV1", "ENV2" ] - } - ( M.run "echo hello" ) - - let cacheExample = assert : -'' - ./buildkite/scripts/cache-through.sh data.tar "docker run -it --rm --entrypoint /bin/sh --init --volume /var/buildkite/shared:/shared --volume \$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello > /tmp/data/foo.txt && tar cvf data.tar /tmp/data'"'' - === - M.format ( - M.cacheThrough - M.Docker::{ - image = "foo/bar:tag", - extraEnv = [ "ENV1", "ENV2" ] - } - "data.tar" - M.CacheSetupCmd::{ - create = M.run "echo hello > /tmp/data/foo.txt", - package = M.run "tar cvf data.tar /tmp/data" - } - ) - in - "" - -in -module ../Constants/ContainerEnvVars.dhall + let M = module [ "TEST" ] + + let dockerExample = + assert + : { line = + "docker run -it --rm --entrypoint /bin/sh --init --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello'" + , readable = Some "Docker@foo/bar:tag ( echo hello )" + } + === M.inDocker + M.Docker::{ + , image = "foo/bar:tag" + , extraEnv = [ "ENV1", "ENV2" ] + } + (M.run "echo hello") + + let cacheExample = + assert + : "./buildkite/scripts/cache-through.sh data.tar \"docker run -it --rm --entrypoint /bin/sh --init --volume /var/buildkite/shared:/shared --volume \\\$BUILDKITE_BUILD_CHECKOUT_PATH:/workdir --workdir /workdir --env ENV1 --env ENV2 --env TEST foo/bar:tag -c 'echo hello > /tmp/data/foo.txt && tar cvf data.tar /tmp/data'\"" + === M.format + ( M.cacheThrough + M.Docker::{ + , image = "foo/bar:tag" + , extraEnv = [ "ENV1", "ENV2" ] + } + "data.tar" + M.CacheSetupCmd::{ + , create = M.run "echo hello > /tmp/data/foo.txt" + , package = M.run "tar cvf data.tar /tmp/data" + } + ) + + in "" +in module ../Constants/ContainerEnvVars.dhall diff --git a/buildkite/src/Lib/Decorate.dhall b/buildkite/src/Lib/Decorate.dhall index 9a97b432c42..e4d29154dc3 100644 --- a/buildkite/src/Lib/Decorate.dhall +++ b/buildkite/src/Lib/Decorate.dhall @@ -6,29 +6,24 @@ let Optional/fold_ = Prelude.Optional.fold let Cmd = ./Cmds.dhall +let decorate + : Cmd.Type -> List Text + = \(run : Cmd.Type) + -> Optional/fold_ + Text + run.readable + (List Text) + ( \(readable : Text) + -> [ "echo \"--\"" + , "echo \"-- Running: ${readable} --\"" + , "echo \"--\"" + , run.line + ] + ) + [ run.line ] -let decorate : Cmd.Type -> List Text = \(run : Cmd.Type) -> - Optional/fold_ - Text - run.readable - (List Text) - (\(readable : Text) -> - [ - "echo \"--\"", - "echo \"-- Running: ${readable} --\"", - "echo \"--\"", - run.line - ]) - [ run.line ] -in - -{ - decorate = decorate, - decorateAll = \(commands : List Cmd.Type) -> - Prelude.List.concatMap - Cmd.Type - Text - decorate - commands -} - +in { decorate = decorate + , decorateAll = + \(commands : List Cmd.Type) + -> Prelude.List.concatMap Cmd.Type Text decorate commands + } diff --git a/buildkite/src/Lib/SelectFiles.dhall b/buildkite/src/Lib/SelectFiles.dhall index 9a87a0ab2c0..ef5d7370221 100644 --- a/buildkite/src/Lib/SelectFiles.dhall +++ b/buildkite/src/Lib/SelectFiles.dhall @@ -1,131 +1,165 @@ -- A DSL for representing file globbing; compiles into egrep-compatible regexp let Prelude = ../External/Prelude.dhall + let P = Prelude -let PathPattern = < Lit : Text | Any > -- Any == * -let FilePattern = { - Type = { - dir: Optional (List PathPattern), - exts: Optional (List Text), - strictStart: Bool, - -- strictEnd implicitly set if exts is present - strictEnd: Bool - }, - default = { - dir = None (List PathPattern), - exts = None (List Text), - strictStart = False, - strictEnd = False - } -} - - -let strictlyStart : FilePattern.Type -> FilePattern.Type = - \(pattern : FilePattern.Type) -> pattern // { strictStart = True } - -let strictlyEnd : FilePattern.Type -> FilePattern.Type = - \(pattern : FilePattern.Type) -> pattern // { strictEnd = True } - -let strictly : FilePattern.Type -> FilePattern.Type = - \(pattern : FilePattern.Type) -> strictlyEnd (strictlyStart pattern) - -let everything : FilePattern.Type = FilePattern.default - -let contains : Text -> FilePattern.Type = - \(s : Text) -> - FilePattern::{ dir = Some [PathPattern.Lit s] } - -let exactly : Text -> Text -> FilePattern.Type = - \(dir : Text) -> \(ext : Text) -> - FilePattern::{ - dir = Some [PathPattern.Lit dir], - exts = Some [ext], - strictStart = True, - strictEnd = True - } - -let exactly_noext : Text -> FilePattern.Type = - \(dir : Text) -> - FilePattern::{ - dir = Some [PathPattern.Lit dir], - strictStart = True, - strictEnd = True - } - --- compile things -let any : List Text -> Text = - P.Text.concatSep "|" - -let compileExt : Text -> Text = - \(ext : Text) -> "\\." ++ ext - -let compileExts : List Text -> Text = - \(exts : List Text) -> - let exts_ = P.List.map Text Text compileExt exts in - "(" ++ (any exts_) ++ ")" - -let compilePath : PathPattern -> Text = - \(path : PathPattern) -> - merge { - Lit = \(text : Text) -> text, - Any = ".*" - } path - -let compilePaths : List PathPattern -> Text = - \(paths : List PathPattern) -> - let paths_ = P.List.map PathPattern Text compilePath paths in - P.Text.concat paths_ - -let compileFile : FilePattern.Type -> Text = - \(pattern : FilePattern.Type) -> - let dirPart = - P.Optional.fold (List PathPattern) pattern.dir Text compilePaths ".*" - let extPart = - P.Optional.fold (List Text) pattern.exts Text compileExts "" - let startPart = - if pattern.strictStart then "^" else "" - let endPart = - if pattern.strictEnd || (P.Bool.not (P.Optional.null (List Text) pattern.exts)) then - "\\\$" - else - "" - in - startPart ++ dirPart ++ extPart ++ endPart - -let compile : List FilePattern.Type -> Text = - \(patterns : List FilePattern.Type) -> - let patterns_ = P.List.map FilePattern.Type Text compileFile patterns in - any patterns_ - - -let everythingExample = assert : compile [everything] === ".*" - -let exactlyExample = assert : compile [exactly "scripts/compile" "py"] === "^scripts/compile(\\.py)\\\$" - -let containsExample = assert : compile [contains "buildkite/Makefile"] === "buildkite/Makefile" - -let strictContainsExample = assert : compile [strictly (contains "buildkite/Makefile")] === "^buildkite/Makefile\\\$" +let PathPattern = < Lit : Text | Any > + +let FilePattern = + { Type = + { dir : Optional (List PathPattern) + , exts : Optional (List Text) + , strictStart : Bool + , strictEnd : Bool + } + , default = + { dir = None (List PathPattern) + , exts = None (List Text) + , strictStart = False + , strictEnd = False + } + } + +let strictlyStart + : FilePattern.Type -> FilePattern.Type + = \(pattern : FilePattern.Type) -> pattern // { strictStart = True } + +let strictlyEnd + : FilePattern.Type -> FilePattern.Type + = \(pattern : FilePattern.Type) -> pattern // { strictEnd = True } + +let strictly + : FilePattern.Type -> FilePattern.Type + = \(pattern : FilePattern.Type) -> strictlyEnd (strictlyStart pattern) + +let everything + : FilePattern.Type + = FilePattern.default + +let contains + : Text -> FilePattern.Type + = \(s : Text) -> FilePattern::{ dir = Some [ PathPattern.Lit s ] } + +let exactly + : Text -> Text -> FilePattern.Type + = \(dir : Text) + -> \(ext : Text) + -> FilePattern::{ + , dir = Some [ PathPattern.Lit dir ] + , exts = Some [ ext ] + , strictStart = True + , strictEnd = True + } + +let exactly_noext + : Text -> FilePattern.Type + = \(dir : Text) + -> FilePattern::{ + , dir = Some [ PathPattern.Lit dir ] + , strictStart = True + , strictEnd = True + } + +let any + : List Text -> Text + = P.Text.concatSep "|" + +let compileExt + : Text -> Text + = \(ext : Text) -> "\\." ++ ext + +let compileExts + : List Text -> Text + = \(exts : List Text) + -> let exts_ = P.List.map Text Text compileExt exts + + in "(" ++ any exts_ ++ ")" + +let compilePath + : PathPattern -> Text + = \(path : PathPattern) + -> merge { Lit = \(text : Text) -> text, Any = ".*" } path + +let compilePaths + : List PathPattern -> Text + = \(paths : List PathPattern) + -> let paths_ = P.List.map PathPattern Text compilePath paths + + in P.Text.concat paths_ + +let compileFile + : FilePattern.Type -> Text + = \(pattern : FilePattern.Type) + -> let dirPart = + P.Optional.fold + (List PathPattern) + pattern.dir + Text + compilePaths + ".*" + + let extPart = + P.Optional.fold (List Text) pattern.exts Text compileExts "" + + let startPart = if pattern.strictStart then "^" else "" + + let endPart = + if pattern.strictEnd + || P.Bool.not + (P.Optional.null (List Text) pattern.exts) + + then "\\\$" + + else "" + + in startPart ++ dirPart ++ extPart ++ endPart + +let compile + : List FilePattern.Type -> Text + = \(patterns : List FilePattern.Type) + -> let patterns_ = P.List.map FilePattern.Type Text compileFile patterns + + in any patterns_ + +let everythingExample = assert : compile [ everything ] === ".*" + +let exactlyExample = + assert + : compile [ exactly "scripts/compile" "py" ] + === "^scripts/compile(\\.py)\\\$" + +let containsExample = + assert + : compile [ contains "buildkite/Makefile" ] === "buildkite/Makefile" + +let strictContainsExample = + assert + : compile [ strictly (contains "buildkite/Makefile") ] + === "^buildkite/Makefile\\\$" let D = PathPattern -let realisticExample = assert : compile [ - strictly (FilePattern::{ - dir = Some [D.Lit "buildkite/", D.Any], - exts = Some ["dhall"] - }), - strictly (contains "buildkite/Makefile"), - exactly "buildkite/scripts/generate-jobs" "sh" -] === "^buildkite/.*(\\.dhall)\\\$|^buildkite/Makefile\\\$|^buildkite/scripts/generate-jobs(\\.sh)\\\$" - - -in -{ compile = compile, - everything = everything, - contains = contains, - exactly = exactly, - exactly_noext = exactly_noext, - strictlyStart = strictlyStart, - strictlyEnd = strictlyEnd, - strictly = strictly, - -- D for DSL - PathPattern = PathPattern -} // FilePattern + +let realisticExample = + assert + : compile + [ strictly + FilePattern::{ + , dir = Some [ D.Lit "buildkite/", D.Any ] + , exts = Some [ "dhall" ] + } + , strictly (contains "buildkite/Makefile") + , exactly "buildkite/scripts/generate-jobs" "sh" + ] + === "^buildkite/.*(\\.dhall)\\\$|^buildkite/Makefile\\\$|^buildkite/scripts/generate-jobs(\\.sh)\\\$" + +in { compile = compile + , everything = everything + , contains = contains + , exactly = exactly + , exactly_noext = exactly_noext + , strictlyStart = strictlyStart + , strictlyEnd = strictlyEnd + , strictly = strictly + , PathPattern = PathPattern + } + // FilePattern diff --git a/buildkite/src/Monorepo.dhall b/buildkite/src/Monorepo.dhall index a31778c96c7..7106d8358e5 100644 --- a/buildkite/src/Monorepo.dhall +++ b/buildkite/src/Monorepo.dhall @@ -1,115 +1,137 @@ let Prelude = ./External/Prelude.dhall -let List/map = Prelude.List.map -let List/filter = Prelude.List.filter +let List/map = Prelude.List.map let SelectFiles = ./Lib/SelectFiles.dhall + let Cmd = ./Lib/Cmds.dhall let Command = ./Command/Base.dhall + let Docker = ./Command/Docker/Type.dhall + let JobSpec = ./Pipeline/JobSpec.dhall + let Pipeline = ./Pipeline/Dsl.dhall + let PipelineMode = ./Pipeline/Mode.dhall + let PipelineFilter = ./Pipeline/Filter.dhall + let PipelineTag = ./Pipeline/Tag.dhall + let Size = ./Command/Size.dhall + let triggerCommand = ./Pipeline/TriggerCommand.dhall -let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" -let include_tags = env:BUILDKITE_PIPELINE_TAGS_INCL as Text ? "Fast" -let exclude_tags = env:BUILDKITE_PIPELINE_TAGS_EXCL as Text ? "" - -let jobs : List JobSpec.Type = - List/map - Pipeline.CompoundType - JobSpec.Type - (\(composite: Pipeline.CompoundType) -> composite.spec) - ./gen/Jobs.dhall - -let prefixCommands = [ - Cmd.run "git config --global http.sslCAInfo /etc/ssl/certs/ca-bundle.crt", -- Tell git where to find certs for https connections - Cmd.run "./buildkite/scripts/refresh_code.sh", - Cmd.run "./buildkite/scripts/generate-diff.sh > _computed_diff.txt" -] - - --- Run a job if we touched a dirty path -let commands: PipelineFilter.Type -> PipelineMode.Type -> List Cmd.Type = \(filter: PipelineFilter.Type) -> \(mode: PipelineMode.Type) -> - List/map - JobSpec.Type - Cmd.Type - (\(job: JobSpec.Type) -> - let jobMode = PipelineMode.capitalName job.mode - let targetMode = PipelineMode.capitalName mode - let targetTags = PipelineFilter.tags filter - let filter = PipelineFilter.show filter - let isIncluded = Prelude.Bool.show (PipelineTag.contains job.tags targetTags) - let dirtyWhen = SelectFiles.compile job.dirtyWhen - let trigger = triggerCommand "src/Jobs/${job.path}/${job.name}.dhall" - let pipelineHandlers = { - PullRequest = '' - if [ "${targetMode}" == "PullRequest" ]; then - if [ "${isIncluded}" == "True" ]; then - if (cat _computed_diff.txt | egrep -q '${dirtyWhen}'); then - echo "Triggering ${job.name} for reason:" - cat _computed_diff.txt | egrep '${dirtyWhen}' - ${Cmd.format trigger} - fi - else - echo "Skipping ${job.name} because this is a ${filter} stage" - fi - elif [ "${targetMode}" == "Stable" ]; then - if [ "${isIncluded}" == "True" ]; then - echo "Triggering ${job.name} because this is a stable buildkite run" - ${Cmd.format trigger} - else - echo "Skipping ${job.name} because this is a ${filter} stage" - fi - else - echo "Skipping ${job.name} because this is a stable buildkite run" - fi - '', - Stable = '' - if [ "${targetMode}" == "Stable" ]; then - if [ "${isIncluded}" == "True" ]; then - echo "Triggering ${job.name} because this is a stable buildkite run" - ${Cmd.format trigger} - else - echo "Skipping ${job.name} because this is a ${filter} stage" - fi - else - echo "Skipping ${job.name} because this is a PR buildkite run" - fi - '' - } - in Cmd.quietly (merge pipelineHandlers job.mode) - ) - jobs - -in - -(\(args : { filter : PipelineFilter.Type, mode: PipelineMode.Type }) -> - let pipelineType = Pipeline.build Pipeline.Config::{ - spec = JobSpec::{ - name = "monorepo-triage-${PipelineFilter.show args.filter}", - -- TODO: Clean up this code so we don't need an unused dirtyWhen here - dirtyWhen = [ SelectFiles.everything ] - }, - steps = [ - Command.build - Command.Config::{ - commands = prefixCommands # (commands args.filter args.mode), - label = "Monorepo triage ${PipelineFilter.show args.filter}", - key = "cmds-${PipelineFilter.show args.filter}", - target = Size.Small, - docker = Some Docker::{ - image = (./Constants/ContainerImages.dhall).toolchainBase, - environment = ["BUILDKITE_AGENT_ACCESS_TOKEN", "BUILDKITE_INCREMENTAL"] - } - } - ] - } - in pipelineType.pipeline -) +let jobs + : List JobSpec.Type + = List/map + Pipeline.CompoundType + JobSpec.Type + (\(composite : Pipeline.CompoundType) -> composite.spec) + ./gen/Jobs.dhall + +let prefixCommands = + [ Cmd.run + "git config --global http.sslCAInfo /etc/ssl/certs/ca-bundle.crt" + , Cmd.run "./buildkite/scripts/refresh_code.sh" + , Cmd.run "./buildkite/scripts/generate-diff.sh > _computed_diff.txt" + ] + +let commands + : PipelineFilter.Type -> PipelineMode.Type -> List Cmd.Type + = \(filter : PipelineFilter.Type) + -> \(mode : PipelineMode.Type) + -> List/map + JobSpec.Type + Cmd.Type + ( \(job : JobSpec.Type) + -> let targetMode = PipelineMode.capitalName mode + + let targetTags = PipelineFilter.tags filter + + let filter = PipelineFilter.show filter + + let isIncluded = + Prelude.Bool.show + (PipelineTag.contains job.tags targetTags) + + let dirtyWhen = SelectFiles.compile job.dirtyWhen + + let trigger = + triggerCommand "src/Jobs/${job.path}/${job.name}.dhall" + + let pipelineHandlers = + { PullRequest = + '' + if [ "${targetMode}" == "PullRequest" ]; then + if [ "${isIncluded}" == "True" ]; then + if (cat _computed_diff.txt | egrep -q '${dirtyWhen}'); then + echo "Triggering ${job.name} for reason:" + cat _computed_diff.txt | egrep '${dirtyWhen}' + ${Cmd.format trigger} + fi + else + echo "Skipping ${job.name} because this is a ${filter} stage" + fi + elif [ "${targetMode}" == "Stable" ]; then + if [ "${isIncluded}" == "True" ]; then + echo "Triggering ${job.name} because this is a stable buildkite run" + ${Cmd.format trigger} + else + echo "Skipping ${job.name} because this is a ${filter} stage" + fi + else + echo "Skipping ${job.name} because this is a stable buildkite run" + fi + '' + , Stable = + '' + if [ "${targetMode}" == "Stable" ]; then + if [ "${isIncluded}" == "True" ]; then + echo "Triggering ${job.name} because this is a stable buildkite run" + ${Cmd.format trigger} + else + echo "Skipping ${job.name} because this is a ${filter} stage" + fi + else + echo "Skipping ${job.name} because this is a PR buildkite run" + fi + '' + } + + in Cmd.quietly (merge pipelineHandlers job.mode) + ) + jobs + +in \(args : { filter : PipelineFilter.Type, mode : PipelineMode.Type }) + -> let pipelineType = + Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , name = "monorepo-triage-${PipelineFilter.show args.filter}" + , dirtyWhen = [ SelectFiles.everything ] + } + , steps = + [ Command.build + Command.Config::{ + , commands = + prefixCommands # commands args.filter args.mode + , label = + "Monorepo triage ${PipelineFilter.show args.filter}" + , key = "cmds-${PipelineFilter.show args.filter}" + , target = Size.Small + , docker = Some Docker::{ + , image = + (./Constants/ContainerImages.dhall).toolchainBase + , environment = + [ "BUILDKITE_AGENT_ACCESS_TOKEN" + , "BUILDKITE_INCREMENTAL" + ] + } + } + ] + } + in pipelineType.pipeline diff --git a/buildkite/src/Pipeline/Dsl.dhall b/buildkite/src/Pipeline/Dsl.dhall index ea92cb6d24c..ec8ea0e3b71 100644 --- a/buildkite/src/Pipeline/Dsl.dhall +++ b/buildkite/src/Pipeline/Dsl.dhall @@ -4,36 +4,49 @@ -- monorepo triage step determines it be necessary. let Prelude = ../External/Prelude.dhall + let List/map = Prelude.List.map let Command = ../Command/Base.dhall + let JobSpec = ./JobSpec.dhall + let Pipeline/Type = ./Type.dhall --- We build a pipeline out of a spec and the commands in a step -let Config = { - Type = { - spec: JobSpec.Type, - -- TODO: Union type with block steps - steps: List Command.Type - }, - default = {=} -} - -let CompoundType = { pipeline: Pipeline/Type, spec: JobSpec.Type } - -let build : Config.Type -> CompoundType = \(c : Config.Type) -> - let name = c.spec.name - let buildCommand = \(c : Command.Type) -> - c // { key = - let key = - Prelude.Optional.fold Text c.key Text (\(k : Text) -> k) "" - in - Some "_${name}-${key}" - } - in - { pipeline = { steps = List/map Command.Type Command.Type buildCommand c.steps }, - spec = c.spec - } +let Config = + { Type = { spec : JobSpec.Type, steps : List Command.Type } + , default = {=} + } + +let CompoundType = { pipeline : Pipeline/Type, spec : JobSpec.Type } -in {Config = Config, build = build, Type = Pipeline/Type, CompoundType = CompoundType } +let build + : Config.Type -> CompoundType + = \(c : Config.Type) + -> let name = c.spec.name + + let buildCommand = + \(c : Command.Type) + -> c + // { key = + let key = + Prelude.Optional.fold + Text + c.key + Text + (\(k : Text) -> k) + "" + + in Some "_${name}-${key}" + } + + in { pipeline.steps = + List/map Command.Type Command.Type buildCommand c.steps + , spec = c.spec + } + +in { Config = Config + , build = build + , Type = Pipeline/Type + , CompoundType = CompoundType + } diff --git a/buildkite/src/Pipeline/Filter.dhall b/buildkite/src/Pipeline/Filter.dhall index 9f1720cc583..8603f5e8f0f 100644 --- a/buildkite/src/Pipeline/Filter.dhall +++ b/buildkite/src/Pipeline/Filter.dhall @@ -1,39 +1,54 @@ --- Tag defines pipeline +-- Tag defines pipeline -- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags --- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline -let Prelude = ../External/Prelude.dhall let Tag = ./Tag.dhall -let Filter : Type = < FastOnly | Long | LongAndVeryLong | TearDownOnly | ToolchainsOnly | AllTests | Release | HardforkPackageGeneration > +let Filter + : Type + = < FastOnly + | Long + | LongAndVeryLong + | TearDownOnly + | ToolchainsOnly + | AllTests + | Release + | HardforkPackageGeneration + > -let tags: Filter -> List Tag.Type = \(filter: Filter) -> - merge { - FastOnly = [ Tag.Type.Fast ] - , LongAndVeryLong = [ Tag.Type.Long, Tag.Type.VeryLong ] - , Long = [ Tag.Type.Long ] - , TearDownOnly = [ Tag.Type.TearDown ] - , ToolchainsOnly = [ Tag.Type.Toolchain ] - , AllTests = [ Tag.Type.Lint, Tag.Type.Release, Tag.Type.Test, Tag.Type.Hardfork ] - , Release = [ Tag.Type.Release ] - , HardforkPackageGeneration = [ Tag.Type.Hardfork ] - } filter +let tags + : Filter -> List Tag.Type + = \(filter : Filter) + -> merge + { FastOnly = [ Tag.Type.Fast ] + , LongAndVeryLong = [ Tag.Type.Long, Tag.Type.VeryLong ] + , Long = [ Tag.Type.Long ] + , TearDownOnly = [ Tag.Type.TearDown ] + , ToolchainsOnly = [ Tag.Type.Toolchain ] + , AllTests = + [ Tag.Type.Lint + , Tag.Type.Release + , Tag.Type.Test + , Tag.Type.Hardfork + ] + , Release = [ Tag.Type.Release ] + , HardforkPackageGeneration = [ Tag.Type.Hardfork ] + } + filter -let show: Filter -> Text = \(filter: Filter) -> - merge { - FastOnly = "FastOnly" - , LongAndVeryLong = "LongAndVeryLong" - , Long = "Long" - , ToolchainsOnly = "Toolchain" - , TearDownOnly = "TearDownOnly" - , AllTests = "AllTests" - , Release = "Release" - , HardforkPackageGeneration = "HardforkPackageGeneration" - } filter +let show + : Filter -> Text + = \(filter : Filter) + -> merge + { FastOnly = "FastOnly" + , LongAndVeryLong = "LongAndVeryLong" + , Long = "Long" + , ToolchainsOnly = "Toolchain" + , TearDownOnly = "TearDownOnly" + , AllTests = "AllTests" + , Release = "Release" + , HardforkPackageGeneration = "HardforkPackageGeneration" + } + filter -in -{ - Type = Filter, - tags = tags, - show = show -} +in { Type = Filter, tags = tags, show = show } diff --git a/buildkite/src/Pipeline/JobSpec.dhall b/buildkite/src/Pipeline/JobSpec.dhall index ea0373a0c2d..5c4299754d5 100644 --- a/buildkite/src/Pipeline/JobSpec.dhall +++ b/buildkite/src/Pipeline/JobSpec.dhall @@ -1,22 +1,19 @@ let SelectFiles = ../Lib/SelectFiles.dhall + let PipelineMode = ./Mode.dhall -let PipelineTag = ./Tag.dhall -in +let PipelineTag = ./Tag.dhall --- Defines info used for selecting a job to run --- path is relative to `src/jobs/` -{ - Type = { - path: Text, - name: Text, - mode: PipelineMode.Type, - tags: List PipelineTag.Type, - dirtyWhen: List SelectFiles.Type - }, - default = { - path = ".", - mode = PipelineMode.Type.PullRequest, - tags = [ PipelineTag.Type.Fast ] - } -} +in { Type = + { path : Text + , name : Text + , mode : PipelineMode.Type + , tags : List PipelineTag.Type + , dirtyWhen : List SelectFiles.Type + } + , default = + { path = "." + , mode = PipelineMode.Type.PullRequest + , tags = [ PipelineTag.Type.Fast ] + } + } diff --git a/buildkite/src/Pipeline/Mode.dhall b/buildkite/src/Pipeline/Mode.dhall index a224dc50b8a..374457c082d 100644 --- a/buildkite/src/Pipeline/Mode.dhall +++ b/buildkite/src/Pipeline/Mode.dhall @@ -3,18 +3,10 @@ -- Goal of the pipeline can be either quick feedback for CI changes -- or Nightly run which supposed to be run only on stable changes. -let Prelude = ../External/Prelude.dhall +let Mode = < PullRequest | Stable > -let Mode = < PullRequest | Stable > +let capitalName = + \(pipelineMode : Mode) + -> merge { PullRequest = "PullRequest", Stable = "Stable" } pipelineMode -let capitalName = \(pipelineMode : Mode) -> - merge { - PullRequest = "PullRequest" - , Stable = "Stable" - } pipelineMode - -in -{ - Type = Mode, - capitalName = capitalName -} +in { Type = Mode, capitalName = capitalName } diff --git a/buildkite/src/Pipeline/Tag.dhall b/buildkite/src/Pipeline/Tag.dhall index 87b70d8979d..a64824b0896 100644 --- a/buildkite/src/Pipeline/Tag.dhall +++ b/buildkite/src/Pipeline/Tag.dhall @@ -1,69 +1,93 @@ --- Tag defines pipeline +-- Tag defines pipeline -- Using tags one can tailor pipeline for any need. Each job should be tagged with one or several tags --- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline +-- then on pipeline settings we can define which tagged jobs to include or exclue in pipeline let Prelude = ../External/Prelude.dhall -let List/any = Prelude.List.any - -let Tag : Type = < Fast | Long | VeryLong | TearDown | Lint | Release | Hardfork | Test | Toolchain > -let toNatural: Tag -> Natural = \(tag: Tag) -> - merge { - Fast = 1 - , Long = 2 - , VeryLong = 3 - , TearDown = 4 - , Lint = 5 - , Release = 6 - , Test = 7 - , Toolchain = 8 - , Hardfork = 9 - } tag +let List/any = Prelude.List.any -let equal: Tag -> Tag -> Bool = \(left: Tag) -> \(right: Tag) -> - Prelude.Natural.equal (toNatural left) (toNatural right) +let Tag + : Type + = < Fast + | Long + | VeryLong + | TearDown + | Lint + | Release + | Hardfork + | Test + | Toolchain + > +let toNatural + : Tag -> Natural + = \(tag : Tag) + -> merge + { Fast = 1 + , Long = 2 + , VeryLong = 3 + , TearDown = 4 + , Lint = 5 + , Release = 6 + , Test = 7 + , Toolchain = 8 + , Hardfork = 9 + } + tag -let hasAny: Tag -> List Tag -> Bool = \(input: Tag) -> \(tags: List Tag) -> - List/any Tag (\(x: Tag) -> equal x input ) tags +let equal + : Tag -> Tag -> Bool + = \(left : Tag) + -> \(right : Tag) + -> Prelude.Natural.equal (toNatural left) (toNatural right) -let contains: List Tag -> List Tag -> Bool = \(input: List Tag) -> \(tags: List Tag) -> - List/any Tag (\(x: Tag) -> hasAny x tags ) input +let hasAny + : Tag -> List Tag -> Bool + = \(input : Tag) + -> \(tags : List Tag) + -> List/any Tag (\(x : Tag) -> equal x input) tags -let capitalName = \(tag : Tag) -> - merge { - Fast = "Fast" - , Long = "Long" - , VeryLong = "VeryLong" - , TearDown = "TearDown" - , Lint = "Lint" - , Release = "Release" - , Test = "Test" - , Toolchain = "Toolchain" - , Hardfork = "Hardfork" - } tag +let contains + : List Tag -> List Tag -> Bool + = \(input : List Tag) + -> \(tags : List Tag) + -> List/any Tag (\(x : Tag) -> hasAny x tags) input -let lowerName = \(tag : Tag) -> - merge { - Fast = "fast" - , Long = "long" - , VeryLong = "veryLong" - , TearDown = "tearDown" - , Lint = "lint" - , Release = "release" - , Test = "test" - , Toolchain = "toolchain" - , Hardfork = "hardfork" - } tag +let capitalName = + \(tag : Tag) + -> merge + { Fast = "Fast" + , Long = "Long" + , VeryLong = "VeryLong" + , TearDown = "TearDown" + , Lint = "Lint" + , Release = "Release" + , Test = "Test" + , Toolchain = "Toolchain" + , Hardfork = "Hardfork" + } + tag +let lowerName = + \(tag : Tag) + -> merge + { Fast = "fast" + , Long = "long" + , VeryLong = "veryLong" + , TearDown = "tearDown" + , Lint = "lint" + , Release = "release" + , Test = "test" + , Toolchain = "toolchain" + , Hardfork = "hardfork" + } + tag -in -{ - Type = Tag, - capitalName = capitalName, - lowerName = lowerName, - toNatural = toNatural, - equal = equal, - hasAny = hasAny, - contains = contains -} \ No newline at end of file +in { Type = Tag + , capitalName = capitalName + , lowerName = lowerName + , toNatural = toNatural + , equal = equal + , hasAny = hasAny + , contains = contains + } diff --git a/buildkite/src/Pipeline/TriggerCommand.dhall b/buildkite/src/Pipeline/TriggerCommand.dhall index c8010354cde..37bcd389ebc 100644 --- a/buildkite/src/Pipeline/TriggerCommand.dhall +++ b/buildkite/src/Pipeline/TriggerCommand.dhall @@ -1,7 +1,7 @@ let Cmd = ../Lib/Cmds.dhall -in -( - \(dhallPipelineRelativeToBuildKiteDir : Text) -> - Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/${dhallPipelineRelativeToBuildKiteDir}).pipeline' | buildkite-agent pipeline upload" -) : Text -> Cmd.Type +in ( \(dhallPipelineRelativeToBuildKiteDir : Text) + -> Cmd.quietly + "dhall-to-yaml --quoted <<< '(./buildkite/${dhallPipelineRelativeToBuildKiteDir}).pipeline' | buildkite-agent pipeline upload" + ) + : Text -> Cmd.Type diff --git a/buildkite/src/Pipeline/Type.dhall b/buildkite/src/Pipeline/Type.dhall index dfcfca322a7..eaf46704e8b 100644 --- a/buildkite/src/Pipeline/Type.dhall +++ b/buildkite/src/Pipeline/Type.dhall @@ -1,5 +1 @@ -let Command = ../Command/Base.dhall -in - --- TODO: Union type with block steps -{ steps : List Command.Type } +let Command = ../Command/Base.dhall in { steps : List Command.Type } diff --git a/buildkite/src/Prepare.dhall b/buildkite/src/Prepare.dhall index 5b78ff24369..82cdb1c4df6 100644 --- a/buildkite/src/Prepare.dhall +++ b/buildkite/src/Prepare.dhall @@ -2,43 +2,50 @@ -- Keep these rules lean! They have to run unconditionally. let SelectFiles = ./Lib/SelectFiles.dhall + let Cmd = ./Lib/Cmds.dhall let Command = ./Command/Base.dhall + let Docker = ./Command/Docker/Type.dhall + let JobSpec = ./Pipeline/JobSpec.dhall + let Pipeline = ./Pipeline/Dsl.dhall -let PipelineMode = ./Pipeline/Mode.dhall -let PipelineFilter = ./Pipeline/Filter.dhall -let PipelineTag = ./Pipeline/Tag.dhall + let Size = ./Command/Size.dhall -let triggerCommand = ./Pipeline/TriggerCommand.dhall let mode = env:BUILDKITE_PIPELINE_MODE as Text ? "PullRequest" + let filter = env:BUILDKITE_PIPELINE_FILTER as Text ? "FastOnly" -let config : Pipeline.Config.Type = Pipeline.Config::{ - spec = JobSpec::{ - name = "prepare", - -- TODO: Clean up this code so we don't need an unused dirtyWhen here - dirtyWhen = [ SelectFiles.everything ] - }, - steps = [ - Command.build Command.Config::{ - commands = [ - Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}", - Cmd.run "export BUILDKITE_PIPELINE_FILTER=${filter}", - Cmd.run "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall", - Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { mode=(./buildkite/src/Pipeline/Mode.dhall).Type.${mode}, filter=(./buildkite/src/Pipeline/Filter.dhall).Type.${filter} }' | buildkite-agent pipeline upload" - ], - label = "Prepare monorepo triage", - key = "monorepo-${mode}-${filter}", - target = Size.Small, - docker = Some Docker::{ - image = (./Constants/ContainerImages.dhall).toolchainBase, - environment = ["BUILDKITE_AGENT_ACCESS_TOKEN"] +let config + : Pipeline.Config.Type + = Pipeline.Config::{ + , spec = JobSpec::{ + , name = "prepare" + , dirtyWhen = [ SelectFiles.everything ] + } + , steps = + [ Command.build + Command.Config::{ + , commands = + [ Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}" + , Cmd.run "export BUILDKITE_PIPELINE_FILTER=${filter}" + , Cmd.run + "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall" + , Cmd.quietly + "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { mode=(./buildkite/src/Pipeline/Mode.dhall).Type.${mode}, filter=(./buildkite/src/Pipeline/Filter.dhall).Type.${filter} }' | buildkite-agent pipeline upload" + ] + , label = "Prepare monorepo triage" + , key = "monorepo-${mode}-${filter}" + , target = Size.Small + , docker = Some Docker::{ + , image = (./Constants/ContainerImages.dhall).toolchainBase + , environment = [ "BUILDKITE_AGENT_ACCESS_TOKEN" ] + } + } + ] } - } - ] -} -in (Pipeline.build config).pipeline + +in (Pipeline.build config).pipeline diff --git a/buildkite/src/Projects/ValidationService.dhall b/buildkite/src/Projects/ValidationService.dhall index 33d809c6136..36b5ebcc468 100644 --- a/buildkite/src/Projects/ValidationService.dhall +++ b/buildkite/src/Projects/ValidationService.dhall @@ -1,24 +1,25 @@ let Cmd = ../Lib/Cmds.dhall let rootPath = "src/app/validation" + let containerImage = (../Constants/ContainerImages.dhall).elixirToolchain -let docker = Cmd.Docker::{image = containerImage} + +let docker = Cmd.Docker::{ image = containerImage } + let runCmd = \(cmd : Text) -> Cmd.run "(cd ${rootPath} && ${cmd})" + let runMix = \(cmd : Text) -> runCmd "mix ${cmd}" --- TODO: support mix archive caching let initCommands = - [ - runMix "local.hex --force", - runMix "local.rebar --force", - runMix "deps.get" - ] + [ runMix "local.hex --force" + , runMix "local.rebar --force" + , runMix "deps.get" + ] -in { - rootPath = rootPath, - containerImage = containerImage, - docker = docker, - runCmd = runCmd, - runMix = runMix, - initCommands = initCommands -} +in { rootPath = rootPath + , containerImage = containerImage + , docker = docker + , runCmd = runCmd + , runMix = runMix + , initCommands = initCommands + } diff --git a/buildkite/src/gen/Jobs.dhall b/buildkite/src/gen/Jobs.dhall index a7622630d5b..f3763e5b86d 100644 --- a/buildkite/src/gen/Jobs.dhall +++ b/buildkite/src/gen/Jobs.dhall @@ -1,4 +1,3 @@ -- This file is autogenerated during builds. It remains checked in to ensure -- dhall configuration can still execute locally without running codegen. -let Pipeline = ../Pipeline/Dsl.dhall in -[ ] : List Pipeline.CompoundType +let Pipeline = ../Pipeline/Dsl.dhall in [] : List Pipeline.CompoundType