diff --git a/buildkite/scripts/replayer-test.sh b/buildkite/scripts/replayer-test.sh index 08a84fd05f4..5439d58e3a2 100755 --- a/buildkite/scripts/replayer-test.sh +++ b/buildkite/scripts/replayer-test.sh @@ -1,67 +1,3 @@ #!/bin/bash -TEST_DIR=/workdir/src/app/replayer/ - -set -eox pipefail - -echo "Updating apt, installing packages" -apt-get update - -# Don't prompt for answers during apt-get install -export DEBIAN_FRONTEND=noninteractive - -apt-get install -y git apt-transport-https ca-certificates curl wget - -git config --global --add safe.directory $BUILDKITE_BUILD_CHECKOUT_PATH - -source buildkite/scripts/export-git-env-vars.sh - -DB=archive -DOCKER_IMAGE=12.4-alpine -CONTAINER_FILE=docker.container - -PG_PORT=5433 -PG_PASSWORD=somepassword -DOCKER_IMAGE=12.4-alpine -CONTAINER_FILE=docker.container - -function cleanup () { - CONTAINER=`cat $CONTAINER_FILE` - - if [[ ! -z $CONTAINER ]] ; then - echo "Killing, removing docker container" - for action in kill rm; do - docker container $action $CONTAINER - done - fi - - rm -f $CONTAINER_FILE -} - -docker network create replayer || true - -# -v mounts dir with Unix socket on host -echo "Starting docker with Postgresql" -docker run \ - --network replayer \ - --volume $BUILDKITE_BUILD_CHECKOUT_PATH:/workdir \ - --name replayer-postgres -d -p $PG_PORT:5432 \ - -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=$PG_PASSWORD -e POSTGRES_DB=$DB postgres:$DOCKER_IMAGE > $CONTAINER_FILE - -trap "cleanup; exit 1" SIGINT - -# wait for Postgresql to become available -sleep 5 - -echo "Populating archive database" - -NETWORK_GATEWAY=$(docker network inspect -f "{{(index .IPAM.Config 0).Gateway}}" replayer) - -PG_CONN="postgres://postgres:$PG_PASSWORD@$NETWORK_GATEWAY:$PG_PORT/$DB" - - -docker exec replayer-postgres psql $PG_CONN -f $TEST_DIR/test/archive_db.sql - -docker run --network replayer --volume $BUILDKITE_BUILD_CHECKOUT_PATH:/workdir gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG /workdir/scripts/replayer-test.sh -d $TEST_DIR -a mina-replayer -p $PG_CONN - -cleanup +source scripts/replayer-test.sh -i src/test/archive/sample_db/replayer_input_file.json -p $PG_CONN -a mina-replayer \ No newline at end of file diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 0352fb26ef0..ce45f13b8bd 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -1,10 +1,11 @@ let B = ../External/Buildkite.dhall let Prelude = ../External/Prelude.dhall - +let Artifacts = ../Constants/Artifacts.dhall let Command = ./Base.dhall let Docker = ./Docker/Type.dhall let Size = ./Size.dhall +let RunWithPostgres = ./RunWithPostgres.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type @@ -14,11 +15,14 @@ let Cmd = ../Lib/Cmds.dhall in Command.build Command.Config::{ commands = [ - Cmd.run "./buildkite/scripts/replayer-test.sh" + RunWithPostgres.runInDockerWithPostgresConn + ([] : List Text) + "./src/test/archive/sample_db/archive_db.sql" + Artifacts.Type.Archive + "./buildkite/scripts/replayer-test.sh" ], - label = "Replayer test", + label = "Archive: Replayer test", key = "replayer-test", - soft_fail = Some (B/SoftFail.Boolean True), target = Size.Large, depends_on = dependsOn } diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall new file mode 100644 index 00000000000..f309df20a66 --- /dev/null +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -0,0 +1,58 @@ +let Prelude = ../External/Prelude.dhall +let P = Prelude +let Text/concatMap = P.Text.concatMap + +let Cmd = ../Lib/Cmds.dhall +let Mina = ../Command/Mina.dhall +let S = ../Lib/SelectFiles.dhall +let ContainerImages = ../Constants/ContainerImages.dhall +let Artifacts= ../Constants/Artifacts.dhall + +let r = Cmd.run + +let runInDockerWithPostgresConn : List Text -> Text -> Artifacts.Type -> Text -> Cmd.Type = + \(environment : List Text) -> + \(initScript: Text ) -> + \(docker: Artifacts.Type) -> + \(innerScript : Text) -> + -- port is used only in pg_conn variable. By using --network host we are disabling + -- any control over postgres port + let port="5432" + let user="postgres" + let password="postgres" + let postgresDockerName="postgres" + let dockerVersion= ContainerImages.postgres + let dbName="archive" + let pg_conn="postgres://${user}:${password}@localhost:${port}/${dbName}" + + let envVars = + Text/concatMap + Text + (\(var : Text) -> " --env ${var}") + ([ + "PG_PORT=${port}", + "POSTGRES_USER=${user}", + "POSTGRES_PASSWORD=${password}", + "POSTGRES_DB=${dbName}", + "PG_CONN=${pg_conn}" + ] # environment) + in + + let outerDir : Text = + "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" + in + + Cmd.chain [ + "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true", + "source buildkite/scripts/export-git-env-vars.sh", + "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}", + "sleep 5", + "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}", + "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName docker}:\\\$MINA_DOCKER_TAG ${innerScript}" + ] + +in + +{ + runInDockerWithPostgresConn = runInDockerWithPostgresConn +} \ No newline at end of file diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall index 013ead50892..02500d92cd6 100644 --- a/buildkite/src/Constants/ContainerImages.dhall +++ b/buildkite/src/Constants/ContainerImages.dhall @@ -16,6 +16,7 @@ elixirToolchain = "elixir:1.10-alpine", nodeToolchain = "node:14.13.1-stretch-slim", ubuntu2004 = "ubuntu:20.04", + postgres = "postgres:12.4-alpine", xrefcheck = "serokell/xrefcheck@sha256:8fbb35a909abc353364f1bd3148614a1160ef3c111c0c4ae84e58fdf16019eeb", nixos = "gcr.io/o1labs-192920/nix-unstable:1.0.0" } diff --git a/buildkite/src/Lib/Cmds.dhall b/buildkite/src/Lib/Cmds.dhall index 6a5e10241a2..1e500fcd0bd 100644 --- a/buildkite/src/Lib/Cmds.dhall +++ b/buildkite/src/Lib/Cmds.dhall @@ -29,7 +29,11 @@ let module = \(environment : List Text) -> let Cmd = { line: Text, readable: Optional Text } let run : Text -> Cmd = \(script: Text) -> { line = script, readable = Some script } - + + let chain : List Text -> Cmd = + \(chainOfCommands: List Text) -> + run (Text/concatSep " && " chainOfCommands) + let quietly : Text -> Cmd = \(script: Text) -> { line = script, readable = None Text } let true : Cmd = quietly "true" @@ -95,6 +99,7 @@ let module = \(environment : List Text) -> , CacheSetupCmd = CacheSetupCmd , quietly = quietly , run = run + , chain = chain , true = true , false = false , runInDocker = runInDocker diff --git a/scripts/replayer-test.sh b/scripts/replayer-test.sh index 663bf797808..b75c32f4b08 100755 --- a/scripts/replayer-test.sh +++ b/scripts/replayer-test.sh @@ -1,15 +1,16 @@ #!/bin/bash # test replayer on known archive db +set -x -REPLAYER_DIR=src/app/replayer +INPUT_FILE=src/test/archive/sample_db/replayer_input_file.json REPLAYER_APP=_build/default/src/app/replayer/replayer.exe -PG_CONN=PG_CONN=postgres://postgres:postgres@localhost:5433/archive +PG_CONN=postgres://postgres:postgres@localhost:5433/archive while [[ "$#" -gt 0 ]]; do case $1 in - -d|--dir) REPLAYER_DIR="$2"; shift;; + -i|--input-file) INPUT_FILE="$2"; shift;; -a|--app) REPLAYER_APP="$2"; shift;; - -p| --pg) PG_CONN="$2"; shift;; + -p|--pg) PG_CONN="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -22,7 +23,7 @@ function report () { } echo "Running replayer" -$REPLAYER_APP --archive-uri $PG_CONN --input-file $REPLAYER_DIR/test/input.json +$REPLAYER_APP --archive-uri $PG_CONN --input-file $INPUT_FILE RESULT=$? diff --git a/src/app/replayer/test/input.json b/src/app/replayer/test/input.json deleted file mode 100644 index 57cc71bb265..00000000000 --- a/src/app/replayer/test/input.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "genesis_ledger": { - "name": "release", - "num_accounts": 250, - "accounts": [ - { - "pk": "B62qo5raDfFWM6oaooevrgMxMXmp3NnxMGbus3yubqaexzNodKDmock", - "sk": null, - "balance": "5.000000000", - "delegate": null - }, - { - "pk": "B62qpXKjMPqNsUBPeAGhpjao8BE8FqdYdk8zS2RwxaEBadByS3nbfD1", - "sk": null, - "balance": "65500.000000000", - "delegate": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5" - }, - { - "pk": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5", - "sk": null, - "delegate": null, - "balance": "500.000000000" - }, - { - "pk": "B62qmYXqVz13eCpiHEEbvkWDU7jQAwfuf6zv4HLfiRthmLMZMQDXLPi", - "sk": null, - "balance": "11550000.000000000", - "delegate": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz" - }, - { - "pk": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz", - "sk": null, - "balance": "0.000000000", - "delegate": null - }, - { - "pk": "B62qkXPw6LbqeY3kaEYHrc6MeP6Wi7wcDc5fzbegn5yRoj2UvyCoe7M", - "sk": null, - "balance": "11550000.000000000", - "delegate": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS" - }, - { - "pk": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS", - "sk": null, - "balance": "0.000000000", - "delegate": null - } - ] - } -} \ No newline at end of file diff --git a/src/app/replayer/test/README.md b/src/test/archive/sample_db/README.md similarity index 100% rename from src/app/replayer/test/README.md rename to src/test/archive/sample_db/README.md diff --git a/src/app/replayer/test/archive_db.sql b/src/test/archive/sample_db/archive_db.sql similarity index 99% rename from src/app/replayer/test/archive_db.sql rename to src/test/archive/sample_db/archive_db.sql index 4bd72e7033d..046de959c57 100644 --- a/src/app/replayer/test/archive_db.sql +++ b/src/test/archive/sample_db/archive_db.sql @@ -9878,4 +9878,3 @@ ALTER TABLE ONLY public.zkapp_verification_keys -- -- PostgreSQL database dump complete -- - diff --git a/src/app/replayer/test/convert_chain_to_canonical.sh b/src/test/archive/sample_db/convert_chain_to_canonical.sh similarity index 100% rename from src/app/replayer/test/convert_chain_to_canonical.sh rename to src/test/archive/sample_db/convert_chain_to_canonical.sh diff --git a/src/test/archive/sample_db/genesis.json b/src/test/archive/sample_db/genesis.json new file mode 100644 index 00000000000..7d2b8574e0b --- /dev/null +++ b/src/test/archive/sample_db/genesis.json @@ -0,0 +1,54 @@ +{ + "genesis": { + "genesis_state_timestamp": "2023-12-05T21:59:37+0100", + "k": 24 + }, + "ledger": { + "name": "release", + "num_accounts": 250, + "accounts": [ + { + "pk": "B62qo5raDfFWM6oaooevrgMxMXmp3NnxMGbus3yubqaexzNodKDmock", + "sk": null, + "balance": "5.000000000", + "delegate": null + }, + { + "pk": "B62qpXKjMPqNsUBPeAGhpjao8BE8FqdYdk8zS2RwxaEBadByS3nbfD1", + "sk": null, + "balance": "65500.000000000", + "delegate": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5" + }, + { + "pk": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5", + "sk": null, + "delegate": null, + "balance": "500.000000000" + }, + { + "pk": "B62qmYXqVz13eCpiHEEbvkWDU7jQAwfuf6zv4HLfiRthmLMZMQDXLPi", + "sk": null, + "balance": "11550000.000000000", + "delegate": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz" + }, + { + "pk": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz", + "sk": null, + "balance": "0.000000000", + "delegate": null + }, + { + "pk": "B62qkXPw6LbqeY3kaEYHrc6MeP6Wi7wcDc5fzbegn5yRoj2UvyCoe7M", + "sk": null, + "balance": "11550000.000000000", + "delegate": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS" + }, + { + "pk": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS", + "sk": null, + "balance": "0.000000000", + "delegate": null + } + ] + } +} diff --git a/src/test/archive/sample_db/precomputed_blocks.zip b/src/test/archive/sample_db/precomputed_blocks.zip new file mode 100644 index 00000000000..d61d6fc7c26 Binary files /dev/null and b/src/test/archive/sample_db/precomputed_blocks.zip differ diff --git a/src/test/archive/sample_db/replayer_input_file.json b/src/test/archive/sample_db/replayer_input_file.json new file mode 100644 index 00000000000..dd769059dd6 --- /dev/null +++ b/src/test/archive/sample_db/replayer_input_file.json @@ -0,0 +1,50 @@ +{ + "genesis_ledger": { + "name": "release", + "num_accounts": 250, + "accounts": [ + { + "pk": "B62qo5raDfFWM6oaooevrgMxMXmp3NnxMGbus3yubqaexzNodKDmock", + "sk": null, + "balance": "5.000000000", + "delegate": null + }, + { + "pk": "B62qpXKjMPqNsUBPeAGhpjao8BE8FqdYdk8zS2RwxaEBadByS3nbfD1", + "sk": null, + "balance": "65500.000000000", + "delegate": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5" + }, + { + "pk": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5", + "sk": null, + "delegate": null, + "balance": "500.000000000" + }, + { + "pk": "B62qmYXqVz13eCpiHEEbvkWDU7jQAwfuf6zv4HLfiRthmLMZMQDXLPi", + "sk": null, + "balance": "11550000.000000000", + "delegate": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz" + }, + { + "pk": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz", + "sk": null, + "balance": "0.000000000", + "delegate": null + }, + { + "pk": "B62qkXPw6LbqeY3kaEYHrc6MeP6Wi7wcDc5fzbegn5yRoj2UvyCoe7M", + "sk": null, + "balance": "11550000.000000000", + "delegate": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS" + }, + { + "pk": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS", + "sk": null, + "balance": "0.000000000", + "delegate": null + } + ] + } + } \ No newline at end of file