Merge pull request #15664 from MinaProtocol/dkijania/introduce_postgr…

…es_side_docker_job

Introduce new Command in Dhall to run job along with postgres docker

buildkite/scripts/replayer-test.sh

@@ -1,67 +1,3 @@
 #!/bin/bash
 
-TEST_DIR=/workdir/src/app/replayer/
-
-set -eox pipefail
-
-echo "Updating apt, installing packages"
-apt-get update
-
-# Don't prompt for answers during apt-get install
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get install -y git apt-transport-https ca-certificates curl wget
-
-git config --global --add safe.directory $BUILDKITE_BUILD_CHECKOUT_PATH
-
-source buildkite/scripts/export-git-env-vars.sh
-
-DB=archive
-DOCKER_IMAGE=12.4-alpine
-CONTAINER_FILE=docker.container
-
-PG_PORT=5433
-PG_PASSWORD=somepassword
-DOCKER_IMAGE=12.4-alpine
-CONTAINER_FILE=docker.container
-
-function cleanup () {
-    CONTAINER=`cat $CONTAINER_FILE`
-
-    if [[ ! -z $CONTAINER ]] ; then
-	echo "Killing, removing docker container"
-	for action in kill rm; do
-	    docker container $action $CONTAINER
-	done
-    fi
-
-    rm -f $CONTAINER_FILE
-}
-
-docker network create replayer || true
-
-# -v mounts dir with Unix socket on host
-echo "Starting docker with Postgresql"
-docker run \
-       --network replayer \
-       --volume $BUILDKITE_BUILD_CHECKOUT_PATH:/workdir \
-       --name replayer-postgres -d -p $PG_PORT:5432 \
-       -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=$PG_PASSWORD -e POSTGRES_DB=$DB postgres:$DOCKER_IMAGE > $CONTAINER_FILE
-
-trap "cleanup; exit 1" SIGINT
-
-# wait for Postgresql to become available
-sleep 5
-
-echo "Populating archive database"
-
-NETWORK_GATEWAY=$(docker network inspect -f "{{(index .IPAM.Config 0).Gateway}}" replayer)
-
-PG_CONN="postgres://postgres:$PG_PASSWORD@$NETWORK_GATEWAY:$PG_PORT/$DB"
-
-
-docker exec replayer-postgres psql $PG_CONN -f $TEST_DIR/test/archive_db.sql
-
-docker run --network replayer --volume $BUILDKITE_BUILD_CHECKOUT_PATH:/workdir gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG /workdir/scripts/replayer-test.sh -d $TEST_DIR -a mina-replayer -p $PG_CONN
-
-cleanup
+source scripts/replayer-test.sh -i src/test/archive/sample_db/replayer_input_file.json -p $PG_CONN -a mina-replayer

buildkite/src/Command/ReplayerTest.dhall

@@ -1,10 +1,11 @@
 let B = ../External/Buildkite.dhall
 
 let Prelude = ../External/Prelude.dhall
-
+let Artifacts = ../Constants/Artifacts.dhall
 let Command = ./Base.dhall
 let Docker = ./Docker/Type.dhall
 let Size = ./Size.dhall
+let RunWithPostgres = ./RunWithPostgres.dhall
 
 let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
 
@@ -14,11 +15,14 @@ let Cmd = ../Lib/Cmds.dhall in
     Command.build
       Command.Config::{
         commands = [
-          Cmd.run "./buildkite/scripts/replayer-test.sh"
+        RunWithPostgres.runInDockerWithPostgresConn
+          ([] : List Text)
+           "./src/test/archive/sample_db/archive_db.sql"
+           Artifacts.Type.Archive 
+           "./buildkite/scripts/replayer-test.sh"
         ],
-        label = "Replayer test",
+        label = "Archive: Replayer test",
         key = "replayer-test",
-        soft_fail = Some (B/SoftFail.Boolean True),
         target = Size.Large,
         depends_on = dependsOn
       }

buildkite/src/Command/RunWithPostgres.dhall

@@ -0,0 +1,58 @@
+let Prelude = ../External/Prelude.dhall
+let P = Prelude
+let Text/concatMap = P.Text.concatMap
+
+let Cmd = ../Lib/Cmds.dhall
+let Mina = ../Command/Mina.dhall
+let S = ../Lib/SelectFiles.dhall
+let ContainerImages = ../Constants/ContainerImages.dhall
+let Artifacts= ../Constants/Artifacts.dhall
+
+let r = Cmd.run
+
+let runInDockerWithPostgresConn : List Text -> Text -> Artifacts.Type -> Text -> Cmd.Type =
+  \(environment : List Text) ->
+  \(initScript: Text ) -> 
+  \(docker: Artifacts.Type) ->
+  \(innerScript : Text) ->
+   -- port is used only in pg_conn variable. By using --network host we are disabling 
+   -- any control over postgres port
+    let port="5432" 
+    let user="postgres"
+    let password="postgres"
+    let postgresDockerName="postgres"
+    let dockerVersion= ContainerImages.postgres
+    let dbName="archive"
+    let pg_conn="postgres://${user}:${password}@localhost:${port}/${dbName}"
+
+    let envVars =
+      Text/concatMap
+        Text
+        (\(var : Text) -> " --env ${var}")
+        ([
+            "PG_PORT=${port}",
+            "POSTGRES_USER=${user}",
+            "POSTGRES_PASSWORD=${password}",
+            "POSTGRES_DB=${dbName}",
+            "PG_CONN=${pg_conn}"
+        ] # environment)
+    in
+
+    let outerDir : Text =
+      "\\\$BUILDKITE_BUILD_CHECKOUT_PATH"
+    in 
+
+    Cmd.chain [
+         "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true", 
+         "source buildkite/scripts/export-git-env-vars.sh",
+         "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}",
+         "sleep 5",
+         "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}",
+         "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir  ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName docker}:\\\$MINA_DOCKER_TAG ${innerScript}"
+    ] 
+
+in
+
+{
+  runInDockerWithPostgresConn = runInDockerWithPostgresConn
+}

buildkite/src/Constants/ContainerImages.dhall

@@ -16,6 +16,7 @@
   elixirToolchain = "elixir:1.10-alpine",
   nodeToolchain = "node:14.13.1-stretch-slim",
   ubuntu2004 = "ubuntu:20.04",
+  postgres = "postgres:12.4-alpine",
   xrefcheck = "serokell/xrefcheck@sha256:8fbb35a909abc353364f1bd3148614a1160ef3c111c0c4ae84e58fdf16019eeb",
   nixos = "gcr.io/o1labs-192920/nix-unstable:1.0.0"
 }

buildkite/src/Lib/Cmds.dhall

@@ -29,7 +29,11 @@ let module = \(environment : List Text) ->
   let Cmd = { line: Text, readable: Optional Text }
   let run : Text -> Cmd =
     \(script: Text) -> { line = script, readable = Some script }
-
+
+  let chain : List Text -> Cmd = 
+    \(chainOfCommands: List Text) ->
+      run (Text/concatSep " && " chainOfCommands)
+
   let quietly : Text -> Cmd =
     \(script: Text) -> { line = script, readable = None Text }
   let true : Cmd = quietly "true"
@@ -95,6 +99,7 @@ let module = \(environment : List Text) ->
   , CacheSetupCmd = CacheSetupCmd
   , quietly = quietly
   , run = run
+  , chain = chain
   , true = true
   , false = false
   , runInDocker = runInDocker

scripts/replayer-test.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
 
 # test replayer on known archive db
+set -x
 
-REPLAYER_DIR=src/app/replayer
+INPUT_FILE=src/test/archive/sample_db/replayer_input_file.json
 REPLAYER_APP=_build/default/src/app/replayer/replayer.exe
-PG_CONN=PG_CONN=postgres://postgres:postgres@localhost:5433/archive
+PG_CONN=postgres://postgres:postgres@localhost:5433/archive
 
 while [[ "$#" -gt 0 ]]; do case $1 in
-  -d|--dir) REPLAYER_DIR="$2"; shift;;
+  -i|--input-file) INPUT_FILE="$2"; shift;;
   -a|--app) REPLAYER_APP="$2"; shift;;
-  -p| --pg) PG_CONN="$2"; shift;;
+  -p|--pg) PG_CONN="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
@@ -22,7 +23,7 @@ function report () {
 }
 
 echo "Running replayer"
-$REPLAYER_APP --archive-uri $PG_CONN --input-file $REPLAYER_DIR/test/input.json
+$REPLAYER_APP --archive-uri $PG_CONN --input-file $INPUT_FILE
 
 RESULT=$?
 

src/app/replayer/test/archive_db.sql → src/test/archive/sample_db/archive_db.sql

@@ -9878,4 +9878,3 @@ ALTER TABLE ONLY public.zkapp_verification_keys
 --
 -- PostgreSQL database dump complete
 --
-

src/test/archive/sample_db/genesis.json

@@ -0,0 +1,54 @@
+{
+  "genesis": {
+    "genesis_state_timestamp": "2023-12-05T21:59:37+0100",
+    "k": 24
+  },
+  "ledger": {
+    "name": "release",
+    "num_accounts": 250,
+    "accounts": [
+      {
+       "pk": "B62qo5raDfFWM6oaooevrgMxMXmp3NnxMGbus3yubqaexzNodKDmock",
+       "sk": null,
+       "balance": "5.000000000",
+       "delegate": null
+      },
+      {
+       "pk": "B62qpXKjMPqNsUBPeAGhpjao8BE8FqdYdk8zS2RwxaEBadByS3nbfD1",
+       "sk": null,
+       "balance": "65500.000000000",
+       "delegate": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5"
+      },
+      {
+       "pk": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5",
+       "sk": null,
+       "delegate": null,
+       "balance": "500.000000000"
+      },
+      {
+       "pk": "B62qmYXqVz13eCpiHEEbvkWDU7jQAwfuf6zv4HLfiRthmLMZMQDXLPi",
+       "sk": null,
+       "balance": "11550000.000000000",
+       "delegate": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz"
+      },
+      {
+       "pk": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz",
+       "sk": null,
+       "balance": "0.000000000",
+       "delegate": null
+      },
+      {
+       "pk": "B62qkXPw6LbqeY3kaEYHrc6MeP6Wi7wcDc5fzbegn5yRoj2UvyCoe7M",
+       "sk": null,
+       "balance": "11550000.000000000",
+       "delegate": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS"
+      },
+      {
+       "pk": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS",
+       "sk": null,
+       "balance": "0.000000000",
+       "delegate": null
+      }
+    ]
+  }
+}

src/test/archive/sample_db/replayer_input_file.json

@@ -0,0 +1,50 @@
+{
+    "genesis_ledger": {
+      "name": "release",
+      "num_accounts": 250,
+      "accounts": [
+       {
+        "pk": "B62qo5raDfFWM6oaooevrgMxMXmp3NnxMGbus3yubqaexzNodKDmock",
+        "sk": null,
+        "balance": "5.000000000",
+        "delegate": null
+       },
+       {
+        "pk": "B62qpXKjMPqNsUBPeAGhpjao8BE8FqdYdk8zS2RwxaEBadByS3nbfD1",
+        "sk": null,
+        "balance": "65500.000000000",
+        "delegate": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5"
+       },
+       {
+        "pk": "B62qov6t5ahQQMdoanV8tgeS8ddpqsL5MCntfh8qzr1aNqGsgnhhvV5",
+        "sk": null,
+        "delegate": null,
+        "balance": "500.000000000"
+       },
+       {
+        "pk": "B62qmYXqVz13eCpiHEEbvkWDU7jQAwfuf6zv4HLfiRthmLMZMQDXLPi",
+        "sk": null,
+        "balance": "11550000.000000000",
+        "delegate": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz"
+       },
+       {
+        "pk": "B62qr7HhRDB6Me7FFXme5MFcoqRa452etDR4mNgT9Zi6U73R238ytmz",
+        "sk": null,
+        "balance": "0.000000000",
+        "delegate": null
+       },
+       {
+        "pk": "B62qkXPw6LbqeY3kaEYHrc6MeP6Wi7wcDc5fzbegn5yRoj2UvyCoe7M",
+        "sk": null,
+        "balance": "11550000.000000000",
+        "delegate": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS"
+       },
+       {
+        "pk": "B62qmQsYKF97qxFVUuwHdmJySUM6F6qNnNYvxfrvYDPdLPzBspCPYLS",
+        "sk": null,
+        "balance": "0.000000000",
+        "delegate": null
+       }
+      ]
+     }
+  }