Enable Genesis mode - transitory approach

IntersectMBO · Jan 29, 2025 · b8e44a4 · b8e44a4
1 parent 9203720
commit b8e44a4
Show file tree

Hide file tree

Showing 6 changed files with 7,644 additions and 140 deletions.
diff --git a/.buildkite/node_sync_tests.sh b/.buildkite/node_sync_tests.sh
@@ -27,18 +27,18 @@ echo " ==== importing packages from nix (https://search.nixos.org/packages)"
 python3 -c "import requests, pandas, psutil, pymysql;"
 
 env=$1
-build_mode=$2
-node_rev1=$3
-tag_no1=$4
-node_topology1=$5
-node_start_arguments1=$6
-node_rev2=$7
-tag_no2=$8
-node_topology2=$9
-node_start_arguments2=${10}
+node_rev1=$2
+tag_no1=$3
+node_topology1=$4
+node_start_arguments1=$5
+node_rev2=$6
+tag_no2=$7
+node_topology2=$8
+node_start_arguments2=$9
+use_genesis_mode=${10}
 
 echo " ==== start sync test"
-python ./sync_tests/tests/node_sync_test.py -b "$build_mode" -e "$env" -t1 "$tag_no1" -t2 "$tag_no2" -r1 "$node_rev1" -r2 "$node_rev2" -n1 "$node_topology1" -n2 "$node_topology2" -a1="$node_start_arguments1" -a2="$node_start_arguments2"
+python ./sync_tests/tests/node_sync_test.py -e "$env" -t1 "$tag_no1" -t2 "$tag_no2" -r1 "$node_rev1" -r2 "$node_rev2" -n1 "$node_topology1" -n2 "$node_topology2" -a1="$node_start_arguments1" -a2="$node_start_arguments2" -g "${use_genesis_mode}"
 
 echo "--- Prepare for adding sync test results to the AWS Database"
 python ./sync_tests/tests/node_write_sync_values_to_db.py -e "$env"
diff --git a/.buildkite/node_sync_tests.yml b/.buildkite/node_sync_tests.yml
@@ -1,11 +1,11 @@
 steps:
   - label: ':drum_with_drumsticks: Run the Cardano node sync test on Mainnet using a Linux machine'
     commands:
-      - nix develop --accept-flake-config .#python --command python ./sync_tests/tests/node_sync_test.py -b "${build_mode}" -e "${env}" -t1 "${tag_no1}" -t2 "${tag_no2}" -r1 "${node_rev1}" -r2 "${node_rev2}" -n1 "${node_topology1}" -n2 "${node_topology2}" -a1 "${node_start_arguments1}" -a2 "${node_start_arguments2}"
+      - nix develop --accept-flake-config .#python --command python ./sync_tests/tests/node_sync_test.py -e "${env}" -t1 "${tag_no1}" -t2 "${tag_no2}" -r1 "${node_rev1}" -r2 "${node_rev2}" -n1 "${node_topology1}" -n2 "${node_topology2}" -a1 "${node_start_arguments1}" -a2 "${node_start_arguments2}" -g "${use_genesis_mode}"
     timeout_in_minutes: 3000
     agents:
       system: x86_64-linux
-      queue: core-tech-bench
+      queue: "${queue_name}"
     artifact_paths:
       - 'logfile.log'
       - 'node.log'

diff --git a/.github/workflows/node_sync_test.yaml b/.github/workflows/node_sync_test.yaml
@@ -11,51 +11,44 @@ on:
         - preprod
         - preview
         - shelley-qa
-        default: preprod
-      build_mode:
-        description: "build-mode"
-        type: choice
-        options: 
-        - nix
-        default: nix
+        default: mainnet
       tag_no1:
         description: "Initial sync - Graph axis label"
         required: true
-        default: "9.2.1"
+        default: "10.2"
       node_rev1:
         description: "Initial sync - cardano-node revision"
         required: true
-        default: "tags/9.2.1"
+        default: "9017d918d0c5143a137a82d94af4a2ac67ae7d2e"
       node_topology1:
         description: "Initial sync - cardano-node topology"
         type: choice
-        options: 
+        options:
         - non-bootstrap-peers
         - bootstrap-peers
         default: legacy
-      node_start_arguments1:
-        description: "Initial sync - Extra arguments when starting the node"
-        required: false
-        default: "None"
-      tag_no2:
-        description: "Upgrade node - Graph axis label"
-        required: true
-        default: "None"
-      node_rev2:
-        description: "Upgrade node - cardano-node revision"
-        required: true
-        default: "None"
-      node_topology2:
-        description: "Upgrade node - cardano-node topology"
+      use_genesis_mode:
+        description: "Use Ouroboros Genesis mode"
+        type: boolean
+        default: true
+      buildkite_queue_name:
+          description: "Buildkite queue name to use"
+          type: choice
+          options:
+            - core-tech-bench
+            - core-tech-bench-af
+            - core-tech-bench-ap
+            - core-tech-bench-eu
+            - core-tech-bench-sa
+      buildkite_pipeline:
+        description: "Buildkite pipeline to use"
         type: choice
-        options: 
-        - legacy
-        - p2p
-        default: legacy
-      node_start_arguments2:
-        description: "Upgrade node - Extra arguments when starting the node"
-        required: false
-        default: "None"
+        options:
+        - node-sync-tests
+        - node-sync-tests-genesis-af
+        - node-sync-tests-genesis-ap
+        - node-sync-tests-genesis-eu
+        - node-sync-tests-genesis-sa
 
 
 jobs:
@@ -90,7 +83,7 @@ jobs:
         uses: 'buildkite/[email protected]'
         env:
           BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_API_ACCESS_TOKEN }}
-          PIPELINE: 'input-output-hk/node-sync-tests'
+          PIPELINE: "input-output-hk/${{ github.event.inputs.buildkite_pipeline }}"
           BRANCH: ${{ github.ref_name }}
           MESSAGE: ':github: Triggered by GitHub Action'
           AWS_DB_USERNAME: ${{ secrets.AWS_DB_USERNAME }}
@@ -100,103 +93,21 @@ jobs:
           BLOCKFROST_API_KEY: ${{ secrets.BLOCKFROST_API_KEY }}
           BUILD_ENV_VARS: '{
           "env":"${{ github.event.inputs.environment }}",
-          "build_mode":"${{ github.event.inputs.build_mode }}",
           "node_rev1":"${{ steps.get_tag.outputs.node_rev1 }}",
           "node_rev2":"${{ github.event.inputs.node_rev2 }}",          
           "tag_no1":"${{ github.event.inputs.tag_no1 }}",
           "tag_no2":"${{ github.event.inputs.tag_no2 }}",
           "node_topology1":"${{ github.event.inputs.node_topology1 }}",
           "node_topology2":"${{ github.event.inputs.node_topology2 }}",
-          "node_start_arguments1":"${{ github.event.inputs.node_start_arguments1 }}",
-          "node_start_arguments2":"${{ github.event.inputs.node_start_arguments2 }}",
+          "node_start_arguments1":"None",
+          "node_start_arguments2":"None",
+          "use_genesis_mode":"${{ github.event.inputs.use_genesis_mode }}",
+          "queue_name":"${{ github.event.inputs.buildkite_queue_name }}",
           
+          "CARDANO_NODE_SOCKET_PATH":"/var/tmp/${{ github.event.inputs.buildkite_pipeline }}/node.socket",
           "BLOCKFROST_API_KEY":"${{ secrets.BLOCKFROST_API_KEY }}",
           "AWS_DB_USERNAME":"${{ secrets.AWS_DB_USERNAME }}",
           "AWS_DB_PASS":"${{ secrets.AWS_DB_PASS }}",
           "AWS_DB_NAME":"${{ secrets.AWS_DB_NAME }}",
           "AWS_DB_HOSTNAME":"${{ secrets.AWS_DB_HOSTNAME }}"
           }'
-
-  node_sync_test:
-    strategy:
-      matrix:
-        env: [preprod, preview]
-        ghc: ["9.6"]
-        cabal: ["3.12.1.0"]
-        sys:
-          - { os: ubuntu-latest, shell: bash }
-          - { os: macos-latest, shell: bash }
-        branch:
-          - ${{ github.ref_name }}
-      fail-fast: false
-    runs-on: ${{ matrix.sys.os }}
-    timeout-minutes: 600
-    env:
-      MSYS2_PATH_TYPE: inherit
-      MSYSTEM: MINGW64
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref_name }}
-
-      - name: Install Nix
-        uses: cachix/install-nix-action@v27
-        if: matrix.sys.os != 'windows-latest'
-        with:
-          extra_nix_config: |
-            trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-            substituters = https://cache.nixos.org https://cache.iog.io
-            allow-import-from-derivation = true
-
-      - name: Enter Nix Development Shell
-        if: matrix.sys.os != 'windows-latest'
-        run: |
-          nix develop --accept-flake-config .#python --command bash -c "
-            echo 'Inside Nix development shell';
-            python -m pip install --upgrade pip;
-            pip install pytest requests allure-pytest pytest-html pytest-order pyyaml psycopg2 psutil blockfrost-python GitPython colorama matplotlib;
-          "
-
-      - name: Run sync test
-        if: runner.os != 'Windows'
-        env:
-          BLOCKFROST_API_KEY: ${{ secrets.BLOCKFROST_API_KEY }}
-        run: |
-          nix develop --accept-flake-config .#python --command python -m sync_tests.tests.node_sync_test \
-            -e ${{ matrix.env }} \
-            -b ${{ github.event.inputs.build_mode }} \
-            -t1 "${{ github.event.inputs.tag_no1 }}" \
-            -t2 "${{ github.event.inputs.tag_no2 }}" \
-            -r1 "${{ github.event.inputs.node_rev1 }}" \
-            -r2 "${{ github.event.inputs.node_rev2 }}" \
-            -n1 "${{ github.event.inputs.node_topology1 }}" \
-            -n2 "${{ github.event.inputs.node_topology2 }}" \
-            -a1="${{ github.event.inputs.node_start_arguments1 }}" \
-            -a2="${{ github.event.inputs.node_start_arguments2 }}"
-
-      - name: Write the test results into the database
-        env:
-          AWS_DB_USERNAME: ${{ secrets.AWS_DB_USERNAME }}
-          AWS_DB_PASS: ${{ secrets.AWS_DB_PASS }}
-          AWS_DB_NAME: ${{ secrets.AWS_DB_NAME }}
-          AWS_DB_HOSTNAME: ${{ secrets.AWS_DB_HOSTNAME }}
-          BLOCKFROST_API_KEY: ${{ secrets.BLOCKFROST_API_KEY }}
-        run: |
-          nix develop --accept-flake-config .#python --command python -m sync_tests.tests.node_write_sync_values_to_db \
-            -e ${{ matrix.env }}
-
-      - name: Prepare node logs
-        if: always()
-        run: |
-          cp logfile.log node.log
-
-      - name: Generate artifacts
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: node_logs_${{ matrix.env }}_${{ matrix.sys.os }}
-          path: |
-            logfile.log
-            node.log
-            sync_results.json
diff --git a/.github/workflows/node_sync_test_genesis.yaml b/.github/workflows/node_sync_test_genesis.yaml
@@ -0,0 +1,110 @@
+name: Node sync tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Environment on which Buildkite agent will run tests"
+        type: choice
+        options: 
+        - mainnet
+        - preprod
+        - preview
+        - shelley-qa
+        default: preprod
+      tag_no1:
+        description: "Initial sync - Graph axis label"
+        required: true
+        default: "9.2.1"
+      node_rev1:
+        description: "Initial sync - cardano-node revision"
+        required: true
+        default: "tags/9.2.1"
+      node_topology1:
+        description: "Initial sync - cardano-node topology"
+        type: choice
+        options: 
+        - non-bootstrap-peers
+        - bootstrap-peers
+        default: legacy
+      use_genesis_mode:
+        description: "Use Ouroboros Genesis mode"
+        type: boolean
+        default: false
+      buildkite_queue_name:
+          description: "Buildkite queue name to use"
+          type: choice
+          options:
+            - core-tech-bench
+            - core-tech-bench-af
+            - core-tech-bench-ap
+            - core-tech-bench-eu
+            - core-tech-bench-sa
+      buildkite_pipeline:
+        description: "Buildkite pipeline to use"
+        type: choice
+        options:
+        - node-sync-tests
+        - node-sync-tests-genesis-af
+
+
+jobs:
+  node_sync_test_buildkite_mainnet:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get latest node tag
+        id: get_tag
+        shell: bash
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "Using manual input for tag_no1"
+            echo "node_rev1=${{ github.event.inputs.node_rev1 }}" >> $GITHUB_OUTPUT
+          else
+            echo "Fetching latest tag for scheduled run"
+            # Fetch the latest tag dynamically
+            node_release_url='https://api.github.com/repos/IntersectMBO/cardano-node/releases/latest'
+            latest_tag=$(curl -s $node_release_url | jq .tag_name -r)
+            echo "node_rev1=$latest_tag" >> $GITHUB_OUTPUT
+            # Fetch more details
+            latest_name=$(curl -s $node_release_url | jq .name -r)
+            latest_url=$(curl -s $node_release_url | jq .html_url -r)
+            latest_archive=$(curl -s $node_release_url | jq .assets[0].browser_download_url -r)
+          
+            echo "Latest Release Name: $latest_name"
+            echo "Release URL: $latest_url"
+            echo "Download Archive URL: $latest_archive"
+          fi
+          
+
+      - name: Trigger the Buildkite pipeline - run sync tests on Mainnet
+        uses: 'buildkite/[email protected]'
+        env:
+          BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_API_ACCESS_TOKEN }}
+          PIPELINE: "input-output-hk/${{ github.event.inputs.buildkite_pipeline }}"
+          BRANCH: ${{ github.ref_name }}
+          MESSAGE: ':github: Triggered by GitHub Action'
+          AWS_DB_USERNAME: ${{ secrets.AWS_DB_USERNAME }}
+          AWS_DB_PASS: ${{ secrets.AWS_DB_PASS }}
+          AWS_DB_NAME: ${{ secrets.AWS_DB_NAME }}
+          AWS_DB_HOSTNAME: ${{ secrets.AWS_DB_HOSTNAME }}
+          BLOCKFROST_API_KEY: ${{ secrets.BLOCKFROST_API_KEY }}
+          BUILD_ENV_VARS: '{
+          "env":"${{ github.event.inputs.environment }}",
+          "node_rev1":"${{ steps.get_tag.outputs.node_rev1 }}",
+          "node_rev2":"${{ github.event.inputs.node_rev2 }}",          
+          "tag_no1":"${{ github.event.inputs.tag_no1 }}",
+          "tag_no2":"${{ github.event.inputs.tag_no2 }}",
+          "node_topology1":"${{ github.event.inputs.node_topology1 }}",
+          "node_topology2":"${{ github.event.inputs.node_topology2 }}",
+          "node_start_arguments1":"None",
+          "node_start_arguments2":"None",
+          "use_genesis_mode":"${{ github.event.inputs.use_genesis_mode }}",
+          "queue_name":"${{ github.event.inputs.buildkite_queue_name }}",
+          
+          "CARDANO_NODE_SOCKET_PATH":"/var/tmp/${{ github.event.inputs.buildkite_pipeline }}/node.socket",
+          "BLOCKFROST_API_KEY":"${{ secrets.BLOCKFROST_API_KEY }}",
+          "AWS_DB_USERNAME":"${{ secrets.AWS_DB_USERNAME }}",
+          "AWS_DB_PASS":"${{ secrets.AWS_DB_PASS }}",
+          "AWS_DB_NAME":"${{ secrets.AWS_DB_NAME }}",
+          "AWS_DB_HOSTNAME":"${{ secrets.AWS_DB_HOSTNAME }}"
+          }'