Merge branch 'develop' into dkijania/introduce_postgres_side_docker_job

automation/terraform/modules/testnet-alerts/templates/testnet-alert-rules.yml.tpl

@@ -193,7 +193,7 @@ groups:
       runbook: "https://www.notion.so/minaprotocol/LongFork-e65e5ad7437f4f4dbac201abbf9ace81"
 
   - alert: OldBestTip
-    expr: min by (testnet) ((time() - 1609459200) - Coda_Transition_frontier_best_tip_slot_time_sec ${rule_filter}) >= 15 * 180
+    expr: min by (testnet) ((time() - 1717545600) - Coda_Transition_frontier_best_tip_slot_time_sec ${rule_filter}) >= 15 * 180
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: critical
@@ -203,7 +203,7 @@ groups:
       runbook: "https://www.notion.so/minaprotocol/OldBestTip-8afa955101b642bd8356edfd0b03b640"
 
   - alert: NoNewSnarks
-    expr: min by (testnet) ((time() - 1609459200) - Coda_Snark_work_useful_snark_work_received_time_sec ${rule_filter}) >= 2 * 180 and max by (testnet) (Coda_Snark_work_pending_snark_work ${rule_filter}) != 0
+    expr: min by (testnet) ((time() - 1717545600) - Coda_Snark_work_useful_snark_work_received_time_sec ${rule_filter}) >= 2 * 180 and max by (testnet) (Coda_Snark_work_pending_snark_work ${rule_filter}) != 0
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: critical
@@ -213,7 +213,7 @@ groups:
       runbook: "https://www.notion.so/minaprotocol/NoNewSnarks-f86d27c81af54954b2fb61378bff9d4d"
 
   - alert: NoNewTransactions
-    expr: min by (testnet) ((time() - 1609459200) - Coda_Transaction_pool_useful_transactions_received_time_sec ${rule_filter}) >= 2 * 180
+    expr: min by (testnet) ((time() - 1717545600) - Coda_Transaction_pool_useful_transactions_received_time_sec ${rule_filter}) >= 2 * 180
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: warning
@@ -269,7 +269,7 @@ groups:
       runbook: "https://www.notion.so/minaprotocol/HighBlockGossipLatency-2096501c7cf34032b44e903ec1a4d79c"
 
   - alert: SomewhatOldBestTip
-    expr: count by (testnet) (((time() - 1609459200) - Coda_Transition_frontier_best_tip_slot_time_sec ${rule_filter}) >= 8 * 180) > 1
+    expr: count by (testnet) (((time() - 1717545600) - Coda_Transition_frontier_best_tip_slot_time_sec ${rule_filter}) >= 8 * 180) > 1
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: warning
@@ -580,7 +580,7 @@ groups:
       runbook: "https://www.notion.so/minaprotocol/LongFork-e65e5ad7437f4f4dbac201abbf9ace81"
 
   - alert: OldBestTip
-    expr: min by (testnet) ((time() - 1609459200) - Coda_Transition_frontier_best_tip_slot_time_sec {${berkeley_testnet},${synced_status_filter}} ) >= 15 * 180
+    expr: min by (testnet) ((time() - 1717545600) - Coda_Transition_frontier_best_tip_slot_time_sec {${berkeley_testnet},${synced_status_filter}} ) >= 15 * 180
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: critical
@@ -590,7 +590,7 @@ groups:
       runbook: "https://www.notion.so/minaprotocol/OldBestTip-8afa955101b642bd8356edfd0b03b640"
 
   - alert: NoNewSnarks
-    expr: min by (testnet) ((time() - 1609459200) - Coda_Snark_work_useful_snark_work_received_time_sec {${berkeley_testnet},${synced_status_filter}} ) >= 2 * 180 and max by (testnet) (Coda_Snark_work_pending_snark_work {${berkeley_testnet},${synced_status_filter}} ) != 0
+    expr: min by (testnet) ((time() - 1717545600) - Coda_Snark_work_useful_snark_work_received_time_sec {${berkeley_testnet},${synced_status_filter}} ) >= 2 * 180 and max by (testnet) (Coda_Snark_work_pending_snark_work {${berkeley_testnet},${synced_status_filter}} ) != 0
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: critical
@@ -600,7 +600,7 @@ groups:
       runbook: "https://www.notion.so/minaprotocol/NoNewSnarks-f86d27c81af54954b2fb61378bff9d4d"
 
   - alert: NoNewTransactions
-    expr: min by (testnet) ((time() - 1609459200) - Coda_Transaction_pool_useful_transactions_received_time_sec {${berkeley_testnet},${synced_status_filter}} ) >= 2 * 180
+    expr: min by (testnet) ((time() - 1717545600) - Coda_Transaction_pool_useful_transactions_received_time_sec {${berkeley_testnet},${synced_status_filter}} ) >= 2 * 180
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: warning
@@ -650,7 +650,7 @@ groups:
       runbook: "https://www.notion.so/minaprotocol/HighBlockGossipLatency-2096501c7cf34032b44e903ec1a4d79c"
 
   - alert: SomewhatOldBestTip
-    expr: count by (testnet) (((time() - 1609459200) - Coda_Transition_frontier_best_tip_slot_time_sec {${berkeley_testnet},${synced_status_filter}} ) >= 8 * 180) > 1
+    expr: count by (testnet) (((time() - 1717545600) - Coda_Transition_frontier_best_tip_slot_time_sec {${berkeley_testnet},${synced_status_filter}} ) >= 8 * 180) > 1
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: warning

buildkite/scripts/rosetta-integration-tests.sh

@@ -93,8 +93,8 @@ cat <<EOF >"$MINA_CONFIG_FILE"
 {
   "genesis": { "genesis_state_timestamp": "$CURRENT_TIME" },
   "proof": { "block_window_duration_ms": 20000 },
+  "daemon": { "network_id": "${MINA_NETWORK}" },
   "ledger": {
-    "name": "${MINA_NETWORK}",
     "accounts": [
       { "pk": "${BLOCK_PRODUCER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },
       { "pk": "${SNARK_PRODUCER_PK}", "balance": "2000000", "delegate": "${BLOCK_PRODUCER_PUB_KEY}", "sk": null },

buildkite/src/Command/RunInToolchain.dhall

@@ -37,7 +37,6 @@ let runInToolchainBuster : List Text -> Text -> List Cmd.Type =
   \(innerScript : Text) ->
     runInToolchainImage ContainerImages.minaToolchainBuster environment innerScript 
 
-
 let runInToolchain : List Text -> Text -> List Cmd.Type =
   \(environment : List Text) ->
   \(innerScript : Text) ->

docs/upgrading-to-berkeley.md

@@ -63,10 +63,73 @@ done
 
 ## Verifying packaged configuration
 
+
 A script is installed (from `./scripts/mina-verify-packaged-fork-config`) that automates this process. If you want to verify that an installed Mina package was generated from the same configuration as the one exported earlier, it is as easy as:
 
 ```
 mina-verify-packaged-fork-config (mainnet|devnet) fork_config.json /tmp/mina-verification
 ```
 
 Many of the script inputs are environment variables that default to the locations used by the debs. If you are building from source, inspect the script, determine what you need to change, and then run it.
+
+Here are some general instructions that should help you get most of the way there.
+
+### Package Verification
+
+#### Environment Setup
+1. **Use a Debian Environment:** 
+    - It is recommended to perform package verification in a Debian environment because macOS Intel emulation doesn’t support the required AVX instructions for libp2p.
+
+2. **Environment Preparation:**
+    - **Install mina-create-legacy-genesis**
+      ```bash
+      echo "deb [trusted=yes] http://packages.o1test.net bullseye stable" > /etc/apt/sources.list.d/o1.list
+      apt-get update
+      apt-get install mina-create-legacy-genesis=1.4.1-97f7d8c
+
+    - **Direct Installation Script:**
+    - ***NOTE this currently needs to be run as root***
+        ``` bash
+        #!/bin/bash
+        export CONFIG_JSON_GZ_URL="https://storage.googleapis.com/tmp-hardfork-testing/fork-config-3NLRTfY4kZyJtvaP4dFenDcxfoMfT3uEpkWS913KkeXLtziyVd15.json.gz"
+        export GENESIS_TIMESTAMP="2024-06-05T00:00:00Z"
+        export NETWORK_NAME="mainnet"
+        mkdir -p genesis_ledgers
+        cp /var/lib/coda/config_93e02797.json genesis_ledgers/mainnet.json
+        curl https://storage.googleapis.com/tmp-hardfork-testing/fork-config-3NLRTfY4kZyJtvaP4dFenDcxfoMfT3uEpkWS913KkeXLtziyVd15.json.gz > config.json.gz && gunzip config.json.gz && mina-verify-packaged-fork-config mainnet config.json /workdir/verification 
+        ```
+
+- **Prepare Environment Variables:** Set up the necessary environment variables manually to match the expected configuration for verification.
+- **Create `genesis_ledgers` Directory:** Temporarily, create a new directory named `genesis_ledgers` in the working directory and place the daemon config for the required chain, typically `mainnet.json` there. Alternatively, make sure that `genesis_ledgers/mainnet.json` is present. The script assumes that it is running in a directory where this is present.
+
+
+3. **Set Environment Variables:**
+    - **GSUTIL Path:**
+        ```bash
+        export GSUTIL=/usr/bin/gsutil
+        ```
+    - **Download Block Data:**
+        - Example command to download the block data:
+            ```bash
+            gsutil cp gs://mina_network_block_data/mainnet-<some-version>.json ./block_data.json
+            export PRECOMPUTED_FORK_BLOCK=./block_data.json
+            ```
+    - **Verify Additional Variables:**
+        - Confirm the following variables are properly set:
+            ```bash
+            CONFIG_JSON_GZ_URL=<url to fork config zip> ex: "https://storage.googleapis.com/fork-config-dryrun.json.gz"
+            GENESIS_TIMESTAMP=<same ts as your genesis config> ex: "2024-05-03T00:00:00Z"
+            NETWORK_NAME=<this should match the intended network> ex: mainnet
+            PRECOMPUTED_BLOCK_GS_PREFIX=<url to the network block data> ex: "gs://mina_network_block_data/mainnet"
+            ```
+
+#### Verification Process
+1. **Run the Verification Command:**
+    - If the fork config is not already on your machine, you can download it with a command similiar to:
+        ```bash
+        curl https://storage.googleapis.com/fork-config-dryrun.json.gz > config.json.gz && gunzip config.json.gz && mina-verify-packaged-fork-config mainnet config.json /workdir/verification gs://mina_network_block_data/mainnet-pre-hf-dry-run-2
+        ```
+    - If the fork config is already available locally, use:
+        ```bash
+        mina-verify-packaged-fork-config mainnet fork-config-dryrun.json /tmp/mina-verification
+        ```

scripts/hardfork/build-and-test.sh

@@ -93,10 +93,10 @@ if [[ $# -gt 0 ]]; then
   git submodule sync --recursive
   git submodule update --init --recursive
 fi
-git apply "$SCRIPT_DIR"/localnet-patches/berkeley-{1,2}.patch
+git apply "$SCRIPT_DIR"/localnet-patches/berkeley.patch
 nix "${NIX_OPTS[@]}" build "$INIT_DIR?submodules=1#devnet" --out-link "$INIT_DIR/fork-devnet"
 nix "${NIX_OPTS[@]}" build "$INIT_DIR?submodules=1#devnet.genesis" --out-link "$INIT_DIR/fork-devnet"
-git apply -R "$SCRIPT_DIR"/localnet-patches/berkeley-{1,2}.patch
+git apply -R "$SCRIPT_DIR"/localnet-patches/berkeley.patch
 
 if [[ "$NIX_CACHE_GCP_ID" != "" ]] && [[ "$NIX_CACHE_GCP_SECRET" != "" ]]; then
   mkdir -p $HOME/.aws