Merge pull request #15870 from MinaProtocol/merge/compatible_to_devel…

…op_260724 Merge compatible to develop 260724
MinaProtocol · Jul 28, 2024 · e809a9e · e809a9e
2 parents 75cafde + 8d0aad9
commit e809a9e
Show file tree

Hide file tree

Showing 247 changed files with 1,615 additions and 1,522 deletions.
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -5,7 +5,6 @@
 /scripts/ @MinaProtocol/infra-eng-reviewers @MinaProtocol/protocol-eng-reviewers
 /Makefile @MinaProtocol/infra-eng-reviewers
 /src/config/dev.mlh @MinaProtocol/infra-eng-reviewers @MinaProtocol/protocol-eng-reviewers
-/src/config/testnet_postake_medium_curves.mlh @MinaProtocol/infra-eng-reviewers @MinaProtocol/protocol-eng-reviewers
 /CODE_OF_CONDUCT.md @bkase
 /CONTRIBUTING.md @MinaProtocol/product-eng-reviewers
 /LICENSE @bkase

diff --git a/Makefile b/Makefile
@@ -193,9 +193,6 @@ check-format: ocaml_checks
 check-snarky-submodule:
 	./scripts/check-snarky-submodule.sh
 
-check-proof-systems-submodule:
-	./scripts/check-proof-systems-submodule.sh
-
 #######################################
 ## Environment setup
 

diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh
@@ -6,7 +6,8 @@ echo "Exporting Variables: "
 export MINA_REPO="https://github.com/MinaProtocol/mina.git"
 
 function find_most_recent_numeric_tag() {
-    git fetch --tags
+    # We use the --prune flag because we've had problems with buildkite agents getting conflicting results here
+    git fetch --tags --prune --prune-tags --force
     TAG=$(git describe --always --abbrev=0 $1 | sed 's!/!-!g; s!_!-!g; s!#!-!g')
     if [[ $TAG != [0-9]* ]]; then
         TAG=$(find_most_recent_numeric_tag $TAG~)

diff --git a/buildkite/scripts/unit-test.sh b/buildkite/scripts/unit-test.sh
@@ -14,6 +14,7 @@ source ~/.profile
 
 export MINA_LIBP2P_PASS="naughty blue worm"
 export NO_JS_BUILD=1 # skip some JS targets which have extra implicit dependencies
+export LAGRANGE_CACHE_DIR="/tmp/lagrange-cache"
 
 echo "--- Make build"
 export LIBP2P_NIXLESS=1 PATH=/usr/lib/go/bin:$PATH GO=/usr/lib/go/bin/go
@@ -29,6 +30,7 @@ time dune runtest "src/app/print_blockchain_snark_vk" --profile="${profile}" -j1
 # be updated.
 export ERROR_ON_PROOF=true
 
+
 # Note: By attempting a re-run on failure here, we can avoid rebuilding and
 # skip running all of the tests that have already succeeded, since dune will
 # only retry those tests that failed.

diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall
@@ -18,7 +18,6 @@ let commands =
       [ Cmd.run "./scripts/lint_codeowners.sh"
       , Cmd.run "./scripts/lint_rfcs.sh"
       , Cmd.run "make check-snarky-submodule"
-      , Cmd.run "./scripts/lint_preprocessor_deps.sh"
       ]
 
 in  Pipeline.build

diff --git a/flake.nix b/flake.nix
@@ -57,22 +57,34 @@
         ref = r: "[34;1m${r}[31;1m";
         command = c: "[37;1m${c}[31;1m";
       in lib.warnIf (!builtins.all (x: x)
-        (map (x: builtins.pathExists ./${x} && builtins.readDir ./${x} != { }) submodules)) ''
-          Some submodules are missing, you may get errors. Consider one of the following:
-          - run ${command "nix/pin.sh"} and use "${ref "mina"}" flake ref, e.g. ${command "nix develop mina"} or ${command "nix build mina"};
-          - use "${ref "git+file://$PWD?submodules=1"}";
-          - use "${ref "git+https://github.com/minaprotocol/mina?submodules=1"}";
-          - use non-flake commands like ${command "nix-build"} and ${command "nix-shell"}.
-        '';
+        (map (x: builtins.pathExists ./${x} && builtins.readDir ./${x} != { })
+          submodules)) ''
+            Some submodules are missing, you may get errors. Consider one of the following:
+            - run ${command "nix/pin.sh"} and use "${
+              ref "mina"
+            }" flake ref, e.g. ${command "nix develop mina"} or ${
+              command "nix build mina"
+            };
+            - use "${ref "git+file://$PWD?submodules=1"}";
+            - use "${
+              ref "git+https://github.com/minaprotocol/mina?submodules=1"
+            }";
+            - use non-flake commands like ${command "nix-build"} and ${
+              command "nix-shell"
+            }.
+          '';
       # Only get the ocaml stuff, to reduce the amount of unnecessary rebuilds
-      ocaml-src =
-        with inputs.nix-filter.lib;
-          filter {
-            root = ./.;
-            include =
-              [ (inDirectory "src") "dune" "dune-project"
-                "./graphql_schema.json" "opam.export" ];
-          };
+      ocaml-src = with inputs.nix-filter.lib;
+        filter {
+          root = ./.;
+          include = [
+            (inDirectory "src")
+            "dune"
+            "dune-project"
+            "./graphql_schema.json"
+            "opam.export"
+          ];
+        };
     in {
       overlays = {
         misc = import ./nix/misc.nix;
@@ -223,14 +235,13 @@
               label = "Run ${test} integration test";
               depends_on = [ "push_mina-image-full" ]
                 ++ lib.optional with-archive "push_mina-archive-image-full";
-              "if" = ''build.pull_request.labels includes "nix-integration-tests"'';
+              "if" =
+                ''build.pull_request.labels includes "nix-integration-tests"'';
               retry = {
-                automatic = [
-                  {
-                    exit_status = "*";
-                    limit = 3;
-                  }
-                ];
+                automatic = [{
+                  exit_status = "*";
+                  limit = 3;
+                }];
               };
             };
         in {
@@ -261,10 +272,12 @@
         };
     } // utils.lib.eachDefaultSystem (system:
       let
-	rocksdbOverlay = pkgs: prev:
-          if prev.stdenv.isDarwin then
-            { rocksdb-mina = pkgs.rocksdb; }
-          else { rocksdb-mina = pkgs.rocksdb511; };
+        rocksdbOverlay = pkgs: prev:
+          if prev.stdenv.isDarwin then {
+            rocksdb-mina = pkgs.rocksdb;
+          } else {
+            rocksdb-mina = pkgs.rocksdb511;
+          };
 
         # nixpkgs with all relevant overlays applied
         pkgs = nixpkgs.legacyPackages.${system}.extend
@@ -279,7 +292,7 @@
                   nodejs = pkgs.nodejs-16_x;
                 };
             })
-          ] ++ builtins.attrValues self.overlays ++ [ rocksdbOverlay ] ));
+          ] ++ builtins.attrValues self.overlays ++ [ rocksdbOverlay ]));
 
         checks = import ./nix/checks.nix inputs pkgs;
 
@@ -302,12 +315,14 @@
         # Main user-facing binaries.
         packages = rec {
           inherit (ocamlPackages)
-            mina devnet mainnet mina_tests mina-ocaml-format mina_client_sdk test_executive with-instrumentation;
+            mina devnet mainnet mina_tests mina-ocaml-format mina_client_sdk
+            test_executive with-instrumentation;
           inherit (pkgs)
-            libp2p_helper kimchi_bindings_stubs snarky_js leaderboard
-            validation trace-tool zkapp-cli;
+            libp2p_helper kimchi_bindings_stubs snarky_js leaderboard validation
+            trace-tool zkapp-cli;
           inherit (dockerImages)
-            mina-image-slim mina-image-full mina-archive-image-full mina-image-instr-full; 
+            mina-image-slim mina-image-full mina-archive-image-full
+            mina-image-instr-full;
           mina-deb = debianPackages.mina;
           default = mina;
         };
@@ -387,5 +402,7 @@
         });
 
         inherit checks;
+
+        formatter = pkgs.nixfmt;
       });
 }
diff --git a/graphql_schema.json b/graphql_schema.json
@@ -5497,6 +5497,29 @@
           "enumValues": null,
           "possibleTypes": null
         },
+        {
+          "kind": "ENUM",
+          "name": "Encoding",
+          "description": null,
+          "fields": null,
+          "inputFields": null,
+          "interfaces": null,
+          "enumValues": [
+            {
+              "name": "JSON",
+              "description": null,
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
+            {
+              "name": "BASE64",
+              "description": null,
+              "isDeprecated": false,
+              "deprecationReason": null
+            }
+          ],
+          "possibleTypes": null
+        },
         {
           "kind": "INPUT_OBJECT",
           "name": "VrfEvaluationInput",
@@ -15348,6 +15371,34 @@
               },
               "isDeprecated": false,
               "deprecationReason": null
+            },
+            {
+              "name": "protocolState",
+              "description":
+                "Get the current protocol state, optionally encoded in Base64",
+              "args": [
+                {
+                  "name": "encoding",
+                  "description": "Encoding format (JSON or BASE64)",
+                  "type": {
+                    "kind": "ENUM",
+                    "name": "Encoding",
+                    "ofType": null
+                  },
+                  "defaultValue": null
+                }
+              ],
+              "type": {
+                "kind": "NON_NULL",
+                "name": null,
+                "ofType": {
+                  "kind": "SCALAR",
+                  "name": "String",
+                  "ofType": null
+                }
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
             }
           ],
           "inputFields": null,

diff --git a/nix/checks.nix b/nix/checks.nix
@@ -1,5 +1,4 @@
 inputs: pkgs: {
-
   # todo: Fast
   lint-codeowners = pkgs.stdenv.mkDerivation {
     # todo: filter source
@@ -20,13 +19,6 @@ inputs: pkgs: {
     touch $out
   '';
   # todo: ./scripts/check-snarky-submodule.sh # submodule issue
-  lint-preprocessor-deps = pkgs.runCommand "lint-preprocessor-deps" {
-    meta.checkDescription = "preprocessor deps";
-  } ''
-    ln -s ${../src} ./src
-    bash ${../scripts/lint_preprocessor_deps.sh}
-    touch $out
-  '';
   # - compare ci diff_types
   # - compare_ci_diff_binables
 

diff --git a/nix/debian.nix b/nix/debian.nix
@@ -1,4 +1,4 @@
-{ rpmDebUtils, ocamlPackages_mina }: {
+{ rpmDebUtils, ocamlPackages_mina, }: {
   # FIXME: This package basically just wraps some nix store paths in the .deb format.
   # It works, but has some problems.
   # In particular, if there's already Nix installed on the target system, it won't work.

diff --git a/nix/docker.nix b/nix/docker.nix
@@ -1,6 +1,7 @@
 { lib, dockerTools, buildEnv, ocamlPackages_mina, runCommand, dumb-init
-, coreutils, findutils, bashInteractive, python3, libp2p_helper, procps, postgresql, curl
-, jq, stdenv, rsync, bash, gnutar, gzip, currentTime, flockenzeit }:
+, coreutils, findutils, bashInteractive, python3, libp2p_helper, procps
+, postgresql, curl, jq, stdenv, rsync, bash, gnutar, gzip, currentTime
+, flockenzeit, }:
 let
   created = flockenzeit.lib.ISO-8601 currentTime;
 
@@ -47,31 +48,31 @@ let
     '';
   };
 
-  mkFullImage = name: packages: additional_envs: dockerTools.streamLayeredImage {
-    name = "${name}-full";
-    inherit created;
-    contents = [
-      dumb-init
-      coreutils
-      findutils 
-      bashInteractive
-      python3
-      libp2p_helper
-      procps
-      curl
-      jq
-    ] ++ packages;
-    extraCommands = ''
-      mkdir root tmp
-      chmod 777 tmp
-    '';
-    config = {
-      env = [ "MINA_TIME_OFFSET=0" ] ++ additional_envs;
-      WorkingDir = "/root";
-      cmd = [ "/bin/dumb-init" "/entrypoint.sh" ];
+  mkFullImage = name: packages: additional_envs:
+    dockerTools.streamLayeredImage {
+      name = "${name}-full";
+      inherit created;
+      contents = [
+        dumb-init
+        coreutils
+        findutils
+        bashInteractive
+        python3
+        libp2p_helper
+        procps
+        curl
+        jq
+      ] ++ packages;
+      extraCommands = ''
+        mkdir root tmp
+        chmod 777 tmp
+      '';
+      config = {
+        env = [ "MINA_TIME_OFFSET=0" ] ++ additional_envs;
+        WorkingDir = "/root";
+        cmd = [ "/bin/dumb-init" "/entrypoint.sh" ];
+      };
     };
-  };
-
 in {
   mina-image-slim = dockerTools.streamLayeredImage {
     name = "mina";
@@ -87,7 +88,7 @@ in {
     mina.mainnet
     mina.genesis
   ]);
-  
+
   # Image with enhanced binary capable of generating coverage report on mina exit
   # For more details please visit: https://github.com/aantron/bisect_ppx/blob/master/doc/advanced.md#sigterm-handling
   mina-image-instr-full = mkFullImage "mina-instr" (with ocamlPackages_mina; [
@@ -97,13 +98,14 @@ in {
     with_instrumentation.out
     mina.mainnet
     mina.genesis
-  ]) ["BISECT_SIGTERM=yes"];
+  ]) [ "BISECT_SIGTERM=yes" ];
 
-  mina-archive-image-full = mkFullImage "mina-archive" (with ocamlPackages_mina; [
-    mina-archive-scripts
-    gnutar
-    gzip
+  mina-archive-image-full = mkFullImage "mina-archive"
+    (with ocamlPackages_mina; [
+      mina-archive-scripts
+      gnutar
+      gzip
 
-    mina.archive
-  ]);
-}
+      mina.archive
+    ]);
+}