From 80779acbe13d931b65647eb074461d5a82abd2bb Mon Sep 17 00:00:00 2001 From: svv232 Date: Mon, 16 Sep 2024 17:53:41 -0400 Subject: [PATCH 01/14] adding a default value for minimum user command so the current runtime config is useable --- src/lib/runtime_config/runtime_config.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/runtime_config/runtime_config.ml b/src/lib/runtime_config/runtime_config.ml index 85c8b45b332..ca1ff7d1912 100644 --- a/src/lib/runtime_config/runtime_config.ml +++ b/src/lib/runtime_config/runtime_config.ml @@ -473,7 +473,7 @@ module Json_layout = struct ; zkapp_cmd_limit_hardcap : int option [@default None] ; slot_tx_end : int option [@default None] ; slot_chain_end : int option [@default None] - ; minimum_user_command_fee : Currency.Fee.t option + ; minimum_user_command_fee : Currency.Fee.t option [@default None] ; network_id : string option [@default None] } [@@deriving yojson, fields] From 236f7078f7bb80c94cbbca9d6244274e3649823d Mon Sep 17 00:00:00 2001 From: martyall Date: Sun, 8 Sep 2024 02:30:57 -0700 Subject: [PATCH 02/14] src/lib compiles --- .../bootstrap_controller.ml | 11 +- .../bootstrap_controller.mli | 2 + src/lib/fake_network/fake_network.ml | 6 +- src/lib/fake_network/fake_network.mli | 3 + src/lib/gossip_net/libp2p.ml | 7 +- src/lib/ledger_catchup/dune | 1 + src/lib/ledger_catchup/ledger_catchup.ml | 2 + src/lib/ledger_catchup/ledger_catchup.mli | 2 + src/lib/ledger_catchup/normal_catchup.ml | 9 ++ src/lib/ledger_catchup/super_catchup.ml | 16 ++- .../transition_frontier_components_intf.ml | 2 + src/lib/mina_lib/mina_lib.ml | 8 ++ src/lib/mina_lib/mina_lib.mli | 2 + src/lib/mina_lib/tests/tests.ml | 6 + src/lib/mina_metrics/mina_metrics.mli | 30 ++--- .../mina_metrics/no_metrics/mina_metrics.ml | 30 ++--- .../prometheus_metrics/mina_metrics.ml | 73 ++++++------- src/lib/mina_net2/mina_net2.ml | 8 +- src/lib/mina_net2/mina_net2.mli | 1 + src/lib/mina_net2/subscription.ml | 6 +- src/lib/mina_net2/subscription.mli | 1 + src/lib/mina_net2/tests/all_ipc.ml | 6 +- src/lib/mina_net2/tests/dune | 1 + src/lib/mina_net2/tests/tests.ml | 9 +- src/lib/mina_net2/validation_callback.ml | 103 +++++++++++++----- src/lib/mina_net2/validation_callback.mli | 6 +- src/lib/mina_networking/mina_networking.ml | 2 + src/lib/mina_networking/mina_networking.mli | 2 + src/lib/mina_networking/rpcs.ml | 2 + src/lib/network_pool/intf.ml | 2 + src/lib/network_pool/network_pool_base.ml | 11 +- src/lib/network_pool/pool_sink.ml | 11 +- src/lib/network_pool/snark_pool.ml | 7 ++ src/lib/network_pool/test.ml | 5 + src/lib/network_pool/transaction_pool.ml | 4 + src/lib/sync_handler/sync_handler.ml | 2 + .../transaction_inclusion_status.ml | 4 + .../transition_frontier_controller.ml | 2 + src/lib/transition_handler/block_sink.ml | 27 ++++- src/lib/transition_handler/block_sink.mli | 2 + .../transition_handler/catchup_scheduler.ml | 22 +++- src/lib/transition_handler/processor.ml | 25 ++++- .../transition_router/transition_router.ml | 2 + 43 files changed, 350 insertions(+), 133 deletions(-) diff --git a/src/lib/bootstrap_controller/bootstrap_controller.ml b/src/lib/bootstrap_controller/bootstrap_controller.ml index 7d4aac346e1..a7f8a13db30 100644 --- a/src/lib/bootstrap_controller/bootstrap_controller.ml +++ b/src/lib/bootstrap_controller/bootstrap_controller.ml @@ -17,6 +17,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end type Structured_log_events.t += Bootstrap_complete @@ -712,6 +714,8 @@ let%test_module "Bootstrap_controller tests" = let constraint_constants = precomputed_values.constraint_constants + let compile_config = Mina_compile_config.For_unit_tests.t + module Context = struct let logger = Logger.create () @@ -721,6 +725,8 @@ let%test_module "Bootstrap_controller tests" = Genesis_constants.For_unit_tests.Constraint_constants.t let consensus_constants = precomputed_values.consensus_constants + + let compile_config = compile_config end let verifier = @@ -769,7 +775,8 @@ let%test_module "Bootstrap_controller tests" = let%bind fake_network = Fake_network.Generator.( gen ~precomputed_values ~verifier ~max_frontier_length - [ fresh_peer; fresh_peer ] ~use_super_catchup:false) + ~compile_config [ fresh_peer; fresh_peer ] + ~use_super_catchup:false) in let%map make_branch = Transition_frontier.Breadcrumb.For_tests.gen_seq ~precomputed_values @@ -899,7 +906,7 @@ let%test_module "Bootstrap_controller tests" = Quickcheck.test ~trials:1 Fake_network.Generator.( gen ~precomputed_values ~verifier ~max_frontier_length - ~use_super_catchup:false + ~use_super_catchup:false ~compile_config [ fresh_peer ; peer_with_branch ~frontier_branch_size:((max_frontier_length * 2) + 2) diff --git a/src/lib/bootstrap_controller/bootstrap_controller.mli b/src/lib/bootstrap_controller/bootstrap_controller.mli index 481f8d58b24..38db030e800 100644 --- a/src/lib/bootstrap_controller/bootstrap_controller.mli +++ b/src/lib/bootstrap_controller/bootstrap_controller.mli @@ -10,6 +10,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end type Structured_log_events.t += Bootstrap_complete [@@deriving register_event] diff --git a/src/lib/fake_network/fake_network.ml b/src/lib/fake_network/fake_network.ml index eb6080b6025..81995f715b2 100644 --- a/src/lib/fake_network/fake_network.ml +++ b/src/lib/fake_network/fake_network.ml @@ -14,6 +14,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end (* There must be at least 2 peers to create a network *) @@ -298,7 +300,7 @@ module Generator = struct let gen ?(logger = Logger.null ()) ~precomputed_values ~verifier ~max_frontier_length ~use_super_catchup - (configs : (peer_config, 'n num_peers) Gadt_lib.Vect.t) = + (configs : (peer_config, 'n num_peers) Gadt_lib.Vect.t) ~compile_config = (* TODO: Pass in *) let module Context = struct let logger = logger @@ -310,6 +312,8 @@ module Generator = struct let consensus_constants = precomputed_values.Precomputed_values.consensus_constants + + let compile_config = compile_config end in let open Quickcheck.Generator.Let_syntax in let%map states = diff --git a/src/lib/fake_network/fake_network.mli b/src/lib/fake_network/fake_network.mli index cee6df672f8..b3d041cf7f6 100644 --- a/src/lib/fake_network/fake_network.mli +++ b/src/lib/fake_network/fake_network.mli @@ -10,6 +10,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end (* There must be at least 2 peers to create a network *) @@ -98,5 +100,6 @@ module Generator : sig -> max_frontier_length:int -> use_super_catchup:bool -> (peer_config, 'n num_peers) Vect.t + -> compile_config:Mina_compile_config.t -> 'n num_peers t Generator.t end diff --git a/src/lib/gossip_net/libp2p.ml b/src/lib/gossip_net/libp2p.ml index a99ad2bc5c1..dbcf212c499 100644 --- a/src/lib/gossip_net/libp2p.ml +++ b/src/lib/gossip_net/libp2p.ml @@ -52,6 +52,7 @@ module Config = struct ; mutable keypair : Mina_net2.Keypair.t option ; all_peers_seen_metric : bool ; known_private_ip_nets : Core.Unix.Cidr.t list + ; block_window_duration : Time.Span.t } [@@deriving make] end @@ -219,7 +220,8 @@ module Make (Rpc_interface : RPC_INTERFACE) : ctx first_peer_ivar high_connectivity_ivar ~added_seeds ~pids ~on_unexpected_termination ~sinks: - (Message.Any_sinks (sinksM, (sink_block, sink_tx, sink_snark_work))) = + (Message.Any_sinks (sinksM, (sink_block, sink_tx, sink_snark_work))) + ~block_window_duration = let module Sinks = (val sinksM) in let ctr = ref 0 in let record_peer_connection () = @@ -256,7 +258,7 @@ module Make (Rpc_interface : RPC_INTERFACE) : ~all_peers_seen_metric:config.all_peers_seen_metric ~on_peer_connected:(fun _ -> record_peer_connection ()) ~on_peer_disconnected:ignore ~logger:config.logger ~conf_dir - ~pids () ) ) + ~pids ~block_window_duration () ) ) with | Ok (Ok net2) -> ( let open Mina_net2 in @@ -628,6 +630,7 @@ module Make (Rpc_interface : RPC_INTERFACE) : create_libp2p ~allow_multiple_instances config rpc_handlers first_peer_ivar high_connectivity_ivar ~added_seeds ~pids ~on_unexpected_termination:restart_libp2p ~sinks + ~block_window_duration:config.block_window_duration in on_libp2p_create libp2p ; Deferred.ignore_m libp2p and restart_libp2p () = don't_wait_for (start_libp2p ()) in diff --git a/src/lib/ledger_catchup/dune b/src/lib/ledger_catchup/dune index 013e42fe06a..66812ce2c05 100644 --- a/src/lib/ledger_catchup/dune +++ b/src/lib/ledger_catchup/dune @@ -68,4 +68,5 @@ mina_net2 internal_tracing mina_runtime_config + mina_compile_config )) diff --git a/src/lib/ledger_catchup/ledger_catchup.ml b/src/lib/ledger_catchup/ledger_catchup.ml index 5e452ab565e..599dbcf587b 100644 --- a/src/lib/ledger_catchup/ledger_catchup.ml +++ b/src/lib/ledger_catchup/ledger_catchup.ml @@ -9,6 +9,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end let run ~context:(module Context : CONTEXT) ~trust_system ~verifier ~network diff --git a/src/lib/ledger_catchup/ledger_catchup.mli b/src/lib/ledger_catchup/ledger_catchup.mli index 0a957e7e835..03f94401b79 100644 --- a/src/lib/ledger_catchup/ledger_catchup.mli +++ b/src/lib/ledger_catchup/ledger_catchup.mli @@ -13,6 +13,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end module Catchup_jobs : sig diff --git a/src/lib/ledger_catchup/normal_catchup.ml b/src/lib/ledger_catchup/normal_catchup.ml index 694ceea39c3..03fe1f65695 100644 --- a/src/lib/ledger_catchup/normal_catchup.ml +++ b/src/lib/ledger_catchup/normal_catchup.ml @@ -15,6 +15,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end (** [Ledger_catchup] is a procedure that connects a foreign external transition @@ -880,6 +882,8 @@ let%test_module "Ledger_catchup tests" = let constraint_constants = precomputed_values.constraint_constants + let compile_config = Mina_compile_config.For_unit_tests.t + let trust_system = Trust_system.null () let time_controller = Block_time.Controller.basic ~logger @@ -901,6 +905,8 @@ let%test_module "Ledger_catchup tests" = let constraint_constants = constraint_constants let consensus_constants = precomputed_values.consensus_constants + + let compile_config = compile_config end let downcast_transition transition = @@ -1025,6 +1031,7 @@ let%test_module "Ledger_catchup tests" = in gen ~precomputed_values ~verifier ~max_frontier_length ~use_super_catchup + ~compile_config:Mina_compile_config.For_unit_tests.t [ fresh_peer ; peer_with_branch ~frontier_branch_size:peer_branch_size ]) @@ -1046,6 +1053,7 @@ let%test_module "Ledger_catchup tests" = Fake_network.Generator.( gen ~precomputed_values ~verifier ~max_frontier_length ~use_super_catchup + ~compile_config:Mina_compile_config.For_unit_tests.t [ fresh_peer; peer_with_branch ~frontier_branch_size:1 ]) ~f:(fun network -> let open Fake_network in @@ -1061,6 +1069,7 @@ let%test_module "Ledger_catchup tests" = Fake_network.Generator.( gen ~precomputed_values ~verifier ~max_frontier_length ~use_super_catchup + ~compile_config:Mina_compile_config.For_unit_tests.t [ fresh_peer ; peer_with_branch ~frontier_branch_size:(max_frontier_length * 2) ]) diff --git a/src/lib/ledger_catchup/super_catchup.ml b/src/lib/ledger_catchup/super_catchup.ml index d544f34c3af..2e6adb55bb8 100644 --- a/src/lib/ledger_catchup/super_catchup.ml +++ b/src/lib/ledger_catchup/super_catchup.ml @@ -16,6 +16,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end (** [Ledger_catchup] is a procedure that connects a foreign external transition @@ -1433,6 +1435,8 @@ let%test_module "Ledger_catchup tests" = let use_super_catchup = true + let compile_config = Mina_compile_config.For_unit_tests.t + let verifier = Async.Thread_safe.block_on_async_exn (fun () -> Verifier.create ~logger ~proof_level ~constraint_constants @@ -1448,6 +1452,8 @@ let%test_module "Ledger_catchup tests" = let constraint_constants = constraint_constants let consensus_constants = precomputed_values.consensus_constants + + let compile_config = compile_config end (* let mock_verifier = @@ -1626,7 +1632,7 @@ let%test_module "Ledger_catchup tests" = Int.gen_incl (max_frontier_length / 2) (max_frontier_length - 1) in gen ~precomputed_values ~verifier ~max_frontier_length - ~use_super_catchup + ~use_super_catchup ~compile_config [ fresh_peer ; peer_with_branch ~frontier_branch_size:peer_branch_size ]) @@ -1646,7 +1652,7 @@ let%test_module "Ledger_catchup tests" = Quickcheck.test ~trials:1 Fake_network.Generator.( gen ~precomputed_values ~verifier ~max_frontier_length - ~use_super_catchup + ~use_super_catchup ~compile_config [ fresh_peer; peer_with_branch ~frontier_branch_size:1 ]) ~f:(fun network -> let open Fake_network in @@ -1662,7 +1668,7 @@ let%test_module "Ledger_catchup tests" = Quickcheck.test ~trials:1 Fake_network.Generator.( gen ~precomputed_values ~verifier ~max_frontier_length - ~use_super_catchup + ~use_super_catchup ~compile_config [ fresh_peer; peer_with_branch ~frontier_branch_size:1 ]) ~f:(fun network -> let open Fake_network in @@ -1679,7 +1685,7 @@ let%test_module "Ledger_catchup tests" = Quickcheck.test ~trials:1 Fake_network.Generator.( gen ~precomputed_values ~verifier ~max_frontier_length - ~use_super_catchup + ~use_super_catchup ~compile_config [ fresh_peer ; peer_with_branch ~frontier_branch_size:((max_frontier_length * 3) + 1) @@ -1757,7 +1763,7 @@ let%test_module "Ledger_catchup tests" = Quickcheck.test ~trials:1 Fake_network.Generator.( gen ~precomputed_values ~verifier ~max_frontier_length - ~use_super_catchup + ~use_super_catchup ~compile_config [ fresh_peer (* ; peer_with_branch ~frontier_branch_size:(max_frontier_length / 2) *) ; peer_with_branch_custom_rpc diff --git a/src/lib/mina_intf/transition_frontier_components_intf.ml b/src/lib/mina_intf/transition_frontier_components_intf.ml index 24717b6fe93..896fdd2ec6c 100644 --- a/src/lib/mina_intf/transition_frontier_components_intf.ml +++ b/src/lib/mina_intf/transition_frontier_components_intf.ml @@ -13,6 +13,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end module type Transition_handler_validator_intf = sig diff --git a/src/lib/mina_lib/mina_lib.ml b/src/lib/mina_lib/mina_lib.ml index 1e627f17b9d..84255950c33 100644 --- a/src/lib/mina_lib/mina_lib.ml +++ b/src/lib/mina_lib/mina_lib.ml @@ -1240,6 +1240,8 @@ module type CONTEXT = sig val zkapp_cmd_limit : int option ref val compaction_interval : Time.Span.t option + + val compile_config : Mina_compile_config.t end let context ~commit_id (config : Config.t) : (module CONTEXT) = @@ -1265,6 +1267,8 @@ let context ~commit_id (config : Config.t) : (module CONTEXT) = let zkapp_cmd_limit = config.zkapp_cmd_limit let compaction_interval = config.compile_config.compaction_interval + + let compile_config = config.compile_config end ) let start t = @@ -1494,6 +1498,7 @@ let create ~commit_id ?wallets (config : Config.t) = let catchup_mode = if config.super_catchup then `Super else `Normal in let constraint_constants = config.precomputed_values.constraint_constants in let consensus_constants = config.precomputed_values.consensus_constants in + let block_window_duration = config.compile_config.block_window_duration in let monitor = Option.value ~default:(Monitor.create ()) config.monitor in Async.Scheduler.within' ~monitor (fun () -> let set_itn_data (type t) (module M : Itn_settable with type t = t) (t : t) @@ -1834,6 +1839,7 @@ let create ~commit_id ?wallets (config : Config.t) = ~on_remote_push:notify_online ~log_gossip_heard: config.net_config.log_gossip_heard.transaction_pool_diff + ~block_window_duration in let snark_pool_config = Network_pool.Snark_pool.Resource_pool.make_config ~verifier @@ -1848,6 +1854,7 @@ let create ~commit_id ?wallets (config : Config.t) = ~on_remote_push:notify_online ~log_gossip_heard: config.net_config.log_gossip_heard.snark_pool_diff + ~block_window_duration in let block_reader, block_sink = Transition_handler.Block_sink.create @@ -1860,6 +1867,7 @@ let create ~commit_id ?wallets (config : Config.t) = ; consensus_constants ; genesis_constants = config.precomputed_values.genesis_constants ; constraint_constants + ; block_window_duration } in let sinks = (block_sink, tx_remote_sink, snark_remote_sink) in diff --git a/src/lib/mina_lib/mina_lib.mli b/src/lib/mina_lib/mina_lib.mli index c5e66de1693..d61b7181146 100644 --- a/src/lib/mina_lib/mina_lib.mli +++ b/src/lib/mina_lib/mina_lib.mli @@ -42,6 +42,8 @@ module type CONTEXT = sig val zkapp_cmd_limit : int option ref val compaction_interval : Time.Span.t option + + val compile_config : Mina_compile_config.t end exception Snark_worker_error of int diff --git a/src/lib/mina_lib/tests/tests.ml b/src/lib/mina_lib/tests/tests.ml index a9bf37afd4b..d41de09099d 100644 --- a/src/lib/mina_lib/tests/tests.ml +++ b/src/lib/mina_lib/tests/tests.ml @@ -100,6 +100,8 @@ let%test_module "Epoch ledger sync tests" = let time_controller = time_controller + let compile_config = Mina_compile_config.For_unit_tests.t + let commit_id = "not specified for unit test" let vrf_poll_interval = @@ -183,6 +185,7 @@ let%test_module "Epoch ledger sync tests" = ; consensus_constants ; genesis_constants = precomputed_values.genesis_constants ; constraint_constants + ; block_window_duration = compile_config.block_window_duration } in let _transaction_pool, tx_remote_sink, _tx_local_sink = @@ -197,6 +200,7 @@ let%test_module "Epoch ledger sync tests" = ~consensus_constants ~time_controller ~logger ~frontier_broadcast_pipe:frontier_broadcast_pipe_r ~on_remote_push ~log_gossip_heard:false + ~block_window_duration:compile_config.block_window_duration in let snark_remote_sink = let config = @@ -209,6 +213,7 @@ let%test_module "Epoch ledger sync tests" = ~consensus_constants ~time_controller ~logger ~frontier_broadcast_pipe:frontier_broadcast_pipe_r ~on_remote_push ~log_gossip_heard:false + ~block_window_duration:compile_config.block_window_duration in snark_remote_sink in @@ -268,6 +273,7 @@ let%test_module "Epoch ledger sync tests" = ; time_controller ; pubsub_v1 ; pubsub_v0 + ; block_window_duration = compile_config.block_window_duration } in Mina_networking.Gossip_net.( diff --git a/src/lib/mina_metrics/mina_metrics.mli b/src/lib/mina_metrics/mina_metrics.mli index d6bd6b4fde4..ba6a825165d 100644 --- a/src/lib/mina_metrics/mina_metrics.mli +++ b/src/lib/mina_metrics/mina_metrics.mli @@ -3,6 +3,10 @@ open Async_kernel val time_offset_sec : float +module type CONTEXT = sig + val block_window_duration : Time.Span.t +end + module Counter : sig type t @@ -121,15 +125,15 @@ module Network : sig val received : Counter.t - module Validation_time : sig + module Validation_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end - module Processing_time : sig + module Processing_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end - module Rejection_time : sig + module Rejection_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end end @@ -143,15 +147,15 @@ module Network : sig val received : Counter.t - module Validation_time : sig + module Validation_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end - module Processing_time : sig + module Processing_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end - module Rejection_time : sig + module Rejection_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end end @@ -165,15 +169,15 @@ module Network : sig val received : Counter.t - module Validation_time : sig + module Validation_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end - module Processing_time : sig + module Processing_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end - module Rejection_time : sig + module Rejection_time (Context : CONTEXT) : sig val update : Time.Span.t -> unit end end @@ -480,7 +484,7 @@ module Block_latency : sig val upload_to_gcloud_blocks : Gauge.t end - module Gossip_slots : sig + module Gossip_slots (Context : CONTEXT) : sig val v : Gauge.t val update : float -> unit @@ -488,7 +492,7 @@ module Block_latency : sig val clear : unit -> unit end - module Gossip_time : sig + module Gossip_time (Context : CONTEXT) : sig val v : Gauge.t val update : Time.Span.t -> unit @@ -496,7 +500,7 @@ module Block_latency : sig val clear : unit -> unit end - module Inclusion_time : sig + module Inclusion_time (Context : CONTEXT) : sig val v : Gauge.t val update : Time.Span.t -> unit @@ -504,7 +508,7 @@ module Block_latency : sig val clear : unit -> unit end - module Validation_acceptance_time : sig + module Validation_acceptance_time (Context : CONTEXT) : sig val v : Gauge.t val update : Time.Span.t -> unit diff --git a/src/lib/mina_metrics/no_metrics/mina_metrics.ml b/src/lib/mina_metrics/no_metrics/mina_metrics.ml index 8c32428ce10..ea0e73c87b6 100644 --- a/src/lib/mina_metrics/no_metrics/mina_metrics.ml +++ b/src/lib/mina_metrics/no_metrics/mina_metrics.ml @@ -3,6 +3,10 @@ open Async_kernel let time_offset_sec = 1609459200. +module type CONTEXT = sig + val block_window_duration : Time.Span.t +end + module Counter = struct type t = unit @@ -131,15 +135,15 @@ module Network = struct let received : Counter.t = () - module Validation_time = struct + module Validation_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end - module Processing_time = struct + module Processing_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end - module Rejection_time = struct + module Rejection_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end end @@ -153,15 +157,15 @@ module Network = struct let received : Counter.t = () - module Validation_time = struct + module Validation_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end - module Processing_time = struct + module Processing_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end - module Rejection_time = struct + module Rejection_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end end @@ -175,15 +179,15 @@ module Network = struct let received : Counter.t = () - module Validation_time = struct + module Validation_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end - module Processing_time = struct + module Processing_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end - module Rejection_time = struct + module Rejection_time (Context : CONTEXT) = struct let update : Time.Span.t -> unit = Fn.ignore end end @@ -492,7 +496,7 @@ module Block_latency = struct let upload_to_gcloud_blocks : Gauge.t = () end - module Gossip_slots = struct + module Gossip_slots (Context : CONTEXT) = struct let v : Gauge.t = () let update : float -> unit = fun _ -> () @@ -500,7 +504,7 @@ module Block_latency = struct let clear : unit -> unit = fun _ -> () end - module Gossip_time = struct + module Gossip_time (Context : CONTEXT) = struct let v : Gauge.t = () let update : Time.Span.t -> unit = fun _ -> () @@ -508,7 +512,7 @@ module Block_latency = struct let clear : unit -> unit = fun _ -> () end - module Inclusion_time = struct + module Inclusion_time (Context : CONTEXT) = struct let v : Gauge.t = () let update : Time.Span.t -> unit = fun _ -> () @@ -516,7 +520,7 @@ module Block_latency = struct let clear : unit -> unit = fun _ -> () end - module Validation_acceptance_time = struct + module Validation_acceptance_time (Context : CONTEXT) = struct let v : Gauge.t = () let update : Time.Span.t -> unit = fun _ -> () diff --git a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml index 124b19443d2..d9b47940622 100644 --- a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml +++ b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml @@ -11,7 +11,9 @@ open Async_kernel let time_offset_sec = 1609459200. -let block_window_duration = Node_config.block_window_duration +module type CONTEXT = sig + val block_window_duration : Time.Span.t +end (* textformat serialization and runtime metrics taken from github.com/mirage/prometheus:/app/prometheus_app.ml *) module TextFormat_0_0_4 = struct @@ -432,12 +434,10 @@ module Network = struct let help = "# of messages received" in Counter.v "messages_received" ~help ~namespace ~subsystem - module Delay_time_spec = struct - let tick_interval = - Core.Time.Span.of_ms (Int.to_float block_window_duration) + module Delay_time_spec (Context : CONTEXT) = struct + let tick_interval = Context.block_window_duration - let rolling_interval = - Core.Time.Span.of_ms (Int.to_float (block_window_duration * 20)) + let rolling_interval = Time.Span.scale Context.block_window_duration 20.0 end module Block = struct @@ -459,10 +459,10 @@ module Network = struct let help = "# of blocks received" in Counter.v "received" ~help ~namespace ~subsystem - module Validation_time = + module Validation_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -473,10 +473,10 @@ module Network = struct end) () - module Processing_time = + module Processing_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -488,10 +488,10 @@ module Network = struct end) () - module Rejection_time = + module Rejection_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -523,10 +523,10 @@ module Network = struct let help = "# of snark work received" in Counter.v "received" ~help ~namespace ~subsystem - module Validation_time = + module Validation_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -538,10 +538,10 @@ module Network = struct end) () - module Processing_time = + module Processing_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -553,10 +553,10 @@ module Network = struct end) () - module Rejection_time = + module Rejection_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -588,10 +588,10 @@ module Network = struct let help = "# of transactions received" in Counter.v "received" ~help ~namespace ~subsystem - module Validation_time = + module Validation_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -603,10 +603,10 @@ module Network = struct end) () - module Processing_time = + module Processing_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -618,10 +618,10 @@ module Network = struct end) () - module Rejection_time = + module Rejection_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec + include Delay_time_spec (Context) let subsystem = subsystem @@ -1442,19 +1442,16 @@ module Block_latency = struct Gauge.v "upload_to_gcloud_blocks" ~help ~namespace ~subsystem end - module Latency_time_spec = struct - let tick_interval = - Core.Time.Span.of_ms (Int.to_float (block_window_duration / 2)) + module Latency_time_spec (Context : CONTEXT) = struct + let tick_interval = Time.Span.scale Context.block_window_duration 0.5 - let rolling_interval = - Core.Time.Span.of_ms (Int.to_float (block_window_duration * 20)) + let rolling_interval = Time.Span.scale Context.block_window_duration 20.0 end - module Gossip_slots = + module Gossip_slots (Context : CONTEXT) = Moving_bucketed_average (struct - let bucket_interval = - Core.Time.Span.of_ms (Int.to_float (block_window_duration / 2)) + let bucket_interval = Time.Span.scale Context.block_window_duration 0.5 let num_buckets = 40 @@ -1475,10 +1472,10 @@ module Block_latency = struct end) () - module Gossip_time = + module Gossip_time (Context : CONTEXT) = Moving_time_average (struct - include Latency_time_spec + include Latency_time_spec (Context) let subsystem = subsystem @@ -1489,10 +1486,10 @@ module Block_latency = struct end) () - module Inclusion_time = + module Inclusion_time (Context : CONTEXT) = Moving_time_average (struct - include Latency_time_spec + include Latency_time_spec (Context) let subsystem = subsystem @@ -1504,10 +1501,10 @@ module Block_latency = struct end) () - module Validation_acceptance_time = + module Validation_acceptance_time (Context : CONTEXT) = Moving_time_average (struct - include Latency_time_spec + include Latency_time_spec (Context) let subsystem = subsystem diff --git a/src/lib/mina_net2/mina_net2.ml b/src/lib/mina_net2/mina_net2.ml index a649a5d978c..d2db2d0bbad 100644 --- a/src/lib/mina_net2/mina_net2.ml +++ b/src/lib/mina_net2/mina_net2.ml @@ -94,6 +94,7 @@ type t = ; mutable banned_ips : Unix.Inet_addr.t list ; peer_connected_callback : string -> unit ; peer_disconnected_callback : string -> unit + ; block_window_duration : Time.Span.t } let banned_ips t = t.banned_ips @@ -382,7 +383,8 @@ let handle_push_message t push_message = upon (O1trace.thread "validate_libp2p_gossip" (fun () -> Subscription.handle_and_validate sub ~validation_expiration - ~sender ~data ) ) + ~sender ~data + ~block_window_duration:t.block_window_duration ) ) (function | `Validation_timeout -> [%log' warn t.logger] @@ -543,7 +545,8 @@ let handle_push_message t push_message = Libp2p_ipc.undefined_union ~context:"DaemonInterface.PushMessage" n let create ?(allow_multiple_instances = false) ~all_peers_seen_metric ~logger - ~pids ~conf_dir ~on_peer_connected ~on_peer_disconnected () = + ~pids ~conf_dir ~on_peer_connected ~on_peer_disconnected + ~block_window_duration () = let open Deferred.Or_error.Let_syntax in let push_message_handler = ref (fun _msg -> @@ -574,6 +577,7 @@ let create ?(allow_multiple_instances = false) ~all_peers_seen_metric ~logger ; peer_disconnected_callback = (fun peer_id -> on_peer_disconnected (Peer.Id.unsafe_of_string peer_id)) ; protocol_handlers = Hashtbl.create (module String) + ; block_window_duration } in (push_message_handler := fun msg -> handle_push_message t msg) ; diff --git a/src/lib/mina_net2/mina_net2.mli b/src/lib/mina_net2/mina_net2.mli index 6b76e6a7049..f6f8cfd927b 100644 --- a/src/lib/mina_net2/mina_net2.mli +++ b/src/lib/mina_net2/mina_net2.mli @@ -139,6 +139,7 @@ val create : -> conf_dir:string -> on_peer_connected:(Peer.Id.t -> unit) -> on_peer_disconnected:(Peer.Id.t -> unit) + -> block_window_duration:Time.Span.t -> unit -> t Deferred.Or_error.t diff --git a/src/lib/mina_net2/subscription.ml b/src/lib/mina_net2/subscription.ml index 69ca2a4959e..61b15d96af5 100644 --- a/src/lib/mina_net2/subscription.ml +++ b/src/lib/mina_net2/subscription.ml @@ -50,7 +50,7 @@ let unsubscribe ~helper sub = else Deferred.Or_error.error_string "already unsubscribed" let handle_and_validate sub ~validation_expiration ~(sender : Peer.t) - ~data:raw_data = + ~data:raw_data ~block_window_duration = let open Libp2p_ipc.Reader.ValidationResult in let wrap_message data = if @@ -65,7 +65,9 @@ let handle_and_validate sub ~validation_expiration ~(sender : Peer.t) Validation_callback.create validation_expiration in let%bind () = sub.validator (wrap_message data) validation_callback in - match%map Validation_callback.await validation_callback with + match%map + Validation_callback.await ~block_window_duration validation_callback + with | Some `Accept -> `Validation_result Accept | Some `Reject -> diff --git a/src/lib/mina_net2/subscription.mli b/src/lib/mina_net2/subscription.mli index 6809c782440..022763618e3 100644 --- a/src/lib/mina_net2/subscription.mli +++ b/src/lib/mina_net2/subscription.mli @@ -29,6 +29,7 @@ val handle_and_validate : -> validation_expiration:Time_ns.t -> sender:Peer.t -> data:string + -> block_window_duration:Time.Span.t -> [ `Validation_result of Libp2p_ipc.validation_result | `Validation_timeout | `Decoding_error of Error.t ] diff --git a/src/lib/mina_net2/tests/all_ipc.ml b/src/lib/mina_net2/tests/all_ipc.ml index 4027c6cea31..de778d0f125 100644 --- a/src/lib/mina_net2/tests/all_ipc.ml +++ b/src/lib/mina_net2/tests/all_ipc.ml @@ -78,6 +78,9 @@ let%test_module "all-ipc test" = let bob_status = "This is major Tom to ground control\nI'm stepping through the door" + let block_window_duration = + Mina_compile_config.For_unit_tests.t.block_window_duration + type messages = { topic_a_msg_1 : string ; topic_a_msg_2 : string @@ -540,7 +543,8 @@ let%test_module "all-ipc test" = let%bind node = create ~all_peers_seen_metric:false ~logger:(Logger.extend logger [ ("name", `String local_name) ]) - ~conf_dir ~pids ~on_peer_connected ~on_peer_disconnected () + ~conf_dir ~pids ~on_peer_connected ~on_peer_disconnected + ~block_window_duration () >>| Or_error.ok_exn in let%bind kp_a = diff --git a/src/lib/mina_net2/tests/dune b/src/lib/mina_net2/tests/dune index de6557ee239..a3b440e62c1 100644 --- a/src/lib/mina_net2/tests/dune +++ b/src/lib/mina_net2/tests/dune @@ -20,6 +20,7 @@ network_peer file_system bounded_types + mina_compile_config ) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_jane ppx_mina ppx_version)) diff --git a/src/lib/mina_net2/tests/tests.ml b/src/lib/mina_net2/tests/tests.ml index 865392609c1..3bfefc9c951 100644 --- a/src/lib/mina_net2/tests/tests.ml +++ b/src/lib/mina_net2/tests/tests.ml @@ -11,6 +11,9 @@ let%test_module "Mina network tests" = let pids = Child_processes.Termination.create_pid_table () + let block_window_duration = + Mina_compile_config.For_unit_tests.t.block_window_duration + let setup_two_nodes network_id = let%bind a_tmp = Unix.mkdtemp "p2p_helper_test_a" in let%bind b_tmp = Unix.mkdtemp "p2p_helper_test_b" in @@ -19,21 +22,21 @@ let%test_module "Mina network tests" = create ~all_peers_seen_metric:false ~logger:(Logger.extend logger [ ("name", `String "a") ]) ~conf_dir:a_tmp ~pids ~on_peer_connected:Fn.ignore - ~on_peer_disconnected:Fn.ignore () + ~on_peer_disconnected:Fn.ignore ~block_window_duration () >>| Or_error.ok_exn in let%bind b = create ~all_peers_seen_metric:false ~logger:(Logger.extend logger [ ("name", `String "b") ]) ~conf_dir:b_tmp ~pids ~on_peer_connected:Fn.ignore - ~on_peer_disconnected:Fn.ignore () + ~on_peer_disconnected:Fn.ignore ~block_window_duration () >>| Or_error.ok_exn in let%bind c = create ~all_peers_seen_metric:false ~logger:(Logger.extend logger [ ("name", `String "c") ]) ~conf_dir:c_tmp ~pids ~on_peer_connected:Fn.ignore - ~on_peer_disconnected:Fn.ignore () + ~on_peer_disconnected:Fn.ignore ~block_window_duration () >>| Or_error.ok_exn in let%bind kp_a = generate_random_keypair a in diff --git a/src/lib/mina_net2/validation_callback.ml b/src/lib/mina_net2/validation_callback.ml index 8d8bb63192b..cba4cac1326 100644 --- a/src/lib/mina_net2/validation_callback.ml +++ b/src/lib/mina_net2/validation_callback.ml @@ -32,6 +32,7 @@ let is_expired cb = | Some expires_at -> Time_ns.(now () >= expires_at) +(* module type Metric_intf = sig val validations_timed_out : Mina_metrics.Counter.t @@ -39,53 +40,95 @@ module type Metric_intf = sig val ignored : Mina_metrics.Counter.t - module Validation_time : sig + module Validation_time(Context : Mina_metrics.CONTEXT) : sig val update : Time.Span.t -> unit end - module Processing_time : sig + module Processing_time(Context : Mina_metrics.CONTEXT) : sig val update : Time.Span.t -> unit end - module Rejection_time : sig + module Rejection_time(Context : Mina_metrics.CONTEXT) : sig val update : Time.Span.t -> unit end end -let metrics_of_message_type m : (module Metric_intf) option = +let metrics_of_message_type m = match m with | `Unknown -> None | `Block -> - Some (module Mina_metrics.Network.Block) + Some (Mina_metrics.Network.Block.validations_timed_out) | `Snark_work -> - Some (module Mina_metrics.Network.Snark_work) + Some (Mina_metrics.Network.Snark_work.validations_timed_out) | `Transaction -> - Some (module Mina_metrics.Network.Transaction) + Some (Mina_metrics.Network.Transaction.validations_timed_out) + +*) let record_timeout_metrics cb = Mina_metrics.(Counter.inc_one Network.validations_timed_out) ; - match metrics_of_message_type cb.message_type with - | None -> - () - | Some (module M) -> - Mina_metrics.Counter.inc_one M.validations_timed_out - -let record_validation_metrics message_type (result : validation_result) - validation_time processing_time = - match metrics_of_message_type message_type with - | None -> + let counter = + match cb.message_type with + | `Unknown -> + None + | `Block -> + Some Mina_metrics.Network.Block.validations_timed_out + | `Snark_work -> + Some Mina_metrics.Network.Snark_work.validations_timed_out + | `Transaction -> + Some Mina_metrics.Network.Transaction.validations_timed_out + in + Option.iter ~f:Mina_metrics.Counter.inc_one counter + +let record_validation_metrics ~block_window_duration message_type + (result : validation_result) validation_time processing_time = + let open Mina_metrics.Network in + let module Context = struct + let block_window_duration = block_window_duration + end in + match message_type with + | `Unknown -> () - | Some (module M) -> ( + | `Block -> ( + let module Validation_time = Block.Validation_time (Context) in + let module Processing_time = Block.Processing_time (Context) in + let module Rejection_time = Block.Rejection_time (Context) in match result with | `Ignore -> - Mina_metrics.Counter.inc_one M.ignored + Mina_metrics.Counter.inc_one Block.ignored | `Accept -> - M.Validation_time.update validation_time ; - M.Processing_time.update processing_time + Validation_time.update validation_time ; + Processing_time.update processing_time | `Reject -> - Mina_metrics.Counter.inc_one M.rejected ; - M.Rejection_time.update processing_time ) + Mina_metrics.Counter.inc_one Block.rejected ; + Rejection_time.update processing_time ) + | `Snark_work -> ( + let module Validation_time = Snark_work.Validation_time (Context) in + let module Processing_time = Snark_work.Processing_time (Context) in + let module Rejection_time = Snark_work.Rejection_time (Context) in + match result with + | `Ignore -> + Mina_metrics.Counter.inc_one Block.ignored + | `Accept -> + Validation_time.update validation_time ; + Processing_time.update processing_time + | `Reject -> + Mina_metrics.Counter.inc_one Block.rejected ; + Rejection_time.update processing_time ) + | `Transaction -> ( + let module Validation_time = Transaction.Validation_time (Context) in + let module Processing_time = Transaction.Processing_time (Context) in + let module Rejection_time = Transaction.Rejection_time (Context) in + match result with + | `Ignore -> + Mina_metrics.Counter.inc_one Block.ignored + | `Accept -> + Validation_time.update validation_time ; + Processing_time.update processing_time + | `Reject -> + Mina_metrics.Counter.inc_one Block.rejected ; + Rejection_time.update processing_time ) let await_timeout cb = if is_expired cb then Deferred.return () @@ -98,7 +141,7 @@ let await_timeout cb = ( Time_ns.Span.to_span_float_round_nearest @@ Time_ns.diff expires_at (Time_ns.now ()) ) -let await cb = +let await ~block_window_duration cb = if is_expired cb then (record_timeout_metrics cb ; Deferred.return None) else match cb.expiration with @@ -119,14 +162,18 @@ let await cb = Time_ns.abs_diff (Time_ns.now ()) cb.created_at |> Time_ns.Span.to_ms |> Time.Span.of_ms in - record_validation_metrics cb.message_type result validation_time - processing_time ; + record_validation_metrics ~block_window_duration cb.message_type + result validation_time processing_time ; Some result | `Timeout -> record_timeout_metrics cb ; None ) -let await_exn cb = - match%map await cb with None -> failwith "timeout" | Some result -> result +let await_exn ~block_window_duration cb = + match%map await ~block_window_duration cb with + | None -> + failwith "timeout" + | Some result -> + result let fire_if_not_already_fired cb result = if not (is_expired cb) then ( diff --git a/src/lib/mina_net2/validation_callback.mli b/src/lib/mina_net2/validation_callback.mli index ac87f3a8b48..352dd161642 100644 --- a/src/lib/mina_net2/validation_callback.mli +++ b/src/lib/mina_net2/validation_callback.mli @@ -11,9 +11,11 @@ val create_without_expiration : unit -> t val is_expired : t -> bool -val await : t -> validation_result option Deferred.t +val await : + block_window_duration:Time.Span.t -> t -> validation_result option Deferred.t -val await_exn : t -> validation_result Deferred.t +val await_exn : + block_window_duration:Time.Span.t -> t -> validation_result Deferred.t (** May return a deferred that never resolves, in the case of callbacks without expiration. *) val await_timeout : t -> unit Deferred.t diff --git a/src/lib/mina_networking/mina_networking.ml b/src/lib/mina_networking/mina_networking.ml index 15dbc9ac89b..93a9522ec33 100644 --- a/src/lib/mina_networking/mina_networking.ml +++ b/src/lib/mina_networking/mina_networking.ml @@ -42,6 +42,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end module Node_status = Node_status diff --git a/src/lib/mina_networking/mina_networking.mli b/src/lib/mina_networking/mina_networking.mli index d53ae824086..1aaf511d8e2 100644 --- a/src/lib/mina_networking/mina_networking.mli +++ b/src/lib/mina_networking/mina_networking.mli @@ -29,6 +29,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end module Node_status = Node_status diff --git a/src/lib/mina_networking/rpcs.ml b/src/lib/mina_networking/rpcs.ml index 9b4cca901d4..1d91b2f105e 100644 --- a/src/lib/mina_networking/rpcs.ml +++ b/src/lib/mina_networking/rpcs.ml @@ -33,6 +33,8 @@ module type CONTEXT = sig val list_peers : unit -> Peer.t list Deferred.t val get_transition_frontier : unit -> Transition_frontier.t option + + val compile_config : Mina_compile_config.t end type ctx = (module CONTEXT) diff --git a/src/lib/network_pool/intf.ml b/src/lib/network_pool/intf.ml index ed389c0a4ec..c587756a216 100644 --- a/src/lib/network_pool/intf.ml +++ b/src/lib/network_pool/intf.ml @@ -224,6 +224,7 @@ module type Network_pool_base_intf = sig -> logger:Logger.t -> log_gossip_heard:bool -> on_remote_push:(unit -> unit Deferred.t) + -> block_window_duration:Time.Span.t -> t * Remote_sink.t * Local_sink.t val of_resource_pool_and_diffs : @@ -233,6 +234,7 @@ module type Network_pool_base_intf = sig -> tf_diffs:transition_frontier_diff Strict_pipe.Reader.t -> log_gossip_heard:bool -> on_remote_push:(unit -> unit Deferred.t) + -> block_window_duration:Time.Span.t -> t * Remote_sink.t * Local_sink.t val resource_pool : t -> resource_pool diff --git a/src/lib/network_pool/network_pool_base.ml b/src/lib/network_pool/network_pool_base.ml index 05c60796168..cf4c1e36674 100644 --- a/src/lib/network_pool/network_pool_base.ml +++ b/src/lib/network_pool/network_pool_base.ml @@ -105,6 +105,7 @@ end) ; write_broadcasts : Resource_pool.Diff.t With_nonce.t Linear_pipe.Writer.t ; read_broadcasts : Resource_pool.Diff.t With_nonce.t Linear_pipe.Reader.t ; constraint_constants : Genesis_constants.Constraint_constants.t + ; block_window_duration : Time.Span.t } let resource_pool { resource_pool; _ } = resource_pool @@ -167,7 +168,7 @@ end) | Transition_frontier_extension of Resource_pool.transition_frontier_diff let of_resource_pool_and_diffs resource_pool ~logger ~constraint_constants - ~tf_diffs ~log_gossip_heard ~on_remote_push = + ~tf_diffs ~log_gossip_heard ~on_remote_push ~block_window_duration = let read_broadcasts, write_broadcasts = Linear_pipe.create () in let network_pool = { resource_pool @@ -175,6 +176,7 @@ end) ; read_broadcasts ; write_broadcasts ; constraint_constants + ; block_window_duration } in let remote_r, remote_w, remote_rl = @@ -183,6 +185,7 @@ end) ~unwrap:(function | Diff m -> m | _ -> failwith "unexpected message type" ) ~trace_label:Resource_pool.label ~logger resource_pool + ~block_window_duration in let local_r, local_w, _ = Local_sink.create @@ -190,6 +193,7 @@ end) ~unwrap:(function | Diff m -> m | _ -> failwith "unexpected message type" ) ~trace_label:Resource_pool.label ~logger resource_pool + ~block_window_duration in log_rate_limiter_occasionally network_pool remote_rl ; (*priority: Transition frontier diffs > local diffs > incoming diffs*) @@ -262,7 +266,8 @@ end) go () let create ~config ~constraint_constants ~consensus_constants ~time_controller - ~frontier_broadcast_pipe ~logger ~log_gossip_heard ~on_remote_push = + ~frontier_broadcast_pipe ~logger ~log_gossip_heard ~on_remote_push + ~block_window_duration = (* Diffs from transition frontier extensions *) let tf_diff_reader, tf_diff_writer = Strict_pipe.( @@ -274,7 +279,7 @@ end) ~time_controller ~config ~logger ~frontier_broadcast_pipe ~tf_diff_writer ) ~constraint_constants ~logger ~tf_diffs:tf_diff_reader ~log_gossip_heard - ~on_remote_push + ~on_remote_push ~block_window_duration in O1trace.background_thread rebroadcast_loop_thread_label (fun () -> rebroadcast_loop t logger ) ; diff --git a/src/lib/network_pool/pool_sink.ml b/src/lib/network_pool/pool_sink.ml index 8be8f96c3b6..50ca9371593 100644 --- a/src/lib/network_pool/pool_sink.ml +++ b/src/lib/network_pool/pool_sink.ml @@ -26,6 +26,7 @@ module type Pool_sink = sig -> trace_label:string -> logger:Logger.t -> pool + -> block_window_duration:Time.Span.t -> 'wrapped_t Strict_pipe.Reader.t * t * Rate_limiter.t end @@ -71,6 +72,7 @@ module Base ; throttle : unit Throttle.t ; on_push : unit -> unit Deferred.t ; log_gossip_heard : bool + ; block_window_duration : Time.Span.t } -> t | Void @@ -143,6 +145,7 @@ module Base ; throttle ; on_push ; log_gossip_heard + ; block_window_duration } -> O1trace.sync_thread (sprintf "handle_%s_gossip" trace_label) @@ fun () -> @@ -154,7 +157,10 @@ module Base | BC.External cb'' -> Diff.update_metrics env' cb'' ~log_gossip_heard ~logger ; don't_wait_for - ( match%map Mina_net2.Validation_callback.await cb'' with + ( match%map + Mina_net2.Validation_callback.await ~block_window_duration + cb'' + with | None -> let diff = Envelope.Incoming.data env' in [%log error] @@ -191,7 +197,7 @@ module Base Deferred.unit let create ?(on_push = Fn.const Deferred.unit) ?(log_gossip_heard = false) - ~wrap ~unwrap ~trace_label ~logger pool = + ~wrap ~unwrap ~trace_label ~logger pool ~block_window_duration = let r, writer = Strict_pipe.create ~name:"verified network pool diffs" (Buffered @@ -217,6 +223,7 @@ module Base ; throttle ; on_push ; log_gossip_heard + ; block_window_duration } , rate_limiter ) diff --git a/src/lib/network_pool/snark_pool.ml b/src/lib/network_pool/snark_pool.ml index 7d2911e4616..6ede322b927 100644 --- a/src/lib/network_pool/snark_pool.ml +++ b/src/lib/network_pool/snark_pool.ml @@ -570,6 +570,9 @@ let%test_module "random set test" = let precomputed_values = Lazy.force Precomputed_values.for_unit_tests + let block_window_duration = + Mina_compile_config.For_unit_tests.t.block_window_duration + (* SNARK work is rejected if the prover doesn't have an account and the fee is below the account creation fee. So, just to make generating valid SNARK work easier for testing, we set the account creation fee to 0. *) @@ -639,6 +642,7 @@ let%test_module "random set test" = ~consensus_constants ~time_controller ~frontier_broadcast_pipe:frontier_broadcast_pipe_r ~log_gossip_heard:false ~on_remote_push:(Fn.const Deferred.unit) + ~block_window_duration (* |> *) in let pool = Mock_snark_pool.resource_pool mock_pool in @@ -805,6 +809,7 @@ let%test_module "random set test" = ~consensus_constants ~time_controller ~logger ~frontier_broadcast_pipe:frontier_broadcast_pipe_r ~log_gossip_heard:false ~on_remote_push:(Fn.const Deferred.unit) + ~block_window_duration in let priced_proof = { Priced_proof.proof = @@ -877,6 +882,7 @@ let%test_module "random set test" = ~consensus_constants ~time_controller ~frontier_broadcast_pipe:frontier_broadcast_pipe_r ~log_gossip_heard:false ~on_remote_push:(Fn.const Deferred.unit) + ~block_window_duration in List.map (List.take works per_reader) ~f:create_work |> List.map ~f:(fun work -> @@ -961,6 +967,7 @@ let%test_module "random set test" = ~constraint_constants ~consensus_constants ~time_controller ~frontier_broadcast_pipe:frontier_broadcast_pipe_r ~log_gossip_heard:false ~on_remote_push:(Fn.const Deferred.unit) + ~block_window_duration in let resource_pool = Mock_snark_pool.resource_pool network_pool in let%bind () = diff --git a/src/lib/network_pool/test.ml b/src/lib/network_pool/test.ml index 9e5d0193a63..b0ddac4e0a4 100644 --- a/src/lib/network_pool/test.ml +++ b/src/lib/network_pool/test.ml @@ -22,6 +22,9 @@ let%test_module "network pool test" = let time_controller = Block_time.Controller.basic ~logger + let block_window_duration = + Mina_compile_config.For_unit_tests.t.block_window_duration + let verifier = Async.Thread_safe.block_on_async_exn (fun () -> Verifier.create ~logger ~proof_level ~constraint_constants @@ -61,6 +64,7 @@ let%test_module "network pool test" = ~consensus_constants ~time_controller ~frontier_broadcast_pipe:frontier_broadcast_pipe_r ~log_gossip_heard:false ~on_remote_push:(Fn.const Deferred.unit) + ~block_window_duration in let%bind () = Mocks.Transition_frontier.refer_statements tf [ work ] @@ -115,6 +119,7 @@ let%test_module "network pool test" = ~consensus_constants ~time_controller ~frontier_broadcast_pipe:frontier_broadcast_pipe_r ~log_gossip_heard:false ~on_remote_push:(Fn.const Deferred.unit) + ~block_window_duration in List.map (List.take works per_reader) ~f:create_work |> List.map ~f:(fun work -> diff --git a/src/lib/network_pool/transaction_pool.ml b/src/lib/network_pool/transaction_pool.ml index 751dd94c0b0..e99224ecdb8 100644 --- a/src/lib/network_pool/transaction_pool.ml +++ b/src/lib/network_pool/transaction_pool.ml @@ -1641,6 +1641,9 @@ let%test_module _ = let num_extra_keys = 30 + let block_window_duration = + Mina_compile_config.For_unit_tests.t.block_window_duration + (* keys that can be used when generating new accounts *) let extra_keys = Array.init num_extra_keys ~f:(fun _ -> Signature_lib.Keypair.create ()) @@ -1922,6 +1925,7 @@ let%test_module _ = Test.create ~config ~logger ~constraint_constants ~consensus_constants ~time_controller ~frontier_broadcast_pipe:frontier_pipe_r ~log_gossip_heard:false ~on_remote_push:(Fn.const Deferred.unit) + ~block_window_duration in let txn_pool = Test.resource_pool pool_ in let%map () = Async.Scheduler.yield_until_no_jobs_remain () in diff --git a/src/lib/sync_handler/sync_handler.ml b/src/lib/sync_handler/sync_handler.ml index 6cc38a179f8..0fd5643c7bf 100644 --- a/src/lib/sync_handler/sync_handler.ml +++ b/src/lib/sync_handler/sync_handler.ml @@ -14,6 +14,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end module type Inputs_intf = sig diff --git a/src/lib/transaction_inclusion_status/transaction_inclusion_status.ml b/src/lib/transaction_inclusion_status/transaction_inclusion_status.ml index 1d34ed2059d..555bf91eeb6 100644 --- a/src/lib/transaction_inclusion_status/transaction_inclusion_status.ml +++ b/src/lib/transaction_inclusion_status/transaction_inclusion_status.ml @@ -87,6 +87,9 @@ let%test_module "transaction_status" = let pool_max_size = precomputed_values.genesis_constants.txpool_max_size + let block_window_duration = + Mina_compile_config.For_unit_tests.t.block_window_duration + let verifier = Async.Thread_safe.block_on_async_exn (fun () -> Verifier.create ~logger ~proof_level ~constraint_constants @@ -124,6 +127,7 @@ let%test_module "transaction_status" = ~consensus_constants:precomputed_values.consensus_constants ~time_controller ~logger ~frontier_broadcast_pipe ~log_gossip_heard:false ~on_remote_push:(Fn.const Deferred.unit) + ~block_window_duration in don't_wait_for @@ Linear_pipe.iter (Transaction_pool.broadcasts transaction_pool) diff --git a/src/lib/transition_frontier_controller/transition_frontier_controller.ml b/src/lib/transition_frontier_controller/transition_frontier_controller.ml index 7832ae682a6..20f600f37da 100644 --- a/src/lib/transition_frontier_controller/transition_frontier_controller.ml +++ b/src/lib/transition_frontier_controller/transition_frontier_controller.ml @@ -11,6 +11,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end let run ~context:(module Context : CONTEXT) ~trust_system ~verifier ~network diff --git a/src/lib/transition_handler/block_sink.ml b/src/lib/transition_handler/block_sink.ml index 3cd698e9fae..04781e3d582 100644 --- a/src/lib/transition_handler/block_sink.ml +++ b/src/lib/transition_handler/block_sink.ml @@ -19,6 +19,7 @@ type block_sink_config = ; consensus_constants : Consensus.Constants.t ; genesis_constants : Genesis_constants.t ; constraint_constants : Genesis_constants.Constraint_constants.t + ; block_window_duration : Time.Span.t } type t = @@ -32,6 +33,7 @@ type t = ; consensus_constants : Consensus.Constants.t ; genesis_constants : Genesis_constants.t ; constraint_constants : Genesis_constants.Constraint_constants.t + ; block_window_duration : Time.Span.t } | Void @@ -53,10 +55,14 @@ let push sink (`Transition e, `Time_received tm, `Valid_cb cb) = ; consensus_constants ; genesis_constants ; constraint_constants + ; block_window_duration } -> O1trace.sync_thread "handle_block_gossip" @@ fun () -> let%bind () = on_push () in + let module Metrics_Context = struct + let block_window_duration = block_window_duration + end in Mina_metrics.(Counter.inc_one Network.gossip_messages_received) ; let state = Envelope.Incoming.data e in let state_hash = @@ -85,15 +91,19 @@ let push sink (`Transition e, `Time_received tm, `Valid_cb cb) = Block_time.(now time_controller |> to_time_exn) in don't_wait_for - ( match%map Mina_net2.Validation_callback.await cb with + ( match%map + Mina_net2.Validation_callback.await ~block_window_duration cb + with | Some `Accept -> let processing_time_span = Time.diff Block_time.(now time_controller |> to_time_exn) processing_start_time in - Mina_metrics.Block_latency.( - Validation_acceptance_time.update processing_time_span) + let module Validation_acceptance_time = + Mina_metrics.Block_latency.Validation_acceptance_time + (Metrics_Context) in + Validation_acceptance_time.update processing_time_span | Some _ -> () | None -> @@ -182,9 +192,12 @@ let push sink (`Transition e, `Time_received tm, `Valid_cb cb) = (Consensus.Data.Consensus_time.of_time_exn ~constants:consensus_constants tm ) in - Mina_metrics.Block_latency.Gossip_slots.update - (Float.of_int (tm_slot - tn_production_slot)) ; - Mina_metrics.Block_latency.Gossip_time.update + let module Gossip_slots = + Mina_metrics.Block_latency.Gossip_slots (Metrics_Context) in + Gossip_slots.update (Float.of_int (tm_slot - tn_production_slot)) ; + let module Gossip_time = + Mina_metrics.Block_latency.Gossip_time (Metrics_Context) in + Gossip_time.update Block_time.(Span.to_time_span @@ diff tm tn_production_time) ; Deferred.unit @@ -204,6 +217,7 @@ let create ; consensus_constants ; genesis_constants ; constraint_constants + ; block_window_duration } = let rate_limiter = Network_pool.Rate_limiter.create @@ -225,6 +239,7 @@ let create ; consensus_constants ; genesis_constants ; constraint_constants + ; block_window_duration } ) let void = Void diff --git a/src/lib/transition_handler/block_sink.mli b/src/lib/transition_handler/block_sink.mli index 9613de1db76..e3586a01cd3 100644 --- a/src/lib/transition_handler/block_sink.mli +++ b/src/lib/transition_handler/block_sink.mli @@ -1,5 +1,6 @@ open Network_peer open Mina_base +open Core_kernel type Structured_log_events.t += | Block_received of { state_hash : State_hash.t; sender : Envelope.Sender.t } @@ -21,6 +22,7 @@ type block_sink_config = ; consensus_constants : Consensus.Constants.t ; genesis_constants : Genesis_constants.t ; constraint_constants : Genesis_constants.Constraint_constants.t + ; block_window_duration : Time.Span.t } val create : diff --git a/src/lib/transition_handler/catchup_scheduler.ml b/src/lib/transition_handler/catchup_scheduler.ml index 6602464a66f..b1c36a39cd5 100644 --- a/src/lib/transition_handler/catchup_scheduler.ml +++ b/src/lib/transition_handler/catchup_scheduler.ml @@ -194,7 +194,8 @@ let rec remove_tree t parent_hash = let transition, _ = Envelope.Incoming.data (Cached.peek child) in remove_tree t (State_hash.With_state_hashes.state_hash transition) ) -let watch t ~timeout_duration ~cached_transition ~valid_cb = +let watch t ~timeout_duration ~cached_transition ~valid_cb + ~block_window_duration = let transition_with_hash, _ = Envelope.Incoming.data (Cached.peek cached_transition) in @@ -213,7 +214,9 @@ let watch t ~timeout_duration ~cached_transition ~valid_cb = | `Ok -> (* Clean up entry upon callback resolution *) upon - (Deferred.ignore_m @@ Mina_net2.Validation_callback.await data) + ( Deferred.ignore_m + @@ Mina_net2.Validation_callback.await ~block_window_duration data + ) (fun () -> Hashtbl.remove t.validation_callbacks hash) | `Duplicate -> [%log' warn t.logger] "Double validation callback for $state_hash" @@ -330,6 +333,9 @@ let%test_module "Transition_handler.Catchup_scheduler tests" = let create = create ~logger ~trust_system ~time_controller + let block_window_duration = + Mina_compile_config.For_unit_tests.t.block_window_duration + let verifier = Async.Thread_safe.block_on_async_exn (fun () -> Verifier.create ~logger ~proof_level ~constraint_constants @@ -367,7 +373,8 @@ let%test_module "Transition_handler.Catchup_scheduler tests" = in watch scheduler ~timeout_duration ~valid_cb:None ~cached_transition: - (Cached.pure @@ downcast_breadcrumb disjoint_breadcrumb) ; + (Cached.pure @@ downcast_breadcrumb disjoint_breadcrumb) + ~block_window_duration ; Async.Thread_safe.block_on_async_exn (fun () -> match%map Block_time.Timeout.await @@ -427,7 +434,8 @@ let%test_module "Transition_handler.Catchup_scheduler tests" = in watch scheduler ~timeout_duration ~valid_cb:None ~cached_transition: - (Cached.transform ~f:downcast_breadcrumb breadcrumb_2) ; + (Cached.transform ~f:downcast_breadcrumb breadcrumb_2) + ~block_window_duration ; Async.Thread_safe.block_on_async_exn (fun () -> Transition_frontier.add_breadcrumb_exn frontier (Cached.peek breadcrumb_1) ) ; @@ -506,7 +514,8 @@ let%test_module "Transition_handler.Catchup_scheduler tests" = in watch scheduler ~timeout_duration ~valid_cb:None ~cached_transition: - (Cached.pure @@ downcast_breadcrumb oldest_breadcrumb) ; + (Cached.pure @@ downcast_breadcrumb oldest_breadcrumb) + ~block_window_duration ; assert ( has_timeout_parent_hash scheduler (Transition_frontier.Breadcrumb.parent_hash oldest_breadcrumb) ) ; @@ -515,7 +524,8 @@ let%test_module "Transition_handler.Catchup_scheduler tests" = ~f:(fun prev_breadcrumb curr_breadcrumb -> watch scheduler ~timeout_duration ~valid_cb:None ~cached_transition: - (Cached.pure @@ downcast_breadcrumb curr_breadcrumb) ; + (Cached.pure @@ downcast_breadcrumb curr_breadcrumb) + ~block_window_duration ; assert ( not @@ has_timeout_parent_hash scheduler diff --git a/src/lib/transition_handler/processor.ml b/src/lib/transition_handler/processor.ml index 9217894e595..1940c596f22 100644 --- a/src/lib/transition_handler/processor.ml +++ b/src/lib/transition_handler/processor.ml @@ -25,6 +25,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end (* TODO: calculate a sensible value from postake consensus arguments *) @@ -44,7 +46,13 @@ let cached_transform_deferred_result ~transform_cached ~transform_result cached (* add a breadcrumb and perform post processing *) let add_and_finalize ~logger ~frontier ~catchup_scheduler ~processed_transition_writer ~only_if_present ~time_controller ~source - ~valid_cb cached_breadcrumb ~(precomputed_values : Precomputed_values.t) = + ~valid_cb cached_breadcrumb ~(precomputed_values : Precomputed_values.t) + ~block_window_duration = + let module Metrics_context = struct + let block_window_duration = block_window_duration + end in + let module Inclusion_time = + Mina_metrics.Block_latency.Inclusion_time (Metrics_context) in let breadcrumb = if Cached.is_pure cached_breadcrumb then Cached.peek cached_breadcrumb else Cached.invalidate_with_success cached_breadcrumb @@ -94,8 +102,7 @@ let add_and_finalize ~logger ~frontier ~catchup_scheduler (Consensus.Data.Consensus_time.to_time ~constants:consensus_constants transition_time ) in - Mina_metrics.Block_latency.Inclusion_time.update - (Block_time.Span.to_time_span time_elapsed) ) ; + Inclusion_time.update (Block_time.Span.to_time_span time_elapsed) ) ; [%log internal] "Add_and_finalize_done" ; if Writer.is_closed processed_transition_writer then Or_error.error_string "processed transitions closed" @@ -204,7 +211,8 @@ let process_transition ~context:(module Context : CONTEXT) ~trust_system in Catchup_scheduler.watch catchup_scheduler ~timeout_duration ~cached_transition:cached_initially_validated_transition - ~valid_cb ; + ~valid_cb + ~block_window_duration:compile_config.block_window_duration ; return (Error ()) ) in (* TODO: only access parent in transition frontier once (already done in call to validate dependencies) #2485 *) @@ -251,6 +259,7 @@ let process_transition ~context:(module Context : CONTEXT) ~trust_system add_and_finalize ~logger ~frontier ~catchup_scheduler ~processed_transition_writer ~only_if_present:false ~time_controller ~source:`Gossip breadcrumb ~precomputed_values ~valid_cb + ~block_window_duration:compile_config.block_window_duration in ( match result with | Ok () -> @@ -343,6 +352,8 @@ let run ~context:(module Context : CONTEXT) ~verifier ~trust_system let%map result = add_and_finalize ~logger ~only_if_present:true ~source:`Catchup ~valid_cb b + ~block_window_duration: + compile_config.block_window_duration in Internal_tracing.with_state_hash state_hash @@ fun () -> @@ -406,6 +417,8 @@ let run ~context:(module Context : CONTEXT) ~verifier ~trust_system match%map add_and_finalize ~logger ~only_if_present:false ~source:`Internal breadcrumb ~valid_cb:None + ~block_window_duration: + compile_config.block_window_duration with | Ok () -> [%log internal] "Breadcrumb_integrated" ; @@ -453,6 +466,8 @@ let%test_module "Transition_handler.Processor tests" = let trust_system = Trust_system.null () + let compile_config = Mina_compile_config.For_unit_tests.t + let verifier = Async.Thread_safe.block_on_async_exn (fun () -> Verifier.create ~logger ~proof_level ~constraint_constants @@ -468,6 +483,8 @@ let%test_module "Transition_handler.Processor tests" = let constraint_constants = constraint_constants let consensus_constants = precomputed_values.consensus_constants + + let compile_config = compile_config end let downcast_breadcrumb breadcrumb = diff --git a/src/lib/transition_router/transition_router.ml b/src/lib/transition_router/transition_router.ml index 1963be75d00..3f6fa107d4f 100644 --- a/src/lib/transition_router/transition_router.ml +++ b/src/lib/transition_router/transition_router.ml @@ -13,6 +13,8 @@ module type CONTEXT = sig val constraint_constants : Genesis_constants.Constraint_constants.t val consensus_constants : Consensus.Constants.t + + val compile_config : Mina_compile_config.t end type Structured_log_events.t += Starting_transition_frontier_controller From 274ac64e8649e35b166e2f65b0164993a9ca537e Mon Sep 17 00:00:00 2001 From: martyall Date: Sun, 8 Sep 2024 02:40:01 -0700 Subject: [PATCH 03/14] src/app compiles --- src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml | 1 + src/app/cli/src/init/client.ml | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index f9b4f49f279..f29d32a86b2 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -1290,6 +1290,7 @@ Pass one of -peer, -peer-list-file, -seed, -peer-list-url.|} ; ; time_controller ; pubsub_v1 ; pubsub_v0 + ; block_window_duration = compile_config.block_window_duration } in let net_config = diff --git a/src/app/cli/src/init/client.ml b/src/app/cli/src/init/client.ml index dc046388752..6fc0ca0c94e 100644 --- a/src/app/cli/src/init/client.ml +++ b/src/app/cli/src/init/client.ml @@ -1612,12 +1612,14 @@ let generate_libp2p_keypair_do privkey_path = (let open Deferred.Let_syntax in (* FIXME: I'd like to accumulate messages into this logger and only dump them out in failure paths. *) let logger = Logger.null () in + let compile_config = Mina_compile_config.Compiled.t in (* Using the helper only for keypair generation requires no state. *) File_system.with_temp_dir "mina-generate-libp2p-keypair" ~f:(fun tmpd -> match%bind Mina_net2.create ~logger ~conf_dir:tmpd ~all_peers_seen_metric:false ~pids:(Child_processes.Termination.create_pid_table ()) - ~on_peer_connected:ignore ~on_peer_disconnected:ignore () + ~on_peer_connected:ignore ~on_peer_disconnected:ignore + ~block_window_duration:compile_config.block_window_duration () with | Ok net -> let%bind me = Mina_net2.generate_random_keypair net in @@ -1644,12 +1646,14 @@ let dump_libp2p_keypair_do privkey_path = Deferred.ignore_m (let open Deferred.Let_syntax in let logger = Logger.null () in + let compile_config = Mina_compile_config.Compiled.t in (* Using the helper only for keypair generation requires no state. *) File_system.with_temp_dir "mina-dump-libp2p-keypair" ~f:(fun tmpd -> match%bind Mina_net2.create ~logger ~conf_dir:tmpd ~all_peers_seen_metric:false ~pids:(Child_processes.Termination.create_pid_table ()) - ~on_peer_connected:ignore ~on_peer_disconnected:ignore () + ~on_peer_connected:ignore ~on_peer_disconnected:ignore + ~block_window_duration:compile_config.block_window_duration () with | Ok net -> let%bind () = Mina_net2.shutdown net in From 4a9b6bb1ee3e539f257891b308b6c88301af49b4 Mon Sep 17 00:00:00 2001 From: martyall Date: Mon, 9 Sep 2024 12:01:10 -0700 Subject: [PATCH 04/14] remove effectful initialization --- .../prometheus_metrics/metric_generators.ml | 77 ++-- .../prometheus_metrics/mina_metrics.ml | 354 +++++++++--------- src/lib/mina_net2/validation_callback.ml | 83 ++-- 3 files changed, 249 insertions(+), 265 deletions(-) diff --git a/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml b/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml index df675f6f018..f9168fab806 100644 --- a/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml +++ b/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml @@ -19,14 +19,30 @@ module type Bucketed_average_spec_intf = sig val num_buckets : int val render_average : (float * int) list -> float + + val is_initialized : bool ref +end + +module Intervals = struct + type t = { rolling : Time.Span.t; tick : Time.Span.t } + + let make ~rolling_interval ~tick_interval = + let open Time.Span in + let ( = ) = Float.equal in + let ( mod ) = Float.mod_float in + if not (to_ns rolling_interval mod to_ns tick_interval = 0.0) then + failwith + "invalid intervals provided to Moving_time_average -- the \ + tick_interval does not evenly divide the rolling_interval" + else { rolling = rolling_interval; tick = tick_interval } end module type Time_average_spec_intf = sig include Metric_spec_intf - val tick_interval : Core.Time.Span.t + val intervals : Intervals.t - val rolling_interval : Core.Time.Span.t + val is_initialized : bool ref end module type Moving_average_metric_intf = sig @@ -37,9 +53,11 @@ module type Moving_average_metric_intf = sig val update : datum -> unit val clear : unit -> unit + + val initialize : unit -> unit end -module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) () : +module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) : Moving_average_metric_intf with type datum := float = struct open Spec @@ -61,7 +79,7 @@ module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) () : | (value, num_entries) :: t -> buckets := Some ((value +. datum, num_entries + 1) :: t) - let () = + let initialize () = let rec tick () = upon (after (Time_ns.Span.of_ns @@ Time.Span.to_ns bucket_interval)) @@ -76,39 +94,30 @@ module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) () : tick () end -module Moving_time_average (Spec : Time_average_spec_intf) () : +module Moving_time_average (Spec : Time_average_spec_intf) : Moving_average_metric_intf with type datum := Core.Time.Span.t = struct - open Spec + include Moving_bucketed_average (struct + include Spec - let () = - let open Time.Span in - let ( = ) = Float.equal in - let ( mod ) = Float.mod_float in - if not (to_ns rolling_interval mod to_ns tick_interval = 0.0) then - failwith - "invalid intervals provided to Moving_time_average -- the \ - tick_interval does not evenly divide the rolling_interval" + let bucket_interval = Spec.intervals.rolling - include - Moving_bucketed_average - (struct - include Spec - - let bucket_interval = tick_interval - - let num_buckets = - Float.to_int - (Time.Span.to_ns rolling_interval /. Time.Span.to_ns tick_interval) - - let render_average buckets = - let total_sum, count_sum = - List.fold buckets ~init:(0.0, 0) - ~f:(fun (total_sum, count_sum) (total, count) -> - (total_sum +. total, count_sum + count) ) - in - total_sum /. Float.of_int count_sum - end) - () + let num_buckets = + Float.to_int + ( Time.Span.to_ns Spec.intervals.rolling + /. Time.Span.to_ns Spec.intervals.tick ) + + let render_average buckets = + let total_sum, count_sum = + List.fold buckets ~init:(0.0, 0) + ~f:(fun (total_sum, count_sum) (total, count) -> + (total_sum +. total, count_sum + count) ) + in + total_sum /. Float.of_int count_sum + end) + + let () = + if not !Spec.is_initialized then initialize () ; + Spec.is_initialized := true let update span = update (Core.Time.Span.to_sec span) end diff --git a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml index d9b47940622..1e126cda84c 100644 --- a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml +++ b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml @@ -435,9 +435,9 @@ module Network = struct Counter.v "messages_received" ~help ~namespace ~subsystem module Delay_time_spec (Context : CONTEXT) = struct - let tick_interval = Context.block_window_duration - - let rolling_interval = Time.Span.scale Context.block_window_duration 20.0 + let intervals = + Intervals.make ~rolling_interval:Context.block_window_duration + ~tick_interval:Context.block_window_duration end module Block = struct @@ -459,49 +459,52 @@ module Network = struct let help = "# of blocks received" in Counter.v "received" ~help ~namespace ~subsystem - module Validation_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + let validation_time_is_initialized = ref false - let subsystem = subsystem + let processing_time_is_initialized = ref false - let name = "validation_time" + let rejection_time_is_initialized = ref false - let help = - "average time, in ms, for blocks to be validated and rebroadcasted" - end) - () + module Validation_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) - module Processing_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + let subsystem = subsystem - let subsystem = subsystem + let name = "validation_time" - let name = "processing_time" + let help = + "average time, in ms, for blocks to be validated and rebroadcasted" - let help = - "average time, in ms, for blocks to be accepted after the OCaml \ - process receives it" - end) - () + let is_initialized = validation_time_is_initialized + end) - module Rejection_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + module Processing_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) - let subsystem = subsystem + let subsystem = subsystem - let name = "rejection_time" + let name = "processing_time" - let help = - "average time, in ms, for blocks to be rejected after the OCaml \ - process receives it" - end) - () + let help = + "average time, in ms, for blocks to be accepted after the OCaml \ + process receives it" + + let is_initialized = processing_time_is_initialized + end) + + module Rejection_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) + + let subsystem = subsystem + + let name = "rejection_time" + + let help = + "average time, in ms, for blocks to be rejected after the OCaml \ + process receives it" + + let is_initialized = rejection_time_is_initialized + end) end module Snark_work = struct @@ -523,50 +526,52 @@ module Network = struct let help = "# of snark work received" in Counter.v "received" ~help ~namespace ~subsystem - module Validation_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + let validation_time_is_initialized = ref false - let subsystem = subsystem + let processing_time_is_initialized = ref false - let name = "validation_time" + let rejection_time_is_initialized = ref false - let help = - "average delay, in ms, for snark work to be validated and \ - rebroadcasted" - end) - () + module Validation_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) - module Processing_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + let subsystem = subsystem - let subsystem = subsystem + let name = "validation_time" - let name = "processing_time" + let help = + "average delay, in ms, for snark work to be validated and rebroadcasted" - let help = - "average delay, in ms, for snark work to be accepted after the \ - OCaml process receives it" - end) - () + let is_initialized = validation_time_is_initialized + end) - module Rejection_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + module Processing_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) - let subsystem = subsystem + let subsystem = subsystem - let name = "rejection_time" + let name = "processing_time" - let help = - "average time, in ms, for snark work to be rejected after the \ - OCaml process receives it" - end) - () + let help = + "average delay, in ms, for snark work to be accepted after the OCaml \ + process receives it" + + let is_initialized = processing_time_is_initialized + end) + + module Rejection_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) + + let subsystem = subsystem + + let name = "rejection_time" + + let help = + "average time, in ms, for snark work to be rejected after the OCaml \ + process receives it" + + let is_initialized = rejection_time_is_initialized + end) end module Transaction = struct @@ -588,50 +593,53 @@ module Network = struct let help = "# of transactions received" in Counter.v "received" ~help ~namespace ~subsystem - module Validation_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + let validation_time_is_initialized = ref false - let subsystem = subsystem + let processing_time_is_initialized = ref false - let name = "validation_time" + let rejection_time_is_initialized = ref false - let help = - "average delay, in ms, for transactions to be validated and \ - rebroadcasted" - end) - () + module Validation_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) - module Processing_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + let subsystem = subsystem - let subsystem = subsystem + let name = "validation_time" - let name = "processing_time" + let help = + "average delay, in ms, for transactions to be validated and \ + rebroadcasted" - let help = - "average delay, in ms, for transactions to be accepted after the \ - OCaml process receives it" - end) - () + let is_initialized = validation_time_is_initialized + end) - module Rejection_time (Context : CONTEXT) = - Moving_time_average - (struct - include Delay_time_spec (Context) + module Processing_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) - let subsystem = subsystem + let subsystem = subsystem - let name = "rejection_time" + let name = "processing_time" - let help = - "average time, in ms, for transactions to be rejected after the \ - OCaml process receives it" - end) - () + let help = + "average delay, in ms, for transactions to be accepted after the OCaml \ + process receives it" + + let is_initialized = processing_time_is_initialized + end) + + module Rejection_time (Context : CONTEXT) = Moving_time_average (struct + include Delay_time_spec (Context) + + let subsystem = subsystem + + let name = "rejection_time" + + let help = + "average time, in ms, for transactions to be rejected after the OCaml \ + process receives it" + + let is_initialized = rejection_time_is_initialized + end) end let rpc_requests_received : Counter.t = @@ -1275,28 +1283,27 @@ module Transition_frontier = struct let help = "total # of staged txns that have been finalized" in Counter.v "finalized_staged_txns" ~help ~namespace ~subsystem - module TPS_30min = - Moving_bucketed_average - (struct - let bucket_interval = Core.Time.Span.of_min 3.0 + let tps_30min_is_initialized = ref false + + module TPS_30min = Moving_bucketed_average (struct + let bucket_interval = Core.Time.Span.of_min 3.0 - let num_buckets = 10 + let num_buckets = 10 - let render_average buckets = - let total = - List.fold buckets ~init:0.0 ~f:(fun acc (n, _) -> acc +. n) - in - total /. Core.Time.Span.(of_min 30.0 |> to_sec) + let render_average buckets = + let total = List.fold buckets ~init:0.0 ~f:(fun acc (n, _) -> acc +. n) in + total /. Core.Time.Span.(of_min 30.0 |> to_sec) + + let subsystem = subsystem - let subsystem = subsystem + let name = "tps_30min" - let name = "tps_30min" + let help = + "moving average for transaction per second, the rolling interval is set \ + to 30 min" - let help = - "moving average for transaction per second, the rolling interval is \ - set to 30 min" - end) - () + let is_initialized = tps_30min_is_initialized + end) let recently_finalized_staged_txns : Gauge.t = let help = @@ -1443,79 +1450,84 @@ module Block_latency = struct end module Latency_time_spec (Context : CONTEXT) = struct - let tick_interval = Time.Span.scale Context.block_window_duration 0.5 - - let rolling_interval = Time.Span.scale Context.block_window_duration 20.0 + let intervals = + Intervals.make + ~tick_interval:(Time.Span.scale Context.block_window_duration 0.5) + ~rolling_interval:(Time.Span.scale Context.block_window_duration 20.0) end - module Gossip_slots (Context : CONTEXT) = - Moving_bucketed_average - (struct - let bucket_interval = Time.Span.scale Context.block_window_duration 0.5 + let gossip_slots_is_initialized = ref false + + module Gossip_slots (Context : CONTEXT) = Moving_bucketed_average (struct + let bucket_interval = Time.Span.scale Context.block_window_duration 0.5 + + let num_buckets = 40 + + let subsystem = subsystem + + let name = "gossip_slots" - let num_buckets = 40 + let help = + "average delay, in slots, after which produced blocks are received" + + let render_average buckets = + let total_sum, count_sum = + List.fold buckets ~init:(0.0, 0) + ~f:(fun (total_sum, count_sum) (total, count) -> + (total_sum +. total, count_sum + count) ) + in + total_sum /. Float.of_int count_sum + + let is_initialized = gossip_slots_is_initialized + end) + + let gossip_time_is_initialized = ref false - let subsystem = subsystem + module Gossip_time (Context : CONTEXT) = Moving_time_average (struct + include Latency_time_spec (Context) - let name = "gossip_slots" + let subsystem = subsystem - let help = - "average delay, in slots, after which produced blocks are received" + let name = "gossip_time" - let render_average buckets = - let total_sum, count_sum = - List.fold buckets ~init:(0.0, 0) - ~f:(fun (total_sum, count_sum) (total, count) -> - (total_sum +. total, count_sum + count) ) - in - total_sum /. Float.of_int count_sum - end) - () + let help = + "average delay, in seconds, after which produced blocks are received" - module Gossip_time (Context : CONTEXT) = - Moving_time_average - (struct - include Latency_time_spec (Context) + let is_initialized = gossip_time_is_initialized + end) - let subsystem = subsystem + let inclusion_time_is_initialized = ref false - let name = "gossip_time" + module Inclusion_time (Context : CONTEXT) = Moving_time_average (struct + include Latency_time_spec (Context) - let help = - "average delay, in seconds, after which produced blocks are received" - end) - () + let subsystem = subsystem - module Inclusion_time (Context : CONTEXT) = - Moving_time_average - (struct - include Latency_time_spec (Context) + let name = "inclusion_time" - let subsystem = subsystem + let help = + "average delay, in seconds, after which produced blocks are included \ + into our frontier" - let name = "inclusion_time" + let is_initialized = inclusion_time_is_initialized + end) - let help = - "average delay, in seconds, after which produced blocks are included \ - into our frontier" - end) - () + let validation_acceptance_time_is_initialized = ref false module Validation_acceptance_time (Context : CONTEXT) = - Moving_time_average - (struct - include Latency_time_spec (Context) + Moving_time_average (struct + include Latency_time_spec (Context) - let subsystem = subsystem + let subsystem = subsystem + + let name = "validation_acceptance_time" - let name = "validation_acceptance_time" + let help = + "average delay, in seconds, between the time blocks are initially \ + received from the libp2p_helper, and when they are accepted as valid" - let help = - "average delay, in seconds, between the time blocks are initially \ - received from the libp2p_helper, and when they are accepted as \ - valid" - end) - () + let is_initialized = validation_acceptance_time_is_initialized + end) end module Rejected_blocks = struct diff --git a/src/lib/mina_net2/validation_callback.ml b/src/lib/mina_net2/validation_callback.ml index cba4cac1326..53f8e5a4a20 100644 --- a/src/lib/mina_net2/validation_callback.ml +++ b/src/lib/mina_net2/validation_callback.ml @@ -32,7 +32,6 @@ let is_expired cb = | Some expires_at -> Time_ns.(now () >= expires_at) -(* module type Metric_intf = sig val validations_timed_out : Mina_metrics.Counter.t @@ -40,94 +39,58 @@ module type Metric_intf = sig val ignored : Mina_metrics.Counter.t - module Validation_time(Context : Mina_metrics.CONTEXT) : sig + module Validation_time (Context : Mina_metrics.CONTEXT) : sig val update : Time.Span.t -> unit end - module Processing_time(Context : Mina_metrics.CONTEXT) : sig + module Processing_time (Context : Mina_metrics.CONTEXT) : sig val update : Time.Span.t -> unit end - module Rejection_time(Context : Mina_metrics.CONTEXT) : sig + module Rejection_time (Context : Mina_metrics.CONTEXT) : sig val update : Time.Span.t -> unit end end -let metrics_of_message_type m = +let metrics_of_message_type m : (module Metric_intf) option = match m with | `Unknown -> None | `Block -> - Some (Mina_metrics.Network.Block.validations_timed_out) + Some (module Mina_metrics.Network.Block) | `Snark_work -> - Some (Mina_metrics.Network.Snark_work.validations_timed_out) + Some (module Mina_metrics.Network.Snark_work) | `Transaction -> - Some (Mina_metrics.Network.Transaction.validations_timed_out) - -*) + Some (module Mina_metrics.Network.Transaction) let record_timeout_metrics cb = Mina_metrics.(Counter.inc_one Network.validations_timed_out) ; - let counter = - match cb.message_type with - | `Unknown -> - None - | `Block -> - Some Mina_metrics.Network.Block.validations_timed_out - | `Snark_work -> - Some Mina_metrics.Network.Snark_work.validations_timed_out - | `Transaction -> - Some Mina_metrics.Network.Transaction.validations_timed_out - in - Option.iter ~f:Mina_metrics.Counter.inc_one counter - -let record_validation_metrics ~block_window_duration message_type - (result : validation_result) validation_time processing_time = - let open Mina_metrics.Network in + match metrics_of_message_type cb.message_type with + | None -> + () + | Some (module M) -> + Mina_metrics.Counter.inc_one M.validations_timed_out + +let record_validation_metrics message_type (result : validation_result) + validation_time processing_time ~block_window_duration = let module Context = struct let block_window_duration = block_window_duration end in - match message_type with - | `Unknown -> + match metrics_of_message_type message_type with + | None -> () - | `Block -> ( - let module Validation_time = Block.Validation_time (Context) in - let module Processing_time = Block.Processing_time (Context) in - let module Rejection_time = Block.Rejection_time (Context) in - match result with - | `Ignore -> - Mina_metrics.Counter.inc_one Block.ignored - | `Accept -> - Validation_time.update validation_time ; - Processing_time.update processing_time - | `Reject -> - Mina_metrics.Counter.inc_one Block.rejected ; - Rejection_time.update processing_time ) - | `Snark_work -> ( - let module Validation_time = Snark_work.Validation_time (Context) in - let module Processing_time = Snark_work.Processing_time (Context) in - let module Rejection_time = Snark_work.Rejection_time (Context) in - match result with - | `Ignore -> - Mina_metrics.Counter.inc_one Block.ignored - | `Accept -> - Validation_time.update validation_time ; - Processing_time.update processing_time - | `Reject -> - Mina_metrics.Counter.inc_one Block.rejected ; - Rejection_time.update processing_time ) - | `Transaction -> ( - let module Validation_time = Transaction.Validation_time (Context) in - let module Processing_time = Transaction.Processing_time (Context) in - let module Rejection_time = Transaction.Rejection_time (Context) in + | Some (module M) -> ( match result with | `Ignore -> - Mina_metrics.Counter.inc_one Block.ignored + Mina_metrics.Counter.inc_one M.ignored | `Accept -> + let module Validation_time = M.Validation_time (Context) in Validation_time.update validation_time ; + let module Processing_time = M.Processing_time (Context) in Processing_time.update processing_time | `Reject -> - Mina_metrics.Counter.inc_one Block.rejected ; + Mina_metrics.Counter.inc_one M.rejected ; + let module Rejection_time = M.Rejection_time (Context) in Rejection_time.update processing_time ) let await_timeout cb = From a0750c86e042ef5c139447ed56693d473eeaa9b0 Mon Sep 17 00:00:00 2001 From: martyall Date: Mon, 9 Sep 2024 13:45:07 -0700 Subject: [PATCH 05/14] use gauge maps --- .../prometheus_metrics/metric_generators.ml | 13 ++-- .../prometheus_metrics/mina_metrics.ml | 78 +++++++++++++++++++ 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml b/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml index f9168fab806..bac3bb4da2e 100644 --- a/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml +++ b/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml @@ -1,7 +1,6 @@ open Core_kernel open Async_kernel open Prometheus -open Namespace module type Metric_spec_intf = sig val subsystem : string @@ -20,6 +19,8 @@ module type Bucketed_average_spec_intf = sig val render_average : (float * int) list -> float + val v : Gauge.t + val is_initialized : bool ref end @@ -42,18 +43,20 @@ module type Time_average_spec_intf = sig val intervals : Intervals.t + val v : Gauge.t + val is_initialized : bool ref end module type Moving_average_metric_intf = sig type datum - val v : Gauge.t - val update : datum -> unit val clear : unit -> unit + val v : Gauge.t + val initialize : unit -> unit end @@ -61,8 +64,6 @@ module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) : Moving_average_metric_intf with type datum := float = struct open Spec - let v = Gauge.v name ~subsystem ~namespace ~help - let empty_bucket_entry = (0.0, 0) let empty_buckets () = List.init num_buckets ~f:(Fn.const empty_bucket_entry) @@ -79,6 +80,8 @@ module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) : | (value, num_entries) :: t -> buckets := Some ((value +. datum, num_entries + 1) :: t) + let v = Spec.v + let initialize () = let rec tick () = upon diff --git a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml index 1e126cda84c..da909d3cf59 100644 --- a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml +++ b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml @@ -465,6 +465,16 @@ module Network = struct let rejection_time_is_initialized = ref false + module Gauge_map = Metric_map (struct + type t = Gauge.t + + let subsystem = subsystem + + let v = Gauge.v + end) + + let block_table = Gauge_map.of_alist_exn [] + module Validation_time (Context : CONTEXT) = Moving_time_average (struct include Delay_time_spec (Context) @@ -475,6 +485,8 @@ module Network = struct let help = "average time, in ms, for blocks to be validated and rebroadcasted" + let v = Gauge_map.add block_table ~name ~help + let is_initialized = validation_time_is_initialized end) @@ -489,6 +501,8 @@ module Network = struct "average time, in ms, for blocks to be accepted after the OCaml \ process receives it" + let v = Gauge_map.add block_table ~name ~help + let is_initialized = processing_time_is_initialized end) @@ -503,6 +517,8 @@ module Network = struct "average time, in ms, for blocks to be rejected after the OCaml \ process receives it" + let v = Gauge_map.add block_table ~name ~help + let is_initialized = rejection_time_is_initialized end) end @@ -532,6 +548,16 @@ module Network = struct let rejection_time_is_initialized = ref false + module Gauge_map = Metric_map (struct + type t = Gauge.t + + let subsystem = subsystem + + let v = Gauge.v + end) + + let snark_work_table = Gauge_map.of_alist_exn [] + module Validation_time (Context : CONTEXT) = Moving_time_average (struct include Delay_time_spec (Context) @@ -542,6 +568,8 @@ module Network = struct let help = "average delay, in ms, for snark work to be validated and rebroadcasted" + let v = Gauge_map.add snark_work_table ~name ~help + let is_initialized = validation_time_is_initialized end) @@ -556,6 +584,8 @@ module Network = struct "average delay, in ms, for snark work to be accepted after the OCaml \ process receives it" + let v = Gauge_map.add snark_work_table ~name ~help + let is_initialized = processing_time_is_initialized end) @@ -570,6 +600,8 @@ module Network = struct "average time, in ms, for snark work to be rejected after the OCaml \ process receives it" + let v = Gauge_map.add snark_work_table ~name ~help + let is_initialized = rejection_time_is_initialized end) end @@ -599,6 +631,16 @@ module Network = struct let rejection_time_is_initialized = ref false + module Gauge_map = Metric_map (struct + type t = Gauge.t + + let subsystem = subsystem + + let v = Gauge.v + end) + + let transaction_table = Gauge_map.of_alist_exn [] + module Validation_time (Context : CONTEXT) = Moving_time_average (struct include Delay_time_spec (Context) @@ -610,6 +652,8 @@ module Network = struct "average delay, in ms, for transactions to be validated and \ rebroadcasted" + let v = Gauge_map.add transaction_table ~name ~help + let is_initialized = validation_time_is_initialized end) @@ -624,6 +668,8 @@ module Network = struct "average delay, in ms, for transactions to be accepted after the OCaml \ process receives it" + let v = Gauge_map.add transaction_table ~name ~help + let is_initialized = processing_time_is_initialized end) @@ -638,6 +684,8 @@ module Network = struct "average time, in ms, for transactions to be rejected after the OCaml \ process receives it" + let v = Gauge_map.add transaction_table ~name ~help + let is_initialized = rejection_time_is_initialized end) end @@ -1285,6 +1333,16 @@ module Transition_frontier = struct let tps_30min_is_initialized = ref false + module Gauge_map = Metric_map (struct + type t = Gauge.t + + let subsystem = subsystem + + let v = Gauge.v + end) + + let transaction_frontier_table = Gauge_map.of_alist_exn [] + module TPS_30min = Moving_bucketed_average (struct let bucket_interval = Core.Time.Span.of_min 3.0 @@ -1302,6 +1360,8 @@ module Transition_frontier = struct "moving average for transaction per second, the rolling interval is set \ to 30 min" + let v = Gauge_map.add transaction_frontier_table ~name ~help + let is_initialized = tps_30min_is_initialized end) @@ -1458,6 +1518,16 @@ module Block_latency = struct let gossip_slots_is_initialized = ref false + module Gauge_map = Metric_map (struct + type t = Gauge.t + + let subsystem = subsystem + + let v = Gauge.v + end) + + let block_latency_table = Gauge_map.of_alist_exn [] + module Gossip_slots (Context : CONTEXT) = Moving_bucketed_average (struct let bucket_interval = Time.Span.scale Context.block_window_duration 0.5 @@ -1478,6 +1548,8 @@ module Block_latency = struct in total_sum /. Float.of_int count_sum + let v = Gauge_map.add block_latency_table ~name ~help + let is_initialized = gossip_slots_is_initialized end) @@ -1493,6 +1565,8 @@ module Block_latency = struct let help = "average delay, in seconds, after which produced blocks are received" + let v = Gauge_map.add block_latency_table ~name ~help + let is_initialized = gossip_time_is_initialized end) @@ -1509,6 +1583,8 @@ module Block_latency = struct "average delay, in seconds, after which produced blocks are included \ into our frontier" + let v = Gauge_map.add block_latency_table ~name ~help + let is_initialized = inclusion_time_is_initialized end) @@ -1526,6 +1602,8 @@ module Block_latency = struct "average delay, in seconds, between the time blocks are initially \ received from the libp2p_helper, and when they are accepted as valid" + let v = Gauge_map.add block_latency_table ~name ~help + let is_initialized = validation_acceptance_time_is_initialized end) end From 490dafca4cde90c2bf06a689da5cacc2a2722fac Mon Sep 17 00:00:00 2001 From: georgeee Date: Tue, 17 Sep 2024 20:16:40 +0000 Subject: [PATCH 06/14] Alternative take on initializing metrics --- .../src/cli_entrypoint/mina_cli_entrypoint.ml | 13 ++ src/lib/mina_metrics/mina_metrics.mli | 40 ++-- .../mina_metrics/no_metrics/mina_metrics.ml | 40 ++-- .../prometheus_metrics/metric_generators.ml | 57 +++--- .../prometheus_metrics/mina_metrics.ml | 179 ++++-------------- src/lib/mina_net2/validation_callback.ml | 17 +- src/lib/transition_handler/block_sink.ml | 11 +- src/lib/transition_handler/processor.ml | 8 +- 8 files changed, 137 insertions(+), 228 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index f29d32a86b2..4d6035ea5f0 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -783,6 +783,19 @@ let setup_daemon logger ~itn_features ~default_snark_worker_fee = ~proof_level:Genesis_constants.Compiled.proof_level config_files ~genesis_constants ~constraint_constants ~cli_proof_level in + + (* TODO consider moving elsewhere *) + let duration = + constraint_constants.block_window_duration_ms |> Float.of_int + |> Time.Span.of_ms + in + Mina_metrics.Transition_frontier.TPS_30min.initialize duration ; + Mina_metrics.Block_latency.Gossip_slots.initialize duration ; + Mina_metrics.Block_latency.Gossip_time.initialize duration ; + Mina_metrics.Block_latency.Inclusion_time.initialize duration ; + Mina_metrics.Block_latency.Validation_acceptance_time.initialize + duration ; + let rev_daemon_configs = List.rev_filter_map config_jsons ~f:(fun (config_file, config_json) -> diff --git a/src/lib/mina_metrics/mina_metrics.mli b/src/lib/mina_metrics/mina_metrics.mli index ba6a825165d..2007b5cb9f1 100644 --- a/src/lib/mina_metrics/mina_metrics.mli +++ b/src/lib/mina_metrics/mina_metrics.mli @@ -3,10 +3,6 @@ open Async_kernel val time_offset_sec : float -module type CONTEXT = sig - val block_window_duration : Time.Span.t -end - module Counter : sig type t @@ -125,15 +121,15 @@ module Network : sig val received : Counter.t - module Validation_time (Context : CONTEXT) : sig + module Validation_time : sig val update : Time.Span.t -> unit end - module Processing_time (Context : CONTEXT) : sig + module Processing_time : sig val update : Time.Span.t -> unit end - module Rejection_time (Context : CONTEXT) : sig + module Rejection_time : sig val update : Time.Span.t -> unit end end @@ -147,15 +143,15 @@ module Network : sig val received : Counter.t - module Validation_time (Context : CONTEXT) : sig + module Validation_time : sig val update : Time.Span.t -> unit end - module Processing_time (Context : CONTEXT) : sig + module Processing_time : sig val update : Time.Span.t -> unit end - module Rejection_time (Context : CONTEXT) : sig + module Rejection_time : sig val update : Time.Span.t -> unit end end @@ -169,15 +165,15 @@ module Network : sig val received : Counter.t - module Validation_time (Context : CONTEXT) : sig + module Validation_time : sig val update : Time.Span.t -> unit end - module Processing_time (Context : CONTEXT) : sig + module Processing_time : sig val update : Time.Span.t -> unit end - module Rejection_time (Context : CONTEXT) : sig + module Rejection_time : sig val update : Time.Span.t -> unit end end @@ -428,6 +424,8 @@ module Transition_frontier : sig val update : float -> unit val clear : unit -> unit + + val initialize : Core_kernel.Time.Span.t -> unit end val recently_finalized_staged_txns : Gauge.t @@ -484,36 +482,44 @@ module Block_latency : sig val upload_to_gcloud_blocks : Gauge.t end - module Gossip_slots (Context : CONTEXT) : sig + module Gossip_slots : sig val v : Gauge.t val update : float -> unit val clear : unit -> unit + + val initialize : Core_kernel.Time.Span.t -> unit end - module Gossip_time (Context : CONTEXT) : sig + module Gossip_time : sig val v : Gauge.t val update : Time.Span.t -> unit val clear : unit -> unit + + val initialize : Core_kernel.Time.Span.t -> unit end - module Inclusion_time (Context : CONTEXT) : sig + module Inclusion_time : sig val v : Gauge.t val update : Time.Span.t -> unit val clear : unit -> unit + + val initialize : Core_kernel.Time.Span.t -> unit end - module Validation_acceptance_time (Context : CONTEXT) : sig + module Validation_acceptance_time : sig val v : Gauge.t val update : Time.Span.t -> unit val clear : unit -> unit + + val initialize : Core_kernel.Time.Span.t -> unit end end diff --git a/src/lib/mina_metrics/no_metrics/mina_metrics.ml b/src/lib/mina_metrics/no_metrics/mina_metrics.ml index ea0e73c87b6..d87319d626f 100644 --- a/src/lib/mina_metrics/no_metrics/mina_metrics.ml +++ b/src/lib/mina_metrics/no_metrics/mina_metrics.ml @@ -3,10 +3,6 @@ open Async_kernel let time_offset_sec = 1609459200. -module type CONTEXT = sig - val block_window_duration : Time.Span.t -end - module Counter = struct type t = unit @@ -135,15 +131,15 @@ module Network = struct let received : Counter.t = () - module Validation_time (Context : CONTEXT) = struct + module Validation_time = struct let update : Time.Span.t -> unit = Fn.ignore end - module Processing_time (Context : CONTEXT) = struct + module Processing_time = struct let update : Time.Span.t -> unit = Fn.ignore end - module Rejection_time (Context : CONTEXT) = struct + module Rejection_time = struct let update : Time.Span.t -> unit = Fn.ignore end end @@ -157,15 +153,15 @@ module Network = struct let received : Counter.t = () - module Validation_time (Context : CONTEXT) = struct + module Validation_time = struct let update : Time.Span.t -> unit = Fn.ignore end - module Processing_time (Context : CONTEXT) = struct + module Processing_time = struct let update : Time.Span.t -> unit = Fn.ignore end - module Rejection_time (Context : CONTEXT) = struct + module Rejection_time = struct let update : Time.Span.t -> unit = Fn.ignore end end @@ -179,15 +175,15 @@ module Network = struct let received : Counter.t = () - module Validation_time (Context : CONTEXT) = struct + module Validation_time = struct let update : Time.Span.t -> unit = Fn.ignore end - module Processing_time (Context : CONTEXT) = struct + module Processing_time = struct let update : Time.Span.t -> unit = Fn.ignore end - module Rejection_time (Context : CONTEXT) = struct + module Rejection_time = struct let update : Time.Span.t -> unit = Fn.ignore end end @@ -440,6 +436,8 @@ module Transition_frontier = struct let update : float -> unit = fun _ -> () let clear : unit -> unit = fun _ -> () + + let initialize = Fn.ignore end let recently_finalized_staged_txns : Gauge.t = () @@ -496,36 +494,44 @@ module Block_latency = struct let upload_to_gcloud_blocks : Gauge.t = () end - module Gossip_slots (Context : CONTEXT) = struct + module Gossip_slots = struct let v : Gauge.t = () let update : float -> unit = fun _ -> () let clear : unit -> unit = fun _ -> () + + let initialize = Fn.ignore end - module Gossip_time (Context : CONTEXT) = struct + module Gossip_time = struct let v : Gauge.t = () let update : Time.Span.t -> unit = fun _ -> () let clear : unit -> unit = fun _ -> () + + let initialize = Fn.ignore end - module Inclusion_time (Context : CONTEXT) = struct + module Inclusion_time = struct let v : Gauge.t = () let update : Time.Span.t -> unit = fun _ -> () let clear : unit -> unit = fun _ -> () + + let initialize = Fn.ignore end - module Validation_acceptance_time (Context : CONTEXT) = struct + module Validation_acceptance_time = struct let v : Gauge.t = () let update : Time.Span.t -> unit = fun _ -> () let clear : unit -> unit = fun _ -> () + + let initialize = Fn.ignore end end diff --git a/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml b/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml index bac3bb4da2e..12033d14074 100644 --- a/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml +++ b/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml @@ -13,15 +13,11 @@ end module type Bucketed_average_spec_intf = sig include Metric_spec_intf - val bucket_interval : Core.Time.Span.t + val bucket_interval : Core.Time.Span.t -> Core.Time.Span.t - val num_buckets : int + val num_buckets : Core.Time.Span.t -> int val render_average : (float * int) list -> float - - val v : Gauge.t - - val is_initialized : bool ref end module Intervals = struct @@ -41,11 +37,7 @@ end module type Time_average_spec_intf = sig include Metric_spec_intf - val intervals : Intervals.t - - val v : Gauge.t - - val is_initialized : bool ref + val intervals : Time.Span.t -> Intervals.t end module type Moving_average_metric_intf = sig @@ -57,7 +49,7 @@ module type Moving_average_metric_intf = sig val v : Gauge.t - val initialize : unit -> unit + val initialize : Core.Time.Span.t -> unit end module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) : @@ -66,28 +58,34 @@ module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) : let empty_bucket_entry = (0.0, 0) - let empty_buckets () = List.init num_buckets ~f:(Fn.const empty_bucket_entry) - let buckets = ref None - let clear () = buckets := Some (empty_buckets ()) + let clear () = + buckets := + Option.map ~f:(List.map ~f:(Fn.const empty_bucket_entry)) !buckets let update datum = - if Option.is_none !buckets then buckets := Some (empty_buckets ()) ; - match Option.value_exn !buckets with - | [] -> + match !buckets with + | None -> + () + | Some [] -> failwith "Moving_bucketed_average buckets are malformed" - | (value, num_entries) :: t -> + | Some ((value, num_entries) :: t) -> buckets := Some ((value +. datum, num_entries + 1) :: t) - let v = Spec.v + let v = Gauge.v name ~subsystem ~namespace:Namespace.namespace ~help - let initialize () = + let initialize block_window_duration = let rec tick () = upon - (after (Time_ns.Span.of_ns @@ Time.Span.to_ns bucket_interval)) + (after + ( Time_ns.Span.of_ns @@ Time.Span.to_ns + @@ bucket_interval block_window_duration ) ) (fun () -> - if Option.is_none !buckets then buckets := Some (empty_buckets ()) ; + let num_buckets = num_buckets block_window_duration in + if Option.is_none !buckets then + buckets := + Some (List.init num_buckets ~f:(Fn.const empty_bucket_entry)) ; let buckets_val = Option.value_exn !buckets in Gauge.set v (render_average buckets_val) ; buckets := @@ -102,12 +100,13 @@ module Moving_time_average (Spec : Time_average_spec_intf) : include Moving_bucketed_average (struct include Spec - let bucket_interval = Spec.intervals.rolling + let bucket_interval block_window_duration = + (Spec.intervals block_window_duration).rolling - let num_buckets = + let num_buckets block_window_duration = + let intervals = Spec.intervals block_window_duration in Float.to_int - ( Time.Span.to_ns Spec.intervals.rolling - /. Time.Span.to_ns Spec.intervals.tick ) + (Time.Span.to_ns intervals.rolling /. Time.Span.to_ns intervals.tick) let render_average buckets = let total_sum, count_sum = @@ -118,9 +117,5 @@ module Moving_time_average (Spec : Time_average_spec_intf) : total_sum /. Float.of_int count_sum end) - let () = - if not !Spec.is_initialized then initialize () ; - Spec.is_initialized := true - let update span = update (Core.Time.Span.to_sec span) end diff --git a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml index da909d3cf59..61cd15c22da 100644 --- a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml +++ b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml @@ -11,10 +11,6 @@ open Async_kernel let time_offset_sec = 1609459200. -module type CONTEXT = sig - val block_window_duration : Time.Span.t -end - (* textformat serialization and runtime metrics taken from github.com/mirage/prometheus:/app/prometheus_app.ml *) module TextFormat_0_0_4 = struct let re_unquoted_escapes = Re.compile @@ Re.set "\\\n" @@ -225,8 +221,7 @@ module Runtime = struct let process_uptime_ms_total = simple_metric ~metric_type:Counter "process_uptime_ms_total" - (fun () -> - Core.Time.Span.to_ms (Core.Time.diff (Core.Time.now ()) start_time) ) + (fun () -> Time.Span.to_ms (Core.Time.diff (Core.Time.now ()) start_time)) ~help:"Total time the process has been running for in milliseconds." let metrics = @@ -434,10 +429,10 @@ module Network = struct let help = "# of messages received" in Counter.v "messages_received" ~help ~namespace ~subsystem - module Delay_time_spec (Context : CONTEXT) = struct - let intervals = - Intervals.make ~rolling_interval:Context.block_window_duration - ~tick_interval:Context.block_window_duration + module Delay_time_spec = struct + let intervals block_window_duration = + Intervals.make ~rolling_interval:block_window_duration + ~tick_interval:block_window_duration end module Block = struct @@ -459,12 +454,6 @@ module Network = struct let help = "# of blocks received" in Counter.v "received" ~help ~namespace ~subsystem - let validation_time_is_initialized = ref false - - let processing_time_is_initialized = ref false - - let rejection_time_is_initialized = ref false - module Gauge_map = Metric_map (struct type t = Gauge.t @@ -473,10 +462,8 @@ module Network = struct let v = Gauge.v end) - let block_table = Gauge_map.of_alist_exn [] - - module Validation_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Validation_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -484,14 +471,10 @@ module Network = struct let help = "average time, in ms, for blocks to be validated and rebroadcasted" - - let v = Gauge_map.add block_table ~name ~help - - let is_initialized = validation_time_is_initialized end) - module Processing_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Processing_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -500,14 +483,10 @@ module Network = struct let help = "average time, in ms, for blocks to be accepted after the OCaml \ process receives it" - - let v = Gauge_map.add block_table ~name ~help - - let is_initialized = processing_time_is_initialized end) - module Rejection_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Rejection_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -516,10 +495,6 @@ module Network = struct let help = "average time, in ms, for blocks to be rejected after the OCaml \ process receives it" - - let v = Gauge_map.add block_table ~name ~help - - let is_initialized = rejection_time_is_initialized end) end @@ -542,12 +517,6 @@ module Network = struct let help = "# of snark work received" in Counter.v "received" ~help ~namespace ~subsystem - let validation_time_is_initialized = ref false - - let processing_time_is_initialized = ref false - - let rejection_time_is_initialized = ref false - module Gauge_map = Metric_map (struct type t = Gauge.t @@ -556,10 +525,8 @@ module Network = struct let v = Gauge.v end) - let snark_work_table = Gauge_map.of_alist_exn [] - - module Validation_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Validation_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -567,14 +534,10 @@ module Network = struct let help = "average delay, in ms, for snark work to be validated and rebroadcasted" - - let v = Gauge_map.add snark_work_table ~name ~help - - let is_initialized = validation_time_is_initialized end) - module Processing_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Processing_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -583,14 +546,10 @@ module Network = struct let help = "average delay, in ms, for snark work to be accepted after the OCaml \ process receives it" - - let v = Gauge_map.add snark_work_table ~name ~help - - let is_initialized = processing_time_is_initialized end) - module Rejection_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Rejection_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -599,10 +558,6 @@ module Network = struct let help = "average time, in ms, for snark work to be rejected after the OCaml \ process receives it" - - let v = Gauge_map.add snark_work_table ~name ~help - - let is_initialized = rejection_time_is_initialized end) end @@ -625,12 +580,6 @@ module Network = struct let help = "# of transactions received" in Counter.v "received" ~help ~namespace ~subsystem - let validation_time_is_initialized = ref false - - let processing_time_is_initialized = ref false - - let rejection_time_is_initialized = ref false - module Gauge_map = Metric_map (struct type t = Gauge.t @@ -639,10 +588,8 @@ module Network = struct let v = Gauge.v end) - let transaction_table = Gauge_map.of_alist_exn [] - - module Validation_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Validation_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -651,14 +598,10 @@ module Network = struct let help = "average delay, in ms, for transactions to be validated and \ rebroadcasted" - - let v = Gauge_map.add transaction_table ~name ~help - - let is_initialized = validation_time_is_initialized end) - module Processing_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Processing_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -667,14 +610,10 @@ module Network = struct let help = "average delay, in ms, for transactions to be accepted after the OCaml \ process receives it" - - let v = Gauge_map.add transaction_table ~name ~help - - let is_initialized = processing_time_is_initialized end) - module Rejection_time (Context : CONTEXT) = Moving_time_average (struct - include Delay_time_spec (Context) + module Rejection_time = Moving_time_average (struct + include Delay_time_spec let subsystem = subsystem @@ -683,10 +622,6 @@ module Network = struct let help = "average time, in ms, for transactions to be rejected after the OCaml \ process receives it" - - let v = Gauge_map.add transaction_table ~name ~help - - let is_initialized = rejection_time_is_initialized end) end @@ -1331,8 +1266,6 @@ module Transition_frontier = struct let help = "total # of staged txns that have been finalized" in Counter.v "finalized_staged_txns" ~help ~namespace ~subsystem - let tps_30min_is_initialized = ref false - module Gauge_map = Metric_map (struct type t = Gauge.t @@ -1341,16 +1274,14 @@ module Transition_frontier = struct let v = Gauge.v end) - let transaction_frontier_table = Gauge_map.of_alist_exn [] - module TPS_30min = Moving_bucketed_average (struct - let bucket_interval = Core.Time.Span.of_min 3.0 + let bucket_interval _ = Time.Span.of_min 3.0 - let num_buckets = 10 + let num_buckets _ = 10 let render_average buckets = let total = List.fold buckets ~init:0.0 ~f:(fun acc (n, _) -> acc +. n) in - total /. Core.Time.Span.(of_min 30.0 |> to_sec) + total /. Time.Span.(of_min 30.0 |> to_sec) let subsystem = subsystem @@ -1359,10 +1290,6 @@ module Transition_frontier = struct let help = "moving average for transaction per second, the rolling interval is set \ to 30 min" - - let v = Gauge_map.add transaction_frontier_table ~name ~help - - let is_initialized = tps_30min_is_initialized end) let recently_finalized_staged_txns : Gauge.t = @@ -1509,15 +1436,13 @@ module Block_latency = struct Gauge.v "upload_to_gcloud_blocks" ~help ~namespace ~subsystem end - module Latency_time_spec (Context : CONTEXT) = struct - let intervals = + module Latency_time_spec = struct + let intervals block_window_duration = Intervals.make - ~tick_interval:(Time.Span.scale Context.block_window_duration 0.5) - ~rolling_interval:(Time.Span.scale Context.block_window_duration 20.0) + ~tick_interval:(Time.Span.scale block_window_duration 0.5) + ~rolling_interval:(Time.Span.scale block_window_duration 20.0) end - let gossip_slots_is_initialized = ref false - module Gauge_map = Metric_map (struct type t = Gauge.t @@ -1526,12 +1451,11 @@ module Block_latency = struct let v = Gauge.v end) - let block_latency_table = Gauge_map.of_alist_exn [] + module Gossip_slots = Moving_bucketed_average (struct + let bucket_interval block_window_duration = + Time.Span.scale block_window_duration 0.5 - module Gossip_slots (Context : CONTEXT) = Moving_bucketed_average (struct - let bucket_interval = Time.Span.scale Context.block_window_duration 0.5 - - let num_buckets = 40 + let num_buckets _ = 40 let subsystem = subsystem @@ -1547,16 +1471,10 @@ module Block_latency = struct (total_sum +. total, count_sum + count) ) in total_sum /. Float.of_int count_sum - - let v = Gauge_map.add block_latency_table ~name ~help - - let is_initialized = gossip_slots_is_initialized end) - let gossip_time_is_initialized = ref false - - module Gossip_time (Context : CONTEXT) = Moving_time_average (struct - include Latency_time_spec (Context) + module Gossip_time = Moving_time_average (struct + include Latency_time_spec let subsystem = subsystem @@ -1564,16 +1482,10 @@ module Block_latency = struct let help = "average delay, in seconds, after which produced blocks are received" - - let v = Gauge_map.add block_latency_table ~name ~help - - let is_initialized = gossip_time_is_initialized end) - let inclusion_time_is_initialized = ref false - - module Inclusion_time (Context : CONTEXT) = Moving_time_average (struct - include Latency_time_spec (Context) + module Inclusion_time = Moving_time_average (struct + include Latency_time_spec let subsystem = subsystem @@ -1582,17 +1494,10 @@ module Block_latency = struct let help = "average delay, in seconds, after which produced blocks are included \ into our frontier" - - let v = Gauge_map.add block_latency_table ~name ~help - - let is_initialized = inclusion_time_is_initialized end) - let validation_acceptance_time_is_initialized = ref false - - module Validation_acceptance_time (Context : CONTEXT) = - Moving_time_average (struct - include Latency_time_spec (Context) + module Validation_acceptance_time = Moving_time_average (struct + include Latency_time_spec let subsystem = subsystem @@ -1601,10 +1506,6 @@ module Block_latency = struct let help = "average delay, in seconds, between the time blocks are initially \ received from the libp2p_helper, and when they are accepted as valid" - - let v = Gauge_map.add block_latency_table ~name ~help - - let is_initialized = validation_acceptance_time_is_initialized end) end diff --git a/src/lib/mina_net2/validation_callback.ml b/src/lib/mina_net2/validation_callback.ml index 53f8e5a4a20..5fb99cf0545 100644 --- a/src/lib/mina_net2/validation_callback.ml +++ b/src/lib/mina_net2/validation_callback.ml @@ -39,15 +39,15 @@ module type Metric_intf = sig val ignored : Mina_metrics.Counter.t - module Validation_time (Context : Mina_metrics.CONTEXT) : sig + module Validation_time : sig val update : Time.Span.t -> unit end - module Processing_time (Context : Mina_metrics.CONTEXT) : sig + module Processing_time : sig val update : Time.Span.t -> unit end - module Rejection_time (Context : Mina_metrics.CONTEXT) : sig + module Rejection_time : sig val update : Time.Span.t -> unit end end @@ -72,10 +72,7 @@ let record_timeout_metrics cb = Mina_metrics.Counter.inc_one M.validations_timed_out let record_validation_metrics message_type (result : validation_result) - validation_time processing_time ~block_window_duration = - let module Context = struct - let block_window_duration = block_window_duration - end in + validation_time processing_time ~block_window_duration:_ (*TODO remove*) = match metrics_of_message_type message_type with | None -> () @@ -84,13 +81,13 @@ let record_validation_metrics message_type (result : validation_result) | `Ignore -> Mina_metrics.Counter.inc_one M.ignored | `Accept -> - let module Validation_time = M.Validation_time (Context) in + let module Validation_time = M.Validation_time in Validation_time.update validation_time ; - let module Processing_time = M.Processing_time (Context) in + let module Processing_time = M.Processing_time in Processing_time.update processing_time | `Reject -> Mina_metrics.Counter.inc_one M.rejected ; - let module Rejection_time = M.Rejection_time (Context) in + let module Rejection_time = M.Rejection_time in Rejection_time.update processing_time ) let await_timeout cb = diff --git a/src/lib/transition_handler/block_sink.ml b/src/lib/transition_handler/block_sink.ml index 04781e3d582..dd9ac11e3a7 100644 --- a/src/lib/transition_handler/block_sink.ml +++ b/src/lib/transition_handler/block_sink.ml @@ -60,9 +60,6 @@ let push sink (`Transition e, `Time_received tm, `Valid_cb cb) = O1trace.sync_thread "handle_block_gossip" @@ fun () -> let%bind () = on_push () in - let module Metrics_Context = struct - let block_window_duration = block_window_duration - end in Mina_metrics.(Counter.inc_one Network.gossip_messages_received) ; let state = Envelope.Incoming.data e in let state_hash = @@ -102,7 +99,7 @@ let push sink (`Transition e, `Time_received tm, `Valid_cb cb) = in let module Validation_acceptance_time = Mina_metrics.Block_latency.Validation_acceptance_time - (Metrics_Context) in + in Validation_acceptance_time.update processing_time_span | Some _ -> () @@ -192,11 +189,9 @@ let push sink (`Transition e, `Time_received tm, `Valid_cb cb) = (Consensus.Data.Consensus_time.of_time_exn ~constants:consensus_constants tm ) in - let module Gossip_slots = - Mina_metrics.Block_latency.Gossip_slots (Metrics_Context) in + let module Gossip_slots = Mina_metrics.Block_latency.Gossip_slots in Gossip_slots.update (Float.of_int (tm_slot - tn_production_slot)) ; - let module Gossip_time = - Mina_metrics.Block_latency.Gossip_time (Metrics_Context) in + let module Gossip_time = Mina_metrics.Block_latency.Gossip_time in Gossip_time.update Block_time.(Span.to_time_span @@ diff tm tn_production_time) ; Deferred.unit diff --git a/src/lib/transition_handler/processor.ml b/src/lib/transition_handler/processor.ml index 1940c596f22..08aca0f3d79 100644 --- a/src/lib/transition_handler/processor.ml +++ b/src/lib/transition_handler/processor.ml @@ -47,12 +47,8 @@ let cached_transform_deferred_result ~transform_cached ~transform_result cached let add_and_finalize ~logger ~frontier ~catchup_scheduler ~processed_transition_writer ~only_if_present ~time_controller ~source ~valid_cb cached_breadcrumb ~(precomputed_values : Precomputed_values.t) - ~block_window_duration = - let module Metrics_context = struct - let block_window_duration = block_window_duration - end in - let module Inclusion_time = - Mina_metrics.Block_latency.Inclusion_time (Metrics_context) in + ~block_window_duration:_ (*TODO remove unused var*) = + let module Inclusion_time = Mina_metrics.Block_latency.Inclusion_time in let breadcrumb = if Cached.is_pure cached_breadcrumb then Cached.peek cached_breadcrumb else Cached.invalidate_with_success cached_breadcrumb From 4ae63626dfca884de217a76b05ef383df75d575e Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 22:32:57 +0200 Subject: [PATCH 07/14] correct BatchTxn deps and revert removal of MINA_DEB_CODENAME --- buildkite/scripts/export-git-env-vars.sh | 7 ++++++- scripts/export-git-env-vars.sh | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index dfe4e0dd9fd..cdbcf369532 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -25,7 +25,12 @@ export BUILD_URL=${BUILDKITE_BUILD_URL} set -u export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=bullseye} -[[ -n "$BUILDKITE_BRANCH" ]] && export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g; s!#!-!g') || export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +if [[ -n "$BUILDKITE_BRANCH" ]]; then + export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g; s!#!-!g') +else + export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +fi + export RELEASE=unstable diff --git a/scripts/export-git-env-vars.sh b/scripts/export-git-env-vars.sh index 373e2c57d5a..c9f09e101e5 100755 --- a/scripts/export-git-env-vars.sh +++ b/scripts/export-git-env-vars.sh @@ -22,7 +22,11 @@ function find_most_recent_numeric_tag() { export GITHASH=$(git rev-parse --short=7 HEAD) export THIS_COMMIT_TAG=$(git tag --points-at HEAD) -[[ -n "$BRANCH_NAME" ]] && export GITBRANCH=$(echo "$BRANCH_NAME" | sed 's!/!-!g; s!_!-!g; s!#!-!g') || export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +if [[ -n "$BRANCH_NAME" ]]; then + export GITBRANCH=$(echo "$BRANCH_NAME" | sed 's!/!-!g; s!_!-!g; s!#!-!g') +else + export GITBRANCH=$(git name-rev --name-only $GITHASH | sed "s/remotes\/origin\///g" | sed 's!/!-!g; s!_!-!g; s!#!-!g' ) +fi export GITTAG=$(find_most_recent_numeric_tag HEAD) From 6e86ebcc30643140967ccab19b996371fadd6cf7 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 17 Sep 2024 22:33:04 +0200 Subject: [PATCH 08/14] do not bother checking if hashicorp.list file exists --- buildkite/scripts/run-test-executive-local.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 7312a19941a..09c37c2b206 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -37,6 +37,8 @@ fi # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive +rm -f /etc/apt/sources.list.d/hashicorp.list + apt-get update apt-get install -y git apt-transport-https ca-certificates tzdata curl From 954475aa0042089da37c14f37a4ae8fbec842e7d Mon Sep 17 00:00:00 2001 From: svv232 Date: Tue, 17 Sep 2024 17:02:10 -0400 Subject: [PATCH 09/14] adding limit flag to verifier --- src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index f9b4f49f279..2bda3fb1d67 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -1769,6 +1769,9 @@ let internal_commands logger = and format = flag "--format" ~aliases:[ "-format" ] (optional string) ~doc:"sexp/json the format to parse input in" + and limit = + flag "--limit" ~aliases:[ "-limit" ] (optional int) + ~doc:"limit the number of proofs taken from the file" in fun () -> let open Async in @@ -1857,11 +1860,14 @@ let internal_commands logger = ~conf_dir:(Some conf_dir) () in let%bind result = + let cap lst = + Option.value_map ~default:Fn.id ~f:(Fn.flip List.take) limit lst + in match input with | `Transaction input -> - Verifier.verify_transaction_snarks verifier input + input |> cap |> Verifier.verify_transaction_snarks verifier | `Blockchain input -> - Verifier.verify_blockchain_snarks verifier input + input |> cap |> Verifier.verify_blockchain_snarks verifier in match result with | Ok (Ok ()) -> From 39362ea1b61d38065e74e84c0172058507552f33 Mon Sep 17 00:00:00 2001 From: georgeee Date: Wed, 18 Sep 2024 07:47:12 +0000 Subject: [PATCH 10/14] fixup! Alternative take on initializing metrics --- .../cli/src/cli_entrypoint/mina_cli_entrypoint.ml | 13 ++----------- src/lib/mina_metrics/mina_metrics.mli | 2 ++ src/lib/mina_metrics/no_metrics/mina_metrics.ml | 2 ++ .../prometheus_metrics/metric_generators.ml | 10 +++++----- .../mina_metrics/prometheus_metrics/mina_metrics.ml | 7 +++++++ 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index 4d6035ea5f0..525ed1565ec 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -784,17 +784,8 @@ let setup_daemon logger ~itn_features ~default_snark_worker_fee = ~genesis_constants ~constraint_constants ~cli_proof_level in - (* TODO consider moving elsewhere *) - let duration = - constraint_constants.block_window_duration_ms |> Float.of_int - |> Time.Span.of_ms - in - Mina_metrics.Transition_frontier.TPS_30min.initialize duration ; - Mina_metrics.Block_latency.Gossip_slots.initialize duration ; - Mina_metrics.Block_latency.Gossip_time.initialize duration ; - Mina_metrics.Block_latency.Inclusion_time.initialize duration ; - Mina_metrics.Block_latency.Validation_acceptance_time.initialize - duration ; + constraint_constants.block_window_duration_ms |> Float.of_int + |> Time.Span.of_ms |> Mina_metrics.initialize_all ; let rev_daemon_configs = List.rev_filter_map config_jsons diff --git a/src/lib/mina_metrics/mina_metrics.mli b/src/lib/mina_metrics/mina_metrics.mli index 2007b5cb9f1..649f5869284 100644 --- a/src/lib/mina_metrics/mina_metrics.mli +++ b/src/lib/mina_metrics/mina_metrics.mli @@ -563,3 +563,5 @@ module Archive : sig val create_archive_server : ?forward_uri:Uri.t -> port:int -> logger:Logger.t -> unit -> t Deferred.t end + +val initialize_all : Time.Span.t -> unit diff --git a/src/lib/mina_metrics/no_metrics/mina_metrics.ml b/src/lib/mina_metrics/no_metrics/mina_metrics.ml index d87319d626f..5bdfd893ded 100644 --- a/src/lib/mina_metrics/no_metrics/mina_metrics.ml +++ b/src/lib/mina_metrics/no_metrics/mina_metrics.ml @@ -581,3 +581,5 @@ module Archive = struct fun ?forward_uri:_ ~port:_ ~logger:_ _ -> failwith "No metrics server available" end + +let initialize_all = Fn.ignore diff --git a/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml b/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml index 12033d14074..60b91c5b4fa 100644 --- a/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml +++ b/src/lib/mina_metrics/prometheus_metrics/metric_generators.ml @@ -13,9 +13,9 @@ end module type Bucketed_average_spec_intf = sig include Metric_spec_intf - val bucket_interval : Core.Time.Span.t -> Core.Time.Span.t + val bucket_interval : Time.Span.t -> Time.Span.t - val num_buckets : Core.Time.Span.t -> int + val num_buckets : Time.Span.t -> int val render_average : (float * int) list -> float end @@ -49,7 +49,7 @@ module type Moving_average_metric_intf = sig val v : Gauge.t - val initialize : Core.Time.Span.t -> unit + val initialize : Time.Span.t -> unit end module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) : @@ -96,7 +96,7 @@ module Moving_bucketed_average (Spec : Bucketed_average_spec_intf) : end module Moving_time_average (Spec : Time_average_spec_intf) : - Moving_average_metric_intf with type datum := Core.Time.Span.t = struct + Moving_average_metric_intf with type datum := Time.Span.t = struct include Moving_bucketed_average (struct include Spec @@ -117,5 +117,5 @@ module Moving_time_average (Spec : Time_average_spec_intf) : total_sum /. Float.of_int count_sum end) - let update span = update (Core.Time.Span.to_sec span) + let update span = update (Time.Span.to_sec span) end diff --git a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml index 61cd15c22da..bd33c5377ff 100644 --- a/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml +++ b/src/lib/mina_metrics/prometheus_metrics/mina_metrics.ml @@ -1755,3 +1755,10 @@ module Archive = struct ; gauge_metrics = Hashtbl.create (module String) } end + +let initialize_all block_window_duration = + Transition_frontier.TPS_30min.initialize block_window_duration ; + Block_latency.Gossip_slots.initialize block_window_duration ; + Block_latency.Gossip_time.initialize block_window_duration ; + Block_latency.Inclusion_time.initialize block_window_duration ; + Block_latency.Validation_acceptance_time.initialize block_window_duration From f68748f74e6f8b7d6dc3cf72682ff9e7ed185044 Mon Sep 17 00:00:00 2001 From: Mikhail Volkhov Date: Wed, 18 Sep 2024 16:31:50 +0100 Subject: [PATCH 11/14] Bump proof-systems --- src/lib/crypto/proof-systems | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/crypto/proof-systems b/src/lib/crypto/proof-systems index 6866340531e..4b68490fefa 160000 --- a/src/lib/crypto/proof-systems +++ b/src/lib/crypto/proof-systems @@ -1 +1 @@ -Subproject commit 6866340531e593516a9b88b03bc10185d0000560 +Subproject commit 4b68490fefa09973d8dbd3b6af1155b97a2addf0 From 0d5261aa26fd28d3b74ed299180b1b609842c317 Mon Sep 17 00:00:00 2001 From: Mikhail Volkhov Date: Wed, 18 Sep 2024 16:41:29 +0100 Subject: [PATCH 12/14] Update stubs & revendor for stubs --- .../crypto/kimchi_bindings/stubs/Cargo.lock | 375 ++++++++++++++++-- .../stubs/kimchi-stubs-vendors | 2 +- .../crypto/kimchi_bindings/wasm/Cargo.lock | 174 ++++---- 3 files changed, 441 insertions(+), 110 deletions(-) diff --git a/src/lib/crypto/kimchi_bindings/stubs/Cargo.lock b/src/lib/crypto/kimchi_bindings/stubs/Cargo.lock index e51acabc166..d286d15f0ae 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/Cargo.lock +++ b/src/lib/crypto/kimchi_bindings/stubs/Cargo.lock @@ -25,6 +25,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "approx" version = "0.3.2" @@ -45,7 +60,7 @@ dependencies = [ "ark-serialize", "ark-std", "derivative", - "hashbrown", + "hashbrown 0.13.2", "itertools", "num-traits", "rayon", @@ -106,7 +121,7 @@ dependencies = [ "ark-serialize", "ark-std", "derivative", - "hashbrown", + "hashbrown 0.13.2", "rayon", ] @@ -156,6 +171,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bcs" version = "0.1.5" @@ -184,6 +205,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.4.3" @@ -202,6 +229,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "windows-targets", +] + [[package]] name = "const-random" version = "0.1.15" @@ -233,6 +273,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.9" @@ -309,9 +355,9 @@ checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" [[package]] name = "darling" -version = "0.13.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -319,27 +365,37 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] name = "darling_macro" -version = "0.13.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.32", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", ] [[package]] @@ -376,6 +432,12 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "fnv" version = "1.0.7" @@ -412,6 +474,12 @@ dependencies = [ "rand", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.13.2" @@ -421,11 +489,17 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" @@ -442,12 +516,57 @@ dependencies = [ "serde", ] +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", + "serde", +] + [[package]] name = "internal-tracing" version = "0.1.0" @@ -471,6 +590,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kimchi" version = "0.1.0" @@ -520,6 +648,12 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "matrixmultiply" version = "0.3.7" @@ -611,15 +745,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" -version = "0.3.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] @@ -791,6 +931,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -947,18 +1093,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", @@ -978,24 +1124,32 @@ dependencies = [ [[package]] name = "serde_with" -version = "1.14.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.5.0", "serde", + "serde_derive", + "serde_json", "serde_with_macros", + "time", ] [[package]] name = "serde_with_macros" -version = "1.5.2" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] @@ -1038,27 +1192,27 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.24.1" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] @@ -1121,6 +1275,37 @@ dependencies = [ "syn 2.0.32", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -1175,6 +1360,134 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.32", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wires_15_stubs" version = "0.1.0" diff --git a/src/lib/crypto/kimchi_bindings/stubs/kimchi-stubs-vendors b/src/lib/crypto/kimchi_bindings/stubs/kimchi-stubs-vendors index 19588d41d1f..88128f32400 160000 --- a/src/lib/crypto/kimchi_bindings/stubs/kimchi-stubs-vendors +++ b/src/lib/crypto/kimchi_bindings/stubs/kimchi-stubs-vendors @@ -1 +1 @@ -Subproject commit 19588d41d1f100ff5073519ff7fa2d5bb17bdce4 +Subproject commit 88128f324002e83ae073f7a3fab55b60fef5c1cf diff --git a/src/lib/crypto/kimchi_bindings/wasm/Cargo.lock b/src/lib/crypto/kimchi_bindings/wasm/Cargo.lock index 2155e2b11a6..0af12831abe 100644 --- a/src/lib/crypto/kimchi_bindings/wasm/Cargo.lock +++ b/src/lib/crypto/kimchi_bindings/wasm/Cargo.lock @@ -177,6 +177,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bcs" version = "0.1.6" @@ -313,38 +319,14 @@ dependencies = [ "typenum", ] -[[package]] -name = "darling" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" -dependencies = [ - "darling_core 0.13.4", - "darling_macro 0.13.4", -] - [[package]] name = "darling" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ - "darling_core 0.20.3", - "darling_macro 0.20.3", -] - -[[package]] -name = "darling_core" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 1.0.109", + "darling_core", + "darling_macro", ] [[package]] @@ -361,24 +343,13 @@ dependencies = [ "syn 2.0.32", ] -[[package]] -name = "darling_macro" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" -dependencies = [ - "darling_core 0.13.4", - "quote", - "syn 1.0.109", -] - [[package]] name = "darling_macro" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ - "darling_core 0.20.3", + "darling_core", "quote", "syn 2.0.32", ] @@ -389,6 +360,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ + "powerfmt", "serde", ] @@ -426,6 +398,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "fnv" version = "1.0.7" @@ -479,11 +457,17 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" @@ -540,6 +524,17 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", + "serde", +] + [[package]] name = "internal-tracing" version = "0.1.0" @@ -596,7 +591,7 @@ dependencies = [ "rayon", "rmp-serde", "serde", - "serde_with 1.14.0", + "serde_with 3.9.0", "strum", "strum_macros", "thiserror", @@ -663,7 +658,7 @@ dependencies = [ "rand", "rayon", "serde", - "serde_with 1.14.0", + "serde_with 3.9.0", ] [[package]] @@ -711,15 +706,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" -version = "0.3.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] @@ -771,7 +772,7 @@ dependencies = [ "rayon", "rmp-serde", "serde", - "serde_with 1.14.0", + "serde_with 3.9.0", "sha2", "thiserror", ] @@ -797,7 +798,7 @@ dependencies = [ "ark-poly", "ark-serialize", "array-init", - "base64", + "base64 0.13.1", "console_error_panic_hook", "getrandom", "groupmap", @@ -847,10 +848,16 @@ dependencies = [ "rayon", "rmp-serde", "serde", - "serde_with 1.14.0", + "serde_with 3.9.0", "thiserror", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -994,9 +1001,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] @@ -1014,9 +1021,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", @@ -1036,49 +1043,57 @@ dependencies = [ [[package]] name = "serde_with" -version = "1.14.0" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap 1.9.3", "serde", - "serde_with_macros 1.5.2", + "serde_json", + "serde_with_macros 2.3.3", + "time", ] [[package]] name = "serde_with" -version = "2.3.3" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", - "indexmap", + "indexmap 1.9.3", + "indexmap 2.5.0", "serde", + "serde_derive", "serde_json", - "serde_with_macros 2.3.3", + "serde_with_macros 3.9.0", "time", ] [[package]] name = "serde_with_macros" -version = "1.5.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" dependencies = [ - "darling 0.13.4", + "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] name = "serde_with_macros" -version = "2.3.3" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ - "darling 0.20.3", + "darling", "proc-macro2", "quote", "syn 2.0.32", @@ -1130,21 +1145,21 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strum" -version = "0.24.1" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] @@ -1197,12 +1212,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.26" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -1210,16 +1227,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.12" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] From 247c774e498c7759c4f1420f6d68676d17e3df91 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Wed, 18 Sep 2024 18:15:28 +0100 Subject: [PATCH 13/14] Push Pickles_types down into Mina_wire_types --- .../pasta/basic/kimchi_pasta_basic.ml | 5 +- src/lib/mina_wire_types/dune | 3 - .../mina_base/mina_base_zkapp_state.ml | 2 +- src/lib/mina_wire_types/pickles/pickles.ml | 53 +++---- src/lib/mina_wire_types/pickles/pickles.mli | 47 ++++--- .../pickles/pickles_composition_types.ml | 3 +- .../pickles/pickles_composition_types.mli | 3 +- .../pickles/pickles_limb_vector.ml | 6 +- ...messages_for_next_proof_over_same_field.ml | 4 +- src/lib/mina_wire_types/pickles_base.ml | 2 +- src/lib/mina_wire_types/pickles_types.ml | 132 ++++++++++++++++++ src/lib/mina_wire_types/snark_params.ml | 2 +- src/lib/pickles_types/dune | 1 + src/lib/pickles_types/nat.ml | 4 +- src/lib/pickles_types/nat.mli | 4 +- src/lib/pickles_types/plonk_types.ml | 14 +- src/lib/pickles_types/plonk_types.mli | 16 ++- .../plonk_verification_key_evals.ml | 1 + .../plonk_verification_key_evals.mli | 1 + src/lib/pickles_types/shifted_value.ml | 3 +- src/lib/pickles_types/shifted_value.mli | 3 +- src/lib/pickles_types/vector.ml | 4 +- src/lib/pickles_types/vector.mli | 2 +- 23 files changed, 239 insertions(+), 76 deletions(-) create mode 100644 src/lib/mina_wire_types/pickles_types.ml diff --git a/src/lib/crypto/kimchi_backend/pasta/basic/kimchi_pasta_basic.ml b/src/lib/crypto/kimchi_backend/pasta/basic/kimchi_pasta_basic.ml index 245a7c0ad83..9c41f10bab5 100644 --- a/src/lib/crypto/kimchi_backend/pasta/basic/kimchi_pasta_basic.ml +++ b/src/lib/crypto/kimchi_backend/pasta/basic/kimchi_pasta_basic.ml @@ -3,12 +3,11 @@ open Kimchi_backend_common module Rounds : sig open Pickles_types - - module Wrap : Nat.Add.Intf_transparent + module Wrap = Nat.N15 module Wrap_vector : Vector.With_version(Wrap).S - module Step : Nat.Add.Intf_transparent + module Step = Nat.N16 module Step_vector : Vector.With_version(Step).S end = struct diff --git a/src/lib/mina_wire_types/dune b/src/lib/mina_wire_types/dune index f7b58021830..6922cffccd3 100644 --- a/src/lib/mina_wire_types/dune +++ b/src/lib/mina_wire_types/dune @@ -9,11 +9,8 @@ ; Keep these dependencies to an absolute minimum integers pasta_bindings - pickles_types kimchi_types kimchi_bindings - kimchi_pasta - kimchi_pasta.basic blake2 ) (preprocess (pps ppx_version)) diff --git a/src/lib/mina_wire_types/mina_base/mina_base_zkapp_state.ml b/src/lib/mina_wire_types/mina_base/mina_base_zkapp_state.ml index 872c3b1e701..4e82031ac0b 100644 --- a/src/lib/mina_wire_types/mina_base/mina_base_zkapp_state.ml +++ b/src/lib/mina_wire_types/mina_base/mina_base_zkapp_state.ml @@ -1,5 +1,5 @@ module V = struct module V1 = struct - type 'a t = 'a Pickles_types.Vector.Vector_8.Stable.V1.t + type 'a t = ('a, Pickles_types.Nat.eight) Pickles_types.Vector.t end end diff --git a/src/lib/mina_wire_types/pickles/pickles.ml b/src/lib/mina_wire_types/pickles/pickles.ml index 3708c754de9..59f98fd503e 100644 --- a/src/lib/mina_wire_types/pickles/pickles.ml +++ b/src/lib/mina_wire_types/pickles/pickles.ml @@ -12,21 +12,22 @@ module M = struct end module Wrap_wire_proof = struct - module Columns_vec = Pickles_types.Vector.Vector_15 - module Coefficients_vec = Pickles_types.Vector.Vector_15 - module Quotient_polynomial_vec = Pickles_types.Vector.Vector_7 - module Permuts_minus_1_vec = Pickles_types.Vector.Vector_6 + type 'a columns_vec = ('a, Pickles_types.Nat.fifteen) Pickles_types.Vector.t + + type 'a quotient_polynomial_vec = + ('a, Pickles_types.Nat.seven) Pickles_types.Vector.t + + type 'a permuts_minus_1_vec = + ('a, Pickles_types.Nat.six) Pickles_types.Vector.t module Commitments = struct module V1 = struct type t = - { w_comm : - (Pasta_bindings.Fp.t * Pasta_bindings.Fp.t) - Columns_vec.Stable.V1.t + { w_comm : (Pasta_bindings.Fp.t * Pasta_bindings.Fp.t) columns_vec ; z_comm : Pasta_bindings.Fp.t * Pasta_bindings.Fp.t ; t_comm : (Pasta_bindings.Fp.t * Pasta_bindings.Fp.t) - Quotient_polynomial_vec.Stable.V1.t + quotient_polynomial_vec } end end @@ -34,16 +35,11 @@ module M = struct module Evaluations = struct module V1 = struct type t = - { w : - (Pasta_bindings.Fq.t * Pasta_bindings.Fq.t) - Columns_vec.Stable.V1.t + { w : (Pasta_bindings.Fq.t * Pasta_bindings.Fq.t) columns_vec ; coefficients : - (Pasta_bindings.Fq.t * Pasta_bindings.Fq.t) - Columns_vec.Stable.V1.t + (Pasta_bindings.Fq.t * Pasta_bindings.Fq.t) columns_vec ; z : Pasta_bindings.Fq.t * Pasta_bindings.Fq.t - ; s : - (Pasta_bindings.Fq.t * Pasta_bindings.Fq.t) - Permuts_minus_1_vec.Stable.V1.t + ; s : (Pasta_bindings.Fq.t * Pasta_bindings.Fq.t) permuts_minus_1_vec ; generic_selector : Pasta_bindings.Fq.t * Pasta_bindings.Fq.t ; poseidon_selector : Pasta_bindings.Fq.t * Pasta_bindings.Fq.t ; complete_add_selector : Pasta_bindings.Fq.t * Pasta_bindings.Fq.t @@ -62,30 +58,31 @@ module M = struct ; bulletproof : ( Pasta_bindings.Fp.t * Pasta_bindings.Fp.t , Pasta_bindings.Fq.t ) - Pickles_types.Plonk_types.Openings.Bulletproof.Stable.V1.t + Pickles_types.Plonk_types.Openings.Bulletproof.V1.t } end end module Proof = struct type challenge_constant = - Pickles_limb_vector.Constant.Make(Pickles_types.Nat.N2).t + Pickles_types.Nat.two Pickles_limb_vector.Constant.t type tock_affine = Pasta_bindings.Fp.t * Pasta_bindings.Fp.t - type 'a step_bp_vec = 'a Kimchi_pasta.Basic.Rounds.Step_vector.Stable.V1.t + type 'a step_bp_vec = ('a, Pickles_types.Nat.sixteen) Pickles_types.Vector.t module Base = struct module Wrap = struct module V2 = struct type digest_constant = - Pickles_limb_vector.Constant.Make(Pickles_types.Nat.N4).t + Pickles_types.Nat.four Pickles_limb_vector.Constant.t type ('messages_for_next_wrap_proof, 'messages_for_next_step_proof) t = { statement : ( challenge_constant , challenge_constant Kimchi_types.scalar_challenge - , Snark_params.Tick.Field.t Pickles_types.Shifted_value.Type1.t + , Snark_params.Tick.Field.t + Pickles_types.Shifted_value.Type1.V1.t , bool , 'messages_for_next_wrap_proof , digest_constant @@ -98,7 +95,7 @@ module M = struct ; prev_evals : ( Snark_params.Tick.Field.t , Snark_params.Tick.Field.t array ) - Pickles_types.Plonk_types.All_evals.t + Pickles_types.Plonk_types.All_evals.V1.t (* A job half-done may be worse than not done at all. TODO: Migrate Plonk_types here, and actually include the *wire* type, not this in-memory version. @@ -128,7 +125,7 @@ module M = struct module Proofs_verified_2 = struct module V2 = struct - type nonrec t = (Pickles_types.Nat.N2.n, Pickles_types.Nat.N2.n) t + type nonrec t = (Pickles_types.Nat.two, Pickles_types.Nat.two) t end end end @@ -155,7 +152,9 @@ module M = struct Pickles_base.Side_loaded_verification_key.Poly.V2.t end - module Max_width = Pickles_types.Nat.N2 + module Max_width = struct + type n = Pickles_types.Nat.two + end end module Proof = struct @@ -174,14 +173,16 @@ module Types = struct module Proofs_verified_2 : sig module V2 : sig - type nonrec t = (Pickles_types.Nat.N2.n, Pickles_types.Nat.N2.n) t + type nonrec t = (Pickles_types.Nat.two, Pickles_types.Nat.two) t end end end module Side_loaded : sig module Verification_key : sig - module Max_width : module type of Pickles_types.Nat.N2 + module Max_width : sig + type n = Pickles_types.Nat.two + end module V2 : sig type t diff --git a/src/lib/mina_wire_types/pickles/pickles.mli b/src/lib/mina_wire_types/pickles/pickles.mli index 418f7242cb2..312a5befff4 100644 --- a/src/lib/mina_wire_types/pickles/pickles.mli +++ b/src/lib/mina_wire_types/pickles/pickles.mli @@ -7,14 +7,16 @@ module Types : sig module Proofs_verified_2 : sig module V2 : sig - type nonrec t = (Pickles_types.Nat.N2.n, Pickles_types.Nat.N2.n) t + type nonrec t = (Pickles_types.Nat.two, Pickles_types.Nat.two) t end end end module Side_loaded : sig module Verification_key : sig - module Max_width : module type of Pickles_types.Nat.N2 + module Max_width : sig + type n = Pickles_types.Nat.two + end module V2 : sig type t @@ -64,17 +66,21 @@ module Concrete_ : sig type fq_point := Pasta_bindings.Fq.t * Pasta_bindings.Fq.t - module Columns_vec = Pickles_types.Vector.Vector_15 - module Coefficients_vec = Pickles_types.Vector.Vector_15 - module Quotient_polynomial_vec = Pickles_types.Vector.Vector_7 - module Permuts_minus_1_vec = Pickles_types.Vector.Vector_6 + type 'a columns_vec := + ('a, Pickles_types.Nat.fifteen) Pickles_types.Vector.t + + type 'a quotient_polynomial_vec := + ('a, Pickles_types.Nat.seven) Pickles_types.Vector.t + + type 'a permuts_minus_1_vec := + ('a, Pickles_types.Nat.six) Pickles_types.Vector.t module Commitments : sig module V1 : sig type t = - { w_comm : fp_point Columns_vec.Stable.V1.t + { w_comm : fp_point columns_vec ; z_comm : fp_point - ; t_comm : fp_point Quotient_polynomial_vec.Stable.V1.t + ; t_comm : fp_point quotient_polynomial_vec } end end @@ -82,10 +88,10 @@ module Concrete_ : sig module Evaluations : sig module V1 : sig type t = - { w : fq_point Columns_vec.Stable.V1.t - ; coefficients : fq_point Columns_vec.Stable.V1.t + { w : fq_point columns_vec + ; coefficients : fq_point columns_vec ; z : fq_point - ; s : fq_point Permuts_minus_1_vec.Stable.V1.t + ; s : fq_point permuts_minus_1_vec ; generic_selector : fq_point ; poseidon_selector : fq_point ; complete_add_selector : fq_point @@ -104,7 +110,7 @@ module Concrete_ : sig ; bulletproof : ( fp_point , Pasta_bindings.Fq.t ) - Pickles_types.Plonk_types.Openings.Bulletproof.Stable.V1.t + Pickles_types.Plonk_types.Openings.Bulletproof.V1.t } end end @@ -112,23 +118,24 @@ module Concrete_ : sig module Proof : sig (* We define some type aliases directly *) type challenge_constant = - Pickles_limb_vector.Constant.Make(Pickles_types.Nat.N2).t + Pickles_types.Nat.two Pickles_limb_vector.Constant.t type tock_affine = Pasta_bindings.Fp.t * Pasta_bindings.Fp.t - type 'a step_bp_vec = 'a Kimchi_pasta.Basic.Rounds.Step_vector.Stable.V1.t + type 'a step_bp_vec = ('a, Pickles_types.Nat.sixteen) Pickles_types.Vector.t module Base : sig module Wrap : sig module V2 : sig type digest_constant = - Pickles_limb_vector.Constant.Make(Pickles_types.Nat.N4).t + Pickles_types.Nat.four Pickles_limb_vector.Constant.t type ('messages_for_next_wrap_proof, 'messages_for_next_step_proof) t = { statement : ( challenge_constant , challenge_constant Kimchi_types.scalar_challenge - , Snark_params.Tick.Field.t Pickles_types.Shifted_value.Type1.t + , Snark_params.Tick.Field.t + Pickles_types.Shifted_value.Type1.V1.t , bool , 'messages_for_next_wrap_proof , digest_constant @@ -141,7 +148,7 @@ module Concrete_ : sig ; prev_evals : ( Snark_params.Tick.Field.t , Snark_params.Tick.Field.t array ) - Pickles_types.Plonk_types.All_evals.t + Pickles_types.Plonk_types.All_evals.V1.t ; proof : Wrap_wire_proof.V1.t } end @@ -167,7 +174,7 @@ module Concrete_ : sig module Proofs_verified_2 : sig module V2 : sig - type nonrec t = (Pickles_types.Nat.N2.n, Pickles_types.Nat.N2.n) t + type nonrec t = (Pickles_types.Nat.two, Pickles_types.Nat.two) t end end end @@ -194,7 +201,9 @@ module Concrete_ : sig Pickles_base.Side_loaded_verification_key.Poly.V2.t end - module Max_width = Pickles_types.Nat.N2 + module Max_width : sig + type n = Pickles_types.Nat.two + end end module Proof : sig diff --git a/src/lib/mina_wire_types/pickles/pickles_composition_types.ml b/src/lib/mina_wire_types/pickles/pickles_composition_types.ml index b6ae6e4ced1..0e7239fdb41 100644 --- a/src/lib/mina_wire_types/pickles/pickles_composition_types.ml +++ b/src/lib/mina_wire_types/pickles/pickles_composition_types.ml @@ -60,8 +60,7 @@ module Wrap = struct ; gamma : 'challenge ; zeta : 'scalar_challenge ; joint_combiner : 'scalar_challenge option - ; feature_flags : - 'bool Pickles_types.Plonk_types.Features.Stable.V1.t + ; feature_flags : 'bool Pickles_types.Plonk_types.Features.V1.t } end end diff --git a/src/lib/mina_wire_types/pickles/pickles_composition_types.mli b/src/lib/mina_wire_types/pickles/pickles_composition_types.mli index 0c4f79627c8..c153345793c 100644 --- a/src/lib/mina_wire_types/pickles/pickles_composition_types.mli +++ b/src/lib/mina_wire_types/pickles/pickles_composition_types.mli @@ -48,8 +48,7 @@ module Wrap : sig ; gamma : 'challenge ; zeta : 'scalar_challenge ; joint_combiner : 'scalar_challenge option - ; feature_flags : - 'bool Pickles_types.Plonk_types.Features.Stable.V1.t + ; feature_flags : 'bool Pickles_types.Plonk_types.Features.V1.t } end end diff --git a/src/lib/mina_wire_types/pickles/pickles_limb_vector.ml b/src/lib/mina_wire_types/pickles/pickles_limb_vector.ml index be1297d0848..ebf6c6003c0 100644 --- a/src/lib/mina_wire_types/pickles/pickles_limb_vector.ml +++ b/src/lib/mina_wire_types/pickles/pickles_limb_vector.ml @@ -3,9 +3,5 @@ module Constant = struct type t = Int64.t end - module Make (N : Pickles_types.Nat.Intf) = struct - module A = Pickles_types.Vector.With_length (N) - - type t = Hex64.t A.t - end + type 'n t = (Hex64.t, 'n) Pickles_types.Vector.t end diff --git a/src/lib/mina_wire_types/pickles/pickles_reduced_messages_for_next_proof_over_same_field.ml b/src/lib/mina_wire_types/pickles/pickles_reduced_messages_for_next_proof_over_same_field.ml index c101f97f8e5..c1d39f45c48 100644 --- a/src/lib/mina_wire_types/pickles/pickles_reduced_messages_for_next_proof_over_same_field.ml +++ b/src/lib/mina_wire_types/pickles/pickles_reduced_messages_for_next_proof_over_same_field.ml @@ -11,9 +11,9 @@ end module Wrap = struct module Challenges_vector = struct type challenge_constant = - Pickles_limb_vector.Constant.Make(Pickles_types.Nat.N2).t + Pickles_types.Nat.two Pickles_limb_vector.Constant.t - type 'a wrap_bp_vec = 'a Kimchi_pasta.Basic.Rounds.Wrap_vector.Stable.V1.t + type 'a wrap_bp_vec = ('a, Pickles_types.Nat.fifteen) Pickles_types.Vector.t type t = challenge_constant Kimchi_types.scalar_challenge diff --git a/src/lib/mina_wire_types/pickles_base.ml b/src/lib/mina_wire_types/pickles_base.ml index ab162304140..e2f8f22150e 100644 --- a/src/lib/mina_wire_types/pickles_base.ml +++ b/src/lib/mina_wire_types/pickles_base.ml @@ -10,7 +10,7 @@ module Side_loaded_verification_key = struct type ('g, 'proofs_verified, 'vk) t = { max_proofs_verified : 'proofs_verified ; actual_wrap_domain_size : 'proofs_verified - ; wrap_index : 'g Pickles_types.Plonk_verification_key_evals.Stable.V2.t + ; wrap_index : 'g Pickles_types.Plonk_verification_key_evals.V2.t ; wrap_vk : 'vk option } end diff --git a/src/lib/mina_wire_types/pickles_types.ml b/src/lib/mina_wire_types/pickles_types.ml new file mode 100644 index 00000000000..625fccee9e4 --- /dev/null +++ b/src/lib/mina_wire_types/pickles_types.ml @@ -0,0 +1,132 @@ +module Nat = struct + type z = Z of z + + type 'a s = Z | S of 'a + + type _ t = Z : z t | S : 'n t -> 'n s t + + type two = z s s + + type four = z s s s s + + type five = z s s s s s + + type six = z s s s s s s + + type seven = z s s s s s s s + + type eight = z s s s s s s s s + + type fifteen = z s s s s s s s s s s s s s s s + + type sixteen = z s s s s s s s s s s s s s s s s +end + +module Vector = struct + type ('a, _) t = + | [] : ('a, Nat.z) t + | ( :: ) : 'a * ('a, 'n) t -> ('a, 'n Nat.s) t +end + +module Shifted_value = struct + module Type1 = struct + module V1 = struct + type 'f t = Shifted_value of 'f + end + end +end + +module Plonk_types = struct + module Features = struct + module V1 = struct + type 'bool t = + { range_check0 : 'bool + ; range_check1 : 'bool + ; foreign_field_add : 'bool + ; foreign_field_mul : 'bool + ; xor : 'bool + ; rot : 'bool + ; lookup : 'bool + ; runtime_tables : 'bool + } + end + end + + module Openings = struct + module Bulletproof = struct + module V1 = struct + type ('g, 'fq) t = + { lr : ('g * 'g) array + ; z_1 : 'fq + ; z_2 : 'fq + ; delta : 'g + ; challenge_polynomial_commitment : 'g + } + end + end + end + + module Evals = struct + module V2 = struct + type 'a t = + { w : ('a, Nat.fifteen) Vector.t + ; coefficients : ('a, Nat.fifteen) Vector.t + ; z : 'a + ; s : ('a, Nat.six) Vector.t + ; generic_selector : 'a + ; poseidon_selector : 'a + ; complete_add_selector : 'a + ; mul_selector : 'a + ; emul_selector : 'a + ; endomul_scalar_selector : 'a + ; range_check0_selector : 'a option + ; range_check1_selector : 'a option + ; foreign_field_add_selector : 'a option + ; foreign_field_mul_selector : 'a option + ; xor_selector : 'a option + ; rot_selector : 'a option + ; lookup_aggregation : 'a option + ; lookup_table : 'a option + ; lookup_sorted : ('a option, Nat.five) Vector.t + ; runtime_lookup_table : 'a option + ; runtime_lookup_table_selector : 'a option + ; xor_lookup_selector : 'a option + ; lookup_gate_lookup_selector : 'a option + ; range_check_lookup_selector : 'a option + ; foreign_field_mul_lookup_selector : 'a option + } + end + end + + module All_evals = struct + module With_public_input = struct + module V1 = struct + type ('f, 'f_multi) t = + { public_input : 'f; evals : 'f_multi Evals.V2.t } + end + end + + module V1 = struct + type ('f, 'f_multi) t = + { evals : + ('f_multi * 'f_multi, 'f_multi * 'f_multi) With_public_input.V1.t + ; ft_eval1 : 'f + } + end + end +end + +module Plonk_verification_key_evals = struct + module V2 = struct + type 'comm t = + { sigma_comm : ('comm, Nat.seven) Vector.t + ; coefficients_comm : ('comm, Nat.fifteen) Vector.t + ; generic_comm : 'comm + ; psm_comm : 'comm + ; complete_add_comm : 'comm + ; mul_comm : 'comm + ; emul_comm : 'comm + ; endomul_scalar_comm : 'comm + } + end +end diff --git a/src/lib/mina_wire_types/snark_params.ml b/src/lib/mina_wire_types/snark_params.ml index c4248a05fa0..c1164b677a2 100644 --- a/src/lib/mina_wire_types/snark_params.ml +++ b/src/lib/mina_wire_types/snark_params.ml @@ -1,6 +1,6 @@ module Tick = struct module Field = struct - type t = Kimchi_pasta_basic.Fp.t + type t = Pasta_bindings.Fp.t end module Inner_curve = struct diff --git a/src/lib/pickles_types/dune b/src/lib/pickles_types/dune index 2dbfb9890ef..ca4ded5524d 100644 --- a/src/lib/pickles_types/dune +++ b/src/lib/pickles_types/dune @@ -37,4 +37,5 @@ tuple_lib ppx_version.runtime bounded_types + mina_wire_types )) diff --git a/src/lib/pickles_types/nat.ml b/src/lib/pickles_types/nat.ml index 4903cbed816..57f5d4074f1 100644 --- a/src/lib/pickles_types/nat.ml +++ b/src/lib/pickles_types/nat.ml @@ -1,6 +1,6 @@ -type z = Z of z +type z = Mina_wire_types.Pickles_types.Nat.z = Z of z -type 'a s = Z | S of 'a +type 'a s = 'a Mina_wire_types.Pickles_types.Nat.s = Z | S of 'a type _ t = Z : z t | S : 'n t -> 'n s t diff --git a/src/lib/pickles_types/nat.mli b/src/lib/pickles_types/nat.mli index 358d5f3f206..87ee4447247 100644 --- a/src/lib/pickles_types/nat.mli +++ b/src/lib/pickles_types/nat.mli @@ -3,9 +3,9 @@ (** {1 Type definitions} *) (** [z] is uninhabited *) -type z = Z of z +type z = Mina_wire_types.Pickles_types.Nat.z = Z of z -type 'a s = Z | S of 'a +type 'a s = 'a Mina_wire_types.Pickles_types.Nat.s = Z | S of 'a type _ t = Z : z t | S : 'n t -> 'n s t diff --git a/src/lib/pickles_types/plonk_types.ml b/src/lib/pickles_types/plonk_types.ml index 50d4173c8db..8e13a0b4d99 100644 --- a/src/lib/pickles_types/plonk_types.ml +++ b/src/lib/pickles_types/plonk_types.ml @@ -206,6 +206,7 @@ module Features = struct module Stable = struct module V1 = struct type 'bool t = + 'bool Mina_wire_types.Pickles_types.Plonk_types.Features.V1.t = { range_check0 : 'bool ; range_check1 : 'bool ; foreign_field_add : 'bool @@ -475,7 +476,7 @@ module Evals = struct [%%versioned module Stable = struct module V2 = struct - type 'a t = + type 'a t = 'a Mina_wire_types.Pickles_types.Plonk_types.Evals.V2.t = { w : 'a Columns_vec.Stable.V1.t ; coefficients : 'a Columns_vec.Stable.V1.t ; z : 'a @@ -1231,6 +1232,12 @@ module All_evals = struct module Stable = struct module V1 = struct type ('f, 'f_multi) t = + ( 'f + , 'f_multi ) + Mina_wire_types.Pickles_types.Plonk_types.All_evals + .With_public_input + .V1 + .t = { public_input : 'f; evals : 'f_multi Evals.Stable.V2.t } [@@deriving sexp, compare, yojson, hash, equal, hlist] end @@ -1276,6 +1283,7 @@ module All_evals = struct end] type ('f, 'f_multi) t = + ('f, 'f_multi) Mina_wire_types.Pickles_types.Plonk_types.All_evals.V1.t = { evals : ('f_multi * 'f_multi, 'f_multi * 'f_multi) With_public_input.t ; ft_eval1 : 'f } @@ -1327,6 +1335,10 @@ module Openings = struct module Stable = struct module V1 = struct type ('g, 'fq) t = + ( 'g + , 'fq ) + Mina_wire_types.Pickles_types.Plonk_types.Openings.Bulletproof.V1 + .t = { lr : ('g * 'g) Bounded_types.ArrayN16.Stable.V1.t ; z_1 : 'fq ; z_2 : 'fq diff --git a/src/lib/pickles_types/plonk_types.mli b/src/lib/pickles_types/plonk_types.mli index 09e0ee03f36..473ce458eed 100644 --- a/src/lib/pickles_types/plonk_types.mli +++ b/src/lib/pickles_types/plonk_types.mli @@ -42,6 +42,7 @@ module Features : sig module Stable : sig module V1 : sig type 'bool t = + 'bool Mina_wire_types.Pickles_types.Plonk_types.Features.V1.t = { range_check0 : 'bool ; range_check1 : 'bool ; foreign_field_add : 'bool @@ -266,7 +267,7 @@ module Evals : sig -> unit end - type 'a t = + type 'a t = 'a Mina_wire_types.Pickles_types.Plonk_types.Evals.V2.t = { w : 'a Columns_vec.t ; coefficients : 'a Columns_vec.t ; z : 'a @@ -319,6 +320,10 @@ module Openings : sig module Stable : sig module V1 : sig type ('g, 'fq) t = + ( 'g + , 'fq ) + Mina_wire_types.Pickles_types.Plonk_types.Openings.Bulletproof.V1 + .t = { lr : ('g * 'g) array ; z_1 : 'fq ; z_2 : 'fq @@ -380,7 +385,13 @@ end module All_evals : sig module With_public_input : sig - type ('f, 'f_multi) t = { public_input : 'f; evals : 'f_multi Evals.t } + type ('f, 'f_multi) t = + ( 'f + , 'f_multi ) + Mina_wire_types.Pickles_types.Plonk_types.All_evals.With_public_input + .V1 + .t = + { public_input : 'f; evals : 'f_multi Evals.t } module In_circuit : sig type ('f, 'f_multi, 'bool) t = @@ -417,6 +428,7 @@ module All_evals : sig end type ('f, 'f_multi) t = + ('f, 'f_multi) Mina_wire_types.Pickles_types.Plonk_types.All_evals.V1.t = { evals : ('f_multi * 'f_multi, 'f_multi * 'f_multi) With_public_input.t ; ft_eval1 : 'f } diff --git a/src/lib/pickles_types/plonk_verification_key_evals.ml b/src/lib/pickles_types/plonk_verification_key_evals.ml index 87d5ef5a6e9..93e1aca8149 100644 --- a/src/lib/pickles_types/plonk_verification_key_evals.ml +++ b/src/lib/pickles_types/plonk_verification_key_evals.ml @@ -5,6 +5,7 @@ module H_list = Snarky_backendless.H_list module Stable = struct module V2 = struct type 'comm t = + 'comm Mina_wire_types.Pickles_types.Plonk_verification_key_evals.V2.t = { sigma_comm : 'comm Plonk_types.Permuts_vec.Stable.V1.t ; coefficients_comm : 'comm Plonk_types.Columns_vec.Stable.V1.t ; generic_comm : 'comm diff --git a/src/lib/pickles_types/plonk_verification_key_evals.mli b/src/lib/pickles_types/plonk_verification_key_evals.mli index 553ff080b98..be0f4240956 100644 --- a/src/lib/pickles_types/plonk_verification_key_evals.mli +++ b/src/lib/pickles_types/plonk_verification_key_evals.mli @@ -3,6 +3,7 @@ module Stable : sig module V2 : sig type 'comm t = + 'comm Mina_wire_types.Pickles_types.Plonk_verification_key_evals.V2.t = { sigma_comm : 'comm Plonk_types.Permuts_vec.Stable.V1.t ; coefficients_comm : 'comm Plonk_types.Columns_vec.Stable.V1.t ; generic_comm : 'comm diff --git a/src/lib/pickles_types/shifted_value.ml b/src/lib/pickles_types/shifted_value.ml index d48cb0e6cf6..6a467bdb111 100644 --- a/src/lib/pickles_types/shifted_value.ml +++ b/src/lib/pickles_types/shifted_value.ml @@ -95,7 +95,8 @@ module Type1 = struct [%%versioned module Stable = struct module V1 = struct - type 'f t = Shifted_value of 'f + type 'f t = 'f Mina_wire_types.Pickles_types.Shifted_value.Type1.V1.t = + | Shifted_value of 'f [@@deriving sexp, compare, equal, yojson, hash] end end] diff --git a/src/lib/pickles_types/shifted_value.mli b/src/lib/pickles_types/shifted_value.mli index 32d9d3ea675..5c02e5252c3 100644 --- a/src/lib/pickles_types/shifted_value.mli +++ b/src/lib/pickles_types/shifted_value.mli @@ -81,7 +81,8 @@ module Type1 : sig [%%versioned: module Stable : sig module V1 : sig - type 'f t = Shifted_value of 'f + type 'f t = 'f Mina_wire_types.Pickles_types.Shifted_value.Type1.V1.t = + | Shifted_value of 'f [@@deriving sexp, compare, equal, yojson, hash] end end] diff --git a/src/lib/pickles_types/vector.ml b/src/lib/pickles_types/vector.ml index 2ca4650738c..786f6de2c65 100644 --- a/src/lib/pickles_types/vector.ml +++ b/src/lib/pickles_types/vector.ml @@ -7,7 +7,9 @@ type z = Nat.z type 'a s = 'a Nat.s module T = struct - type ('a, _) t = [] : ('a, z) t | ( :: ) : 'a * ('a, 'n) t -> ('a, 'n s) t + type ('a, 'n) t = ('a, 'n) Mina_wire_types.Pickles_types.Vector.t = + | [] : ('a, z) t + | ( :: ) : 'a * ('a, 'n) t -> ('a, 'n s) t end include T diff --git a/src/lib/pickles_types/vector.mli b/src/lib/pickles_types/vector.mli index 5911761c4ab..0bc222f851b 100644 --- a/src/lib/pickles_types/vector.mli +++ b/src/lib/pickles_types/vector.mli @@ -8,7 +8,7 @@ (** Encode a vector at the type level with its size *) module T : sig - type ('a, _) t = + type ('a, 'b) t = ('a, 'b) Mina_wire_types.Pickles_types.Vector.t = | [] : ('a, Nat.z) t | ( :: ) : 'a * ('a, 'n) t -> ('a, 'n Nat.s) t end From 2f07477027a7e08e3452d1179b9bcfa0dfa9a27b Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 19 Sep 2024 09:47:58 +0200 Subject: [PATCH 14/14] pass BRANCH_NAME to export-git-env-vars script properly --- scripts/debian/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index e750ca1a03b..1df887ce9d9 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -9,7 +9,7 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" # In case of running this script on detached head, script has difficulties in finding out # what is the current branch. if [[ -n "$BRANCH_NAME" ]]; then - source ${SCRIPTPATH}/../export-git-env-vars.sh -b $BRANCH_NAME + BRANCH_NAME="$BRANCH_NAME" source ${SCRIPTPATH}/../export-git-env-vars.sh else source ${SCRIPTPATH}/../export-git-env-vars.sh fi