Merge branch 'main' into main

.github/workflows/ci-core-reusable.yml

@@ -259,8 +259,9 @@ jobs:
           ci_run zk external-node &>ext-node.log &
           ci_run sleep 30
 
+      # TODO(PLA-653): Restore bridge tests for EN.
       - name: Integration tests
-        run: ci_run zk test i server --testPathIgnorePatterns 'contract-verification'
+        run: ci_run zk test i server --testPathIgnorePatterns 'contract-verification|custom-erc20-bridge'
 
       - name: Run Cross EN Checker
         run: ci_run zk run cross-en-checker

core/bin/block_reverter/src/main.rs

@@ -2,8 +2,8 @@ use anyhow::Context as _;
 use clap::{Parser, Subcommand};
 use tokio::io::{self, AsyncReadExt};
 
-use zksync_config::{ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig};
-use zksync_dal::{connection::DbVariant, ConnectionPool};
+use zksync_config::{ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, PostgresConfig};
+use zksync_dal::ConnectionPool;
 use zksync_env_config::FromEnv;
 use zksync_types::{L1BatchNumber, U256};
 
@@ -92,12 +92,16 @@ async fn main() -> anyhow::Result<()> {
     let default_priority_fee_per_gas =
         U256::from(eth_sender.gas_adjuster.default_priority_fee_per_gas);
     let contracts = ContractsConfig::from_env().context("ContractsConfig::from_env()")?;
+    let postgres_config = PostgresConfig::from_env().context("PostgresConfig::from_env()")?;
     let config = BlockReverterEthConfig::new(eth_sender, contracts, eth_client.web3_url.clone());
 
-    let connection_pool = ConnectionPool::builder(DbVariant::Master)
-        .build()
-        .await
-        .context("failed to build a connection pool")?;
+    let connection_pool = ConnectionPool::builder(
+        postgres_config.master_url()?,
+        postgres_config.max_connections()?,
+    )
+    .build()
+    .await
+    .context("failed to build a connection pool")?;
     let mut block_reverter = BlockReverter::new(
         db_config.state_keeper_db_path,
         db_config.merkle_tree.path,

core/bin/contract-verifier/src/main.rs

@@ -2,7 +2,7 @@ use std::cell::RefCell;
 
 use anyhow::Context as _;
 use prometheus_exporter::PrometheusExporterConfig;
-use zksync_config::{configs::PrometheusConfig, ApiConfig, ContractVerifierConfig};
+use zksync_config::{configs::PrometheusConfig, ApiConfig, ContractVerifierConfig, PostgresConfig};
 use zksync_dal::ConnectionPool;
 use zksync_env_config::FromEnv;
 use zksync_queued_job_processor::JobProcessor;
@@ -113,7 +113,6 @@ async fn update_compiler_versions(connection_pool: &ConnectionPool) {
 }
 
 use structopt::StructOpt;
-use zksync_dal::connection::DbVariant;
 
 #[derive(StructOpt)]
 #[structopt(name = "zkSync contract code verifier", author = "Matter Labs")]
@@ -132,10 +131,15 @@ async fn main() -> anyhow::Result<()> {
         listener_port: verifier_config.prometheus_port,
         ..ApiConfig::from_env().context("ApiConfig")?.prometheus
     };
-    let pool = ConnectionPool::singleton(DbVariant::Master)
-        .build()
-        .await
-        .unwrap();
+    let postgres_config = PostgresConfig::from_env().context("PostgresConfig")?;
+    let pool = ConnectionPool::singleton(
+        postgres_config
+            .master_url()
+            .context("Master DB URL is absent")?,
+    )
+    .build()
+    .await
+    .unwrap();
 
     #[allow(deprecated)] // TODO (QIT-21): Use centralized configuration approach.
     let log_format = vlog::log_format_from_env();

core/bin/external_node/src/config/mod.rs

@@ -375,11 +375,35 @@ impl RequiredENConfig {
     }
 }
 
+/// Configuration for Postgres database.
+/// While also mandatory, it historically used different naming scheme for corresponding
+/// environment variables.
+/// Thus it is kept separately for backward compatibility and ease of deserialization.
+#[derive(Debug, Clone, Deserialize, PartialEq)]
+pub struct PostgresConfig {
+    pub database_url: String,
+    pub max_connections: u32,
+}
+
+impl PostgresConfig {
+    pub fn from_env() -> anyhow::Result<Self> {
+        Ok(Self {
+            database_url: env::var("DATABASE_URL")
+                .context("DATABASE_URL env variable is not set")?,
+            max_connections: env::var("DATABASE_POOL_SIZE")
+                .context("DATABASE_POOL_SIZE env variable is not set")?
+                .parse()
+                .context("Unable to parse DATABASE_POOL_SIZE env variable")?,
+        })
+    }
+}
+
 /// External Node Config contains all the configuration required for the EN operation.
 /// It is split into three parts: required, optional and remote for easier navigation.
 #[derive(Debug, Deserialize, Clone, PartialEq)]
 pub struct ExternalNodeConfig {
     pub required: RequiredENConfig,
+    pub postgres: PostgresConfig,
     pub optional: OptionalENConfig,
     pub remote: RemoteENConfig,
 }
@@ -440,8 +464,11 @@ impl ExternalNodeConfig {
             );
         }
 
+        let postgres = PostgresConfig::from_env()?;
+
         Ok(Self {
             remote,
+            postgres,
             required,
             optional,
         })

core/bin/external_node/src/main.rs

@@ -29,7 +29,7 @@ use zksync_core::{
         MainNodeClient, SyncState,
     },
 };
-use zksync_dal::{connection::DbVariant, healthcheck::ConnectionPoolHealthCheck, ConnectionPool};
+use zksync_dal::{healthcheck::ConnectionPoolHealthCheck, ConnectionPool};
 use zksync_health_check::CheckHealth;
 use zksync_state::PostgresStorageCaches;
 use zksync_storage::RocksDB;
@@ -122,7 +122,7 @@ async fn init_tasks(
 
     let main_node_client = <dyn MainNodeClient>::json_rpc(&main_node_url)
         .context("Failed creating JSON-RPC client for main node")?;
-    let singleton_pool_builder = ConnectionPool::singleton(DbVariant::Master);
+    let singleton_pool_builder = ConnectionPool::singleton(&config.postgres.database_url);
     let fetcher_cursor = {
         let pool = singleton_pool_builder
             .build()
@@ -183,7 +183,8 @@ async fn init_tasks(
         .await
         .context("failed to build a tree_pool")?;
     // todo: PLA-335
-    let prover_tree_pool = ConnectionPool::singleton(DbVariant::Prover)
+    // Note: This pool isn't actually used by the metadata calculator, but it has to be provided anyway.
+    let prover_tree_pool = ConnectionPool::singleton(&config.postgres.database_url)
         .build()
         .await
         .context("failed to build a prover_tree_pool")?;
@@ -348,10 +349,13 @@ async fn main() -> anyhow::Result<()> {
         .main_node_url()
         .context("Main node URL is incorrect")?;
 
-    let connection_pool = ConnectionPool::builder(DbVariant::Master)
-        .build()
-        .await
-        .context("failed to build a connection_pool")?;
+    let connection_pool = ConnectionPool::builder(
+        &config.postgres.database_url,
+        config.postgres.max_connections,
+    )
+    .build()
+    .await
+    .context("failed to build a connection_pool")?;
 
     if opt.revert_pending_l1_batch {
         tracing::info!("Rolling pending L1 batch back..");

core/bin/storage_logs_dedup_migration/Cargo.toml

@@ -16,4 +16,6 @@ path = "src/consistency.rs"
 tokio = { version = "1" }
 zksync_dal = { path = "../../lib/dal" }
 zksync_types = { path = "../../lib/types" }
+zksync_config = { path = "../../lib/config" }
+zksync_env_config = { path = "../../lib/env_config" }
 clap = { version = "4.2.4", features = ["derive"] }

core/bin/storage_logs_dedup_migration/src/consistency.rs

@@ -1,7 +1,8 @@
 use clap::Parser;
 
-use zksync_dal::connection::DbVariant;
+use zksync_config::PostgresConfig;
 use zksync_dal::ConnectionPool;
+use zksync_env_config::FromEnv;
 use zksync_types::MiniblockNumber;
 
 const MIGRATED_TABLE: &str = "storage_logs";
@@ -23,8 +24,9 @@ struct Cli {
 
 #[tokio::main]
 async fn main() {
+    let config = PostgresConfig::from_env().unwrap();
     let opt = Cli::parse();
-    let pool = ConnectionPool::singleton(DbVariant::Replica)
+    let pool = ConnectionPool::singleton(config.replica_url().unwrap())
         .build()
         .await
         .unwrap();

core/bin/storage_logs_dedup_migration/src/main.rs

@@ -2,8 +2,9 @@ use std::collections::hash_map::{Entry, HashMap};
 
 use clap::Parser;
 
-use zksync_dal::connection::DbVariant;
+use zksync_config::PostgresConfig;
 use zksync_dal::ConnectionPool;
+use zksync_env_config::FromEnv;
 use zksync_types::{MiniblockNumber, H256};
 
 /// When the threshold is reached then the migration is blocked on vacuuming.
@@ -45,8 +46,9 @@ impl StateCache {
 
 #[tokio::main]
 async fn main() {
+    let config = PostgresConfig::from_env().unwrap();
     let opt = Cli::parse();
-    let pool = ConnectionPool::singleton(DbVariant::Master)
+    let pool = ConnectionPool::singleton(config.master_url().unwrap())
         .build()
         .await
         .unwrap();

core/bin/verified_sources_fetcher/Cargo.toml

@@ -13,6 +13,8 @@ publish = false # We don't want to publish our binaries.
 [dependencies]
 zksync_dal = { path = "../../lib/dal" }
 zksync_types = { path = "../../lib/types" }
+zksync_config = { path = "../../lib/config" }
+zksync_env_config = { path = "../../lib/env_config" }
 
 anyhow = "1.0"
 tokio = { version = "1", features = ["full"] }

core/bin/verified_sources_fetcher/src/main.rs

@@ -1,10 +1,13 @@
 use std::io::Write;
-use zksync_dal::{connection::DbVariant, ConnectionPool};
+use zksync_config::PostgresConfig;
+use zksync_dal::ConnectionPool;
+use zksync_env_config::FromEnv;
 use zksync_types::contract_verification_api::SourceCodeData;
 
 #[tokio::main]
 async fn main() {
-    let pool = ConnectionPool::singleton(DbVariant::Replica)
+    let config = PostgresConfig::from_env().unwrap();
+    let pool = ConnectionPool::singleton(config.replica_url().unwrap())
         .build()
         .await
         .unwrap();

core/bin/zksync_server/src/main.rs

@@ -15,7 +15,7 @@ use zksync_config::{
         ProofDataHandlerConfig, ProverGroupConfig, WitnessGeneratorConfig,
     },
     ApiConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, ETHWatchConfig,
-    FetcherConfig, GasAdjusterConfig, ObjectStoreConfig, ProverConfigs,
+    FetcherConfig, GasAdjusterConfig, ObjectStoreConfig, PostgresConfig, ProverConfigs,
 };
 
 use zksync_core::temp_config_store::TempConfigStore;
@@ -91,37 +91,12 @@ async fn main() -> anyhow::Result<()> {
         tracing::info!("No sentry URL was provided");
     }
 
-    if opt.genesis || is_genesis_needed().await {
-        let network = NetworkConfig::from_env().context("NetworkConfig")?;
-        let eth_sender = ETHSenderConfig::from_env().context("ETHSenderConfig")?;
-        let contracts = ContractsConfig::from_env().context("ContractsConfig")?;
-        let eth_client = ETHClientConfig::from_env().context("EthClientConfig")?;
-        genesis_init(&eth_sender, &network, &contracts, &eth_client.web3_url)
-            .await
-            .context("genesis_init")?;
-        if opt.genesis {
-            return Ok(());
-        }
-    }
-
-    let components = if opt.rebuild_tree {
-        vec![Component::Tree]
-    } else {
-        opt.components.0
-    };
-
-    // OneShotWitnessGenerator is the only component that is not expected to run indefinitely
-    // if this value is `false`, we expect all components to run indefinitely: we panic if any component returns.
-    let is_only_oneshot_witness_generator_task = matches!(
-        components.as_slice(),
-        [Component::WitnessGenerator(Some(_), _)]
-    );
-
     // TODO (QIT-22): Only deserialize configs on demand.
     // Right now, we are trying to deserialize all the configs that may be needed by `zksync_core`.
     // "May" is the key word here, since some configs are only used by certain component configuration,
     // hence we are using `Option`s.
-    let configs = TempConfigStore {
+    let configs: TempConfigStore = TempConfigStore {
+        postgres_config: PostgresConfig::from_env().ok(),
         health_check_config: HealthCheckConfig::from_env().ok(),
         merkle_tree_api_config: MerkleTreeApiConfig::from_env().ok(),
         web3_json_rpc_config: Web3JsonRpcConfig::from_env().ok(),
@@ -150,6 +125,40 @@ async fn main() -> anyhow::Result<()> {
         object_store_config: ObjectStoreConfig::from_env().ok(),
     };
 
+    let postgres_config = configs.postgres_config.clone().context("PostgresConfig")?;
+
+    if opt.genesis || is_genesis_needed(&postgres_config).await {
+        let network = NetworkConfig::from_env().context("NetworkConfig")?;
+        let eth_sender = ETHSenderConfig::from_env().context("ETHSenderConfig")?;
+        let contracts = ContractsConfig::from_env().context("ContractsConfig")?;
+        let eth_client = ETHClientConfig::from_env().context("EthClientConfig")?;
+        genesis_init(
+            &postgres_config,
+            &eth_sender,
+            &network,
+            &contracts,
+            &eth_client.web3_url,
+        )
+        .await
+        .context("genesis_init")?;
+        if opt.genesis {
+            return Ok(());
+        }
+    }
+
+    let components = if opt.rebuild_tree {
+        vec![Component::Tree]
+    } else {
+        opt.components.0
+    };
+
+    // OneShotWitnessGenerator is the only component that is not expected to run indefinitely
+    // if this value is `false`, we expect all components to run indefinitely: we panic if any component returns.
+    let is_only_oneshot_witness_generator_task = matches!(
+        components.as_slice(),
+        [Component::WitnessGenerator(Some(_), _)]
+    );
+
     // Run core actors.
     let (core_task_handles, stop_sender, cb_receiver, health_check_handle) =
         initialize_components(&configs, components, is_only_oneshot_witness_generator_task)

core/lib/config/Cargo.toml

@@ -11,8 +11,6 @@ categories = ["cryptography"]
 
 [dependencies]
 zksync_basic_types = { path = "../../lib/basic_types" }
-zksync_contracts = { path = "../../lib/contracts" }
 
 anyhow = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-envy = "0.4"