From b9dc42c17a90b7bbc7e8146f9cc76cb96dad233d Mon Sep 17 00:00:00 2001 From: patrick Date: Tue, 4 Mar 2025 16:24:43 -0700 Subject: [PATCH] address PR comments --- crates/sui-indexer-alt-e2e-tests/src/lib.rs | 2 ++ .../tests/transactional_tests.rs | 2 ++ .../sui-indexer-alt-framework/src/cluster.rs | 2 ++ .../src/ingestion/client.rs | 18 +++++++++++++----- .../src/ingestion/mod.rs | 19 +++++++++++++++++-- .../src/ingestion/rpc_client.rs | 2 +- crates/sui-indexer-alt-framework/src/lib.rs | 2 ++ .../sui-indexer-alt-framework/src/metrics.rs | 3 ++- 8 files changed, 41 insertions(+), 9 deletions(-) diff --git a/crates/sui-indexer-alt-e2e-tests/src/lib.rs b/crates/sui-indexer-alt-e2e-tests/src/lib.rs index efea3ce9ca4fc..85ad22ab0d975 100644 --- a/crates/sui-indexer-alt-e2e-tests/src/lib.rs +++ b/crates/sui-indexer-alt-e2e-tests/src/lib.rs @@ -118,6 +118,8 @@ impl FullCluster { local_ingestion_path: Some(temp_dir.path().to_owned()), remote_store_url: None, rpc_api_url: None, + rpc_username: None, + rpc_password: None, }; let offchain = OffchainCluster::new( diff --git a/crates/sui-indexer-alt-e2e-tests/tests/transactional_tests.rs b/crates/sui-indexer-alt-e2e-tests/tests/transactional_tests.rs index 0152fa935a450..25c49c4f0b0b3 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/transactional_tests.rs +++ b/crates/sui-indexer-alt-e2e-tests/tests/transactional_tests.rs @@ -107,6 +107,8 @@ async fn cluster(config: &OffChainConfig) -> Arc { local_ingestion_path: Some(config.data_ingestion_path.clone()), remote_store_url: None, rpc_api_url: None, + rpc_username: None, + rpc_password: None, }; // This configuration controls how often the RPC service checks for changes to system packages. diff --git a/crates/sui-indexer-alt-framework/src/cluster.rs b/crates/sui-indexer-alt-framework/src/cluster.rs index 3e182acacbf98..d02332e54fc1e 100644 --- a/crates/sui-indexer-alt-framework/src/cluster.rs +++ b/crates/sui-indexer-alt-framework/src/cluster.rs @@ -259,6 +259,8 @@ mod tests { local_ingestion_path: Some(checkpoint_dir.path().to_owned()), remote_store_url: None, rpc_api_url: None, + rpc_username: None, + rpc_password: None, }, indexer_args: IndexerArgs { first_checkpoint: Some(0), diff --git a/crates/sui-indexer-alt-framework/src/ingestion/client.rs b/crates/sui-indexer-alt-framework/src/ingestion/client.rs index 5248a16c8f713..49429ef9a8300 100644 --- a/crates/sui-indexer-alt-framework/src/ingestion/client.rs +++ b/crates/sui-indexer-alt-framework/src/ingestion/client.rs @@ -49,7 +49,7 @@ pub type FetchResult = Result; pub enum FetchData { Raw(Bytes), - CheckPointData(CheckpointData), + CheckpointData(CheckpointData), } #[derive(Clone)] @@ -71,9 +71,17 @@ impl IngestionClient { Self::new_impl(client, metrics) } - pub(crate) fn new_rpc(url: Url, metrics: Arc) -> IngestionResult { - let client = Client::new(url.to_string())? - .with_auth(AuthInterceptor::basic(url.username(), url.password())); + pub(crate) fn new_rpc( + url: Url, + username: Option, + password: Option, + metrics: Arc, + ) -> IngestionResult { + let client = if let Some(username) = username { + Client::new(url.to_string())?.with_auth(AuthInterceptor::basic(username, password)) + } else { + Client::new(url.to_string())? + }; Ok(Self::new_impl(Arc::new(client), metrics)) } @@ -170,7 +178,7 @@ impl IngestionClient { ) })? } - FetchData::CheckPointData(data) => { + FetchData::CheckpointData(data) => { // We are not recording size metric for Checkpoint data (from RPC client). // TODO: Record the metric when we have a good way to get the size information data diff --git a/crates/sui-indexer-alt-framework/src/ingestion/mod.rs b/crates/sui-indexer-alt-framework/src/ingestion/mod.rs index f93c85d4a3668..be5a2b99f6b92 100644 --- a/crates/sui-indexer-alt-framework/src/ingestion/mod.rs +++ b/crates/sui-indexer-alt-framework/src/ingestion/mod.rs @@ -42,10 +42,18 @@ pub struct ClientArgs { #[clap(long, group = "source")] pub local_ingestion_path: Option, - /// Path to the local ingestion directory. + /// Sui fullnode gRPC url to fetch checkpoints from. /// If all remote_store_url, local_ingestion_path and rpc_api_url are provided, remote_store_url will be used. #[clap(long, env, group = "source")] pub rpc_api_url: Option, + + /// Optional username for the gRPC service. + #[clap(long, env)] + pub rpc_username: Option, + + /// Optional password for the gRPC service. + #[clap(long, env)] + pub rpc_password: Option, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -90,7 +98,12 @@ impl IngestionService { } else if let Some(path) = args.local_ingestion_path.as_ref() { IngestionClient::new_local(path.clone(), metrics.clone()) } else if let Some(rpc_api_url) = args.rpc_api_url.as_ref() { - IngestionClient::new_rpc(rpc_api_url.clone(), metrics.clone())? + IngestionClient::new_rpc( + rpc_api_url.clone(), + args.rpc_username, + args.rpc_password, + metrics.clone(), + )? } else { panic!("One of remote_store_url, local_ingestion_path or rpc_api_url must be provided"); }; @@ -214,6 +227,8 @@ mod tests { remote_store_url: Some(Url::parse(&uri).unwrap()), local_ingestion_path: None, rpc_api_url: None, + rpc_username: None, + rpc_password: None, }, IngestionConfig { checkpoint_buffer_size, diff --git a/crates/sui-indexer-alt-framework/src/ingestion/rpc_client.rs b/crates/sui-indexer-alt-framework/src/ingestion/rpc_client.rs index ee95ace702832..9cbf1a74b67a3 100644 --- a/crates/sui-indexer-alt-framework/src/ingestion/rpc_client.rs +++ b/crates/sui-indexer-alt-framework/src/ingestion/rpc_client.rs @@ -19,6 +19,6 @@ impl IngestionClientTrait for RpcClient { error: anyhow!(status), }, })?; - Ok(FetchData::CheckPointData(data)) + Ok(FetchData::CheckpointData(data)) } } diff --git a/crates/sui-indexer-alt-framework/src/lib.rs b/crates/sui-indexer-alt-framework/src/lib.rs index ab26dabecef8c..d71dd71f64833 100644 --- a/crates/sui-indexer-alt-framework/src/lib.rs +++ b/crates/sui-indexer-alt-framework/src/lib.rs @@ -194,6 +194,8 @@ impl Indexer { remote_store_url: None, local_ingestion_path: Some(tempdir().unwrap().into_path()), rpc_api_url: None, + rpc_username: None, + rpc_password: None, }, IngestionConfig::default(), Some(migrations), diff --git a/crates/sui-indexer-alt-framework/src/metrics.rs b/crates/sui-indexer-alt-framework/src/metrics.rs index 307d55882e36e..f0edce8d98935 100644 --- a/crates/sui-indexer-alt-framework/src/metrics.rs +++ b/crates/sui-indexer-alt-framework/src/metrics.rs @@ -179,7 +179,8 @@ impl IndexerMetrics { .unwrap(), total_ingested_bytes: register_int_counter_with_registry!( "indexer_total_ingested_bytes", - "Total number of bytes fetched from the remote store", + "Total number of bytes fetched from the remote store, this metric will not \ + be updated when data are fetched over gRPC.", registry, ) .unwrap(),