Skip to content

Commit

Permalink
address PR comments
Browse files Browse the repository at this point in the history
  • Loading branch information
patrickkuo committed Mar 4, 2025
1 parent 19c6c56 commit b9dc42c
Show file tree
Hide file tree
Showing 8 changed files with 41 additions and 9 deletions.
2 changes: 2 additions & 0 deletions crates/sui-indexer-alt-e2e-tests/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ impl FullCluster {
local_ingestion_path: Some(temp_dir.path().to_owned()),
remote_store_url: None,
rpc_api_url: None,
rpc_username: None,
rpc_password: None,
};

let offchain = OffchainCluster::new(
Expand Down
2 changes: 2 additions & 0 deletions crates/sui-indexer-alt-e2e-tests/tests/transactional_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ async fn cluster(config: &OffChainConfig) -> Arc<OffchainCluster> {
local_ingestion_path: Some(config.data_ingestion_path.clone()),
remote_store_url: None,
rpc_api_url: None,
rpc_username: None,
rpc_password: None,
};

// This configuration controls how often the RPC service checks for changes to system packages.
Expand Down
2 changes: 2 additions & 0 deletions crates/sui-indexer-alt-framework/src/cluster.rs
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,8 @@ mod tests {
local_ingestion_path: Some(checkpoint_dir.path().to_owned()),
remote_store_url: None,
rpc_api_url: None,
rpc_username: None,
rpc_password: None,
},
indexer_args: IndexerArgs {
first_checkpoint: Some(0),
Expand Down
18 changes: 13 additions & 5 deletions crates/sui-indexer-alt-framework/src/ingestion/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ pub type FetchResult = Result<FetchData, FetchError>;

pub enum FetchData {
Raw(Bytes),
CheckPointData(CheckpointData),
CheckpointData(CheckpointData),
}

#[derive(Clone)]
Expand All @@ -71,9 +71,17 @@ impl IngestionClient {
Self::new_impl(client, metrics)
}

pub(crate) fn new_rpc(url: Url, metrics: Arc<IndexerMetrics>) -> IngestionResult<Self> {
let client = Client::new(url.to_string())?
.with_auth(AuthInterceptor::basic(url.username(), url.password()));
pub(crate) fn new_rpc(
url: Url,
username: Option<String>,
password: Option<String>,
metrics: Arc<IndexerMetrics>,
) -> IngestionResult<Self> {
let client = if let Some(username) = username {
Client::new(url.to_string())?.with_auth(AuthInterceptor::basic(username, password))
} else {
Client::new(url.to_string())?
};
Ok(Self::new_impl(Arc::new(client), metrics))
}

Expand Down Expand Up @@ -170,7 +178,7 @@ impl IngestionClient {
)
})?
}
FetchData::CheckPointData(data) => {
FetchData::CheckpointData(data) => {
// We are not recording size metric for Checkpoint data (from RPC client).
// TODO: Record the metric when we have a good way to get the size information
data
Expand Down
19 changes: 17 additions & 2 deletions crates/sui-indexer-alt-framework/src/ingestion/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,18 @@ pub struct ClientArgs {
#[clap(long, group = "source")]
pub local_ingestion_path: Option<PathBuf>,

/// Path to the local ingestion directory.
/// Sui fullnode gRPC url to fetch checkpoints from.
/// If all remote_store_url, local_ingestion_path and rpc_api_url are provided, remote_store_url will be used.
#[clap(long, env, group = "source")]
pub rpc_api_url: Option<Url>,

/// Optional username for the gRPC service.
#[clap(long, env)]
pub rpc_username: Option<String>,

/// Optional password for the gRPC service.
#[clap(long, env)]
pub rpc_password: Option<String>,
}

#[derive(Serialize, Deserialize, Debug, Clone)]
Expand Down Expand Up @@ -90,7 +98,12 @@ impl IngestionService {
} else if let Some(path) = args.local_ingestion_path.as_ref() {
IngestionClient::new_local(path.clone(), metrics.clone())
} else if let Some(rpc_api_url) = args.rpc_api_url.as_ref() {
IngestionClient::new_rpc(rpc_api_url.clone(), metrics.clone())?
IngestionClient::new_rpc(
rpc_api_url.clone(),
args.rpc_username,
args.rpc_password,
metrics.clone(),
)?
} else {
panic!("One of remote_store_url, local_ingestion_path or rpc_api_url must be provided");
};
Expand Down Expand Up @@ -214,6 +227,8 @@ mod tests {
remote_store_url: Some(Url::parse(&uri).unwrap()),
local_ingestion_path: None,
rpc_api_url: None,
rpc_username: None,
rpc_password: None,
},
IngestionConfig {
checkpoint_buffer_size,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ impl IngestionClientTrait for RpcClient {
error: anyhow!(status),
},
})?;
Ok(FetchData::CheckPointData(data))
Ok(FetchData::CheckpointData(data))
}
}
2 changes: 2 additions & 0 deletions crates/sui-indexer-alt-framework/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ impl Indexer {
remote_store_url: None,
local_ingestion_path: Some(tempdir().unwrap().into_path()),
rpc_api_url: None,
rpc_username: None,
rpc_password: None,
},
IngestionConfig::default(),
Some(migrations),
Expand Down
3 changes: 2 additions & 1 deletion crates/sui-indexer-alt-framework/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,8 @@ impl IndexerMetrics {
.unwrap(),
total_ingested_bytes: register_int_counter_with_registry!(
"indexer_total_ingested_bytes",
"Total number of bytes fetched from the remote store",
"Total number of bytes fetched from the remote store, this metric will not \
be updated when data are fetched over gRPC.",
registry,
)
.unwrap(),
Expand Down

0 comments on commit b9dc42c

Please sign in to comment.