Skip to content

Commit

Permalink
Merge pull request #46 from Commit-Boost/ltitanb/builder-events
Browse files Browse the repository at this point in the history 
builder events module
  • Loading branch information
@ltitanb
ltitanb authored Aug 1, 2024
2 parents 90cafdf + 43ced3d commit 2fa4b82
Show file tree
Hide file tree
Showing 52 changed files with 930 additions and 551 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ members = [
"crates/metrics",

"tests",
"examples/da_commit",
"examples/*",
]
resolver = "2"

Expand Down
3 changes: 2 additions & 1 deletion bin/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ pub mod prelude {
pub use cb_common::{
commit,
commit::request::SignRequest,
config::{load_module_config, StartModuleConfig},
config::{load_builder_module_config, load_commit_module_config, StartCommitModuleConfig},
pbs::{BuilderEvent, BuilderEventClient, OnBuilderApiEvent},
utils::{initialize_tracing_log, utcnow_ms, utcnow_ns, utcnow_sec, utcnow_us},
};
pub use cb_metrics::provider::MetricsProvider;
Expand Down
6 changes: 6 additions & 0 deletions config.example.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,11 @@ use_grafana = true

[[modules]]
id = "DA_COMMIT"
type = "commit"
docker_image = "test_da_commit"
sleep_secs = 5

[[modules]]
id = "BUILDER_LOG"
type = "events"
docker_image = "test_builder_log"
218 changes: 135 additions & 83 deletions crates/cli/src/docker_init.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ use std::{path::Path, vec};

use cb_common::{
config::{
CommitBoostConfig, CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV, METRICS_SERVER_ENV,
MODULE_ID_ENV, MODULE_JWT_ENV, SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS,
SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, SIGNER_KEYS_ENV, SIGNER_SERVER_ENV,
CommitBoostConfig, ModuleKind, BUILDER_SERVER_ENV, CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV,
METRICS_SERVER_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV,
SIGNER_DIR_SECRETS, SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, SIGNER_KEYS_ENV,
SIGNER_SERVER_ENV,
},
loader::SignerLoader,
utils::random_jwt,
Expand Down Expand Up @@ -51,17 +52,101 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()>
let signer_port = 20000;
let signer_server = format!("cb_signer:{signer_port}");

let builder_events_port = 30000;
let mut builder_events_modules = Vec::new();

// setup pbs service
targets.push(PrometheusTargetConfig {
targets: vec![format!("cb_pbs:{metrics_port}")],
labels: PrometheusLabelsConfig { job: "pbs".to_owned() },
});

let pbs_envs = IndexMap::from([
let mut pbs_envs = IndexMap::from([
get_env_same(CB_CONFIG_ENV),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
]);

let mut needs_signer_module = cb_config.pbs.with_signer;

// setup modules
if let Some(modules_config) = cb_config.modules {
for module in modules_config {
// TODO: support modules volumes and network
let module_cid = format!("cb_{}", module.id.to_lowercase());

targets.push(PrometheusTargetConfig {
targets: vec![format!("{module_cid}:{metrics_port}")],
labels: PrometheusLabelsConfig { job: module_cid.clone() },
});

let module_service = match module.kind {
// a commit module needs a JWT and access to the signer network
ModuleKind::Commit => {
needs_signer_module = true;

let jwt = random_jwt();
let jwt_name = format!("CB_JWT_{}", module.id.to_uppercase());

// module ids are assumed unique, so envs dont override each other
let module_envs = IndexMap::from([
get_env_val(MODULE_ID_ENV, &module.id),
get_env_same(CB_CONFIG_ENV),
get_env_interp(MODULE_JWT_ENV, &jwt_name),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
get_env_val(SIGNER_SERVER_ENV, &signer_server),
]);

envs.insert(jwt_name.clone(), jwt.clone());
jwts.insert(module.id.clone(), jwt);

Service {
container_name: Some(module_cid.clone()),
image: Some(module.docker_image),
// TODO: allow service to open ports here
networks: Networks::Simple(vec![
METRICS_NETWORK.to_owned(),
SIGNER_NETWORK.to_owned(),
]),
volumes: vec![config_volume.clone()],
environment: Environment::KvPair(module_envs),
depends_on: DependsOnOptions::Simple(vec!["cb_signer".to_owned()]),
..Service::default()
}
}
// an event module just needs a port to listen on
ModuleKind::Events => {
// module ids are assumed unique, so envs dont override each other
let module_envs = IndexMap::from([
get_env_val(MODULE_ID_ENV, &module.id),
get_env_same(CB_CONFIG_ENV),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
get_env_val(BUILDER_SERVER_ENV, &builder_events_port.to_string()),
]);

builder_events_modules.push(format!("{module_cid}:{builder_events_port}"));

Service {
container_name: Some(module_cid.clone()),
image: Some(module.docker_image),
networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]),
volumes: vec![config_volume.clone()],
environment: Environment::KvPair(module_envs),
depends_on: DependsOnOptions::Simple(vec!["cb_pbs".to_owned()]),
..Service::default()
}
}
};

services.insert(module_cid, Some(module_service));
}
};

if !builder_events_modules.is_empty() {
let env = builder_events_modules.join(",");
let (k, v) = get_env_val(BUILDER_SERVER_ENV, &env);
pbs_envs.insert(k, v);
}

let pbs_service = Service {
container_name: Some("cb_pbs".to_owned()),
image: Some(cb_config.pbs.docker_image),
Expand All @@ -77,102 +162,68 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()>

services.insert("cb_pbs".to_owned(), Some(pbs_service));

// setup modules
if let Some(modules_config) = cb_config.modules {
for module in modules_config {
// TODO: support modules volumes and network
let module_cid = format!("cb_{}", module.id.to_lowercase());
// TODO: validate if we have signer modules but not signer config

// setup signer service

if let Some(signer_config) = cb_config.signer {
if needs_signer_module {
let mut volumes = vec![config_volume.clone()];

targets.push(PrometheusTargetConfig {
targets: vec![format!("{module_cid}:{metrics_port}")],
labels: PrometheusLabelsConfig { job: module_cid.clone() },
targets: vec![format!("cb_signer:{metrics_port}")],
labels: PrometheusLabelsConfig { job: "signer".into() },
});

let jwt = random_jwt();
let jwt_name = format!("CB_JWT_{}", module.id.to_uppercase());

// module ids are assumed unique, so envs dont override each other
let module_envs = IndexMap::from([
get_env_val(MODULE_ID_ENV, &module.id),
let mut signer_envs = IndexMap::from([
get_env_same(CB_CONFIG_ENV),
get_env_interp(MODULE_JWT_ENV, &jwt_name),
get_env_same(JWTS_ENV),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
get_env_val(SIGNER_SERVER_ENV, &signer_server),
get_env_val(SIGNER_SERVER_ENV, &signer_port.to_string()),
]);

envs.insert(jwt_name.clone(), jwt.clone());
jwts.insert(module.id.clone(), jwt);
// TODO: generalize this, different loaders may not need volumes but eg ports
match signer_config.loader {
SignerLoader::File { key_path } => {
volumes.push(Volumes::Simple(format!("./{}:{}:ro", key_path, SIGNER_KEYS)));
let (k, v) = get_env_val(SIGNER_KEYS_ENV, SIGNER_KEYS);
signer_envs.insert(k, v);
}
SignerLoader::ValidatorsDir { keys_path, secrets_path } => {
volumes.push(Volumes::Simple(format!("{}:{}:ro", keys_path, SIGNER_DIR_KEYS)));
let (k, v) = get_env_val(SIGNER_DIR_KEYS_ENV, SIGNER_DIR_KEYS);
signer_envs.insert(k, v);

volumes.push(Volumes::Simple(format!(
"{}:{}:ro",
secrets_path, SIGNER_DIR_SECRETS
)));
let (k, v) = get_env_val(SIGNER_DIR_SECRETS_ENV, SIGNER_DIR_SECRETS);
signer_envs.insert(k, v);
}
};

// write jwts to env
let jwts_json = serde_json::to_string(&jwts).unwrap().clone();
envs.insert(JWTS_ENV.into(), format!("{jwts_json:?}"));

let module_service = Service {
container_name: Some(module_cid.clone()),
image: Some(module.docker_image),
// TODO: allow service to open ports here
let signer_service = Service {
container_name: Some("cb_signer".to_owned()),
image: Some(signer_config.docker_image),
networks: Networks::Simple(vec![
METRICS_NETWORK.to_owned(),
SIGNER_NETWORK.to_owned(),
]),
volumes: vec![config_volume.clone()],
environment: Environment::KvPair(module_envs),
depends_on: DependsOnOptions::Simple(vec!["cb_signer".to_owned()]),
volumes,
environment: Environment::KvPair(signer_envs),
..Service::default()
};

services.insert(module_cid, Some(module_service));
services.insert("cb_signer".to_owned(), Some(signer_service));
}
};

// TODO: validate if we have signer modules but not signer config

// setup signer service
if let Some(signer_config) = cb_config.signer {
let mut volumes = vec![config_volume.clone()];

targets.push(PrometheusTargetConfig {
targets: vec![format!("cb_signer:{metrics_port}")],
labels: PrometheusLabelsConfig { job: "signer".into() },
});

let mut signer_envs = IndexMap::from([
get_env_same(CB_CONFIG_ENV),
get_env_same(JWTS_ENV),
get_env_val(METRICS_SERVER_ENV, &metrics_port.to_string()),
get_env_val(SIGNER_SERVER_ENV, &signer_port.to_string()),
]);

// TODO: generalize this, different loaders may not need volumes but eg ports
match signer_config.loader {
SignerLoader::File { key_path } => {
volumes.push(Volumes::Simple(format!("./{}:{}:ro", key_path, SIGNER_KEYS)));
let (k, v) = get_env_val(SIGNER_KEYS_ENV, SIGNER_KEYS);
signer_envs.insert(k, v);
}
SignerLoader::ValidatorsDir { keys_path, secrets_path } => {
volumes.push(Volumes::Simple(format!("{}:{}:ro", keys_path, SIGNER_DIR_KEYS)));
let (k, v) = get_env_val(SIGNER_DIR_KEYS_ENV, SIGNER_DIR_KEYS);
signer_envs.insert(k, v);

volumes
.push(Volumes::Simple(format!("{}:{}:ro", secrets_path, SIGNER_DIR_SECRETS)));
let (k, v) = get_env_val(SIGNER_DIR_SECRETS_ENV, SIGNER_DIR_SECRETS);
signer_envs.insert(k, v);
}
};

// write jwts to env
let jwts_json = serde_json::to_string(&jwts)?.clone();
envs.insert(JWTS_ENV.into(), format!("{jwts_json:?}"));

let signer_service = Service {
container_name: Some("cb_signer".to_owned()),
image: Some(signer_config.docker_image),
networks: Networks::Simple(vec![METRICS_NETWORK.to_owned(), SIGNER_NETWORK.to_owned()]),
volumes,
environment: Environment::KvPair(signer_envs),
..Service::default()
};

services.insert("cb_signer".to_owned(), Some(signer_service));
};
} else if needs_signer_module {
panic!("Signer module required but no signer config provided");
}

// setup metrics services
// TODO: make this metrics optional?
Expand Down Expand Up @@ -274,6 +325,7 @@ fn get_env_interp(k: &str, v: &str) -> (String, Option<SingleValue>) {
get_env_val(k, &format!("${{{v}}}"))
}

// FOO=bar
fn get_env_val(k: &str, v: &str) -> (String, Option<SingleValue>) {
(k.into(), Some(SingleValue::String(v.into())))
}
Expand Down
7 changes: 7 additions & 0 deletions crates/common/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,18 @@ rust-version.workspace = true
# ethereum
alloy = { workspace = true, features = ["ssz"] }
ethereum_ssz.workspace = true
ssz_types.workspace = true
ethereum_ssz_derive.workspace = true
ethereum_serde_utils.workspace = true
ethereum-types.workspace = true

# networking
axum.workspace = true
reqwest.workspace = true

# async / threads
tokio.workspace = true

# serialization
toml.workspace = true
serde.workspace = true
Expand Down
Loading

0 comments on commit 2fa4b82

Please sign in to comment.