diff --git a/Cargo.lock b/Cargo.lock index 9c0dafdcb..6622a2fa4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2933,7 +2933,9 @@ dependencies = [ "fvm_shared", "ipc-api", "ipc-provider", + "lazy_static", "multiaddr", + "regex", "serde", "serde_with 2.3.3", "serial_test", diff --git a/fendermint/app/settings/Cargo.toml b/fendermint/app/settings/Cargo.toml index 146d375bc..b97880871 100644 --- a/fendermint/app/settings/Cargo.toml +++ b/fendermint/app/settings/Cargo.toml @@ -13,7 +13,9 @@ license.workspace = true anyhow = { workspace = true } config = { workspace = true } dirs = { workspace = true } +lazy_static = { workspace = true } multiaddr = { workspace = true } +regex = { workspace = true } serde = { workspace = true } serde_with = { workspace = true } serial_test = { workspace = true } diff --git a/fendermint/app/settings/src/lib.rs b/fendermint/app/settings/src/lib.rs index 3b0fb0247..f89854e46 100644 --- a/fendermint/app/settings/src/lib.rs +++ b/fendermint/app/settings/src/lib.rs @@ -11,6 +11,7 @@ use serde_with::{serde_as, DurationSeconds}; use std::path::{Path, PathBuf}; use std::time::Duration; use tendermint_rpc::Url; +use utils::EnvInterpol; use fendermint_vm_encoding::{human_readable_delegate, human_readable_str}; use fendermint_vm_topdown::BlockHeight; @@ -23,6 +24,7 @@ use ipc_provider::config::deserialize::deserialize_eth_address_from_str; pub mod eth; pub mod fvm; pub mod resolver; +pub mod utils; /// Marker to be used with the `#[serde_as(as = "IsHumanReadable")]` annotations. /// @@ -215,29 +217,6 @@ pub struct Settings { pub ipc: IpcSettings, } -#[macro_export] -macro_rules! home_relative { - // Using this inside something that has a `.home_dir()` function. - ($($name:ident),+) => { - $( - pub fn $name(&self) -> std::path::PathBuf { - expand_path(&self.home_dir(), &self.$name) - } - )+ - }; - - // Using this outside something that requires a `home_dir` parameter to be passed to it. - ($settings:ty { $($name:ident),+ } ) => { - impl $settings { - $( - pub fn $name(&self, home_dir: &std::path::Path) -> std::path::PathBuf { - $crate::expand_path(home_dir, &self.$name) - } - )+ - } - }; -} - impl Settings { home_relative!( data_dir, @@ -252,14 +231,18 @@ impl Settings { /// then overrides from the local environment. pub fn new(config_dir: &Path, home_dir: &Path, run_mode: &str) -> Result { let c = Config::builder() - .add_source(File::from(config_dir.join("default"))) + .add_source(EnvInterpol(File::from(config_dir.join("default")))) // Optional mode specific overrides, checked into git. - .add_source(File::from(config_dir.join(run_mode)).required(false)) + .add_source(EnvInterpol( + File::from(config_dir.join(run_mode)).required(false), + )) // Optional local overrides, not checked into git. - .add_source(File::from(config_dir.join("local")).required(false)) + .add_source(EnvInterpol( + File::from(config_dir.join("local")).required(false), + )) // Add in settings from the environment (with a prefix of FM) // e.g. `FM_DB__DATA_DIR=./foo/bar ./target/app` would set the database location. - .add_source( + .add_source(EnvInterpol( Environment::with_prefix("fm") .prefix_separator("_") .separator("__") @@ -268,7 +251,7 @@ impl Settings { .list_separator(",") // need to list keys explicitly below otherwise it can't pase simple `String` type .with_list_parse_key("resolver.discovery.static_addresses") .with_list_parse_key("resolver.membership.static_subnets"), - ) + )) // Set the home directory based on what was passed to the CLI, // so everything in the config can be relative to it. // The `home_dir` key is not added to `default.toml` so there is no confusion @@ -306,49 +289,12 @@ impl Settings { } } -/// Expand a path which can either be : -/// * absolute, e.g. "/foo/bar" -/// * relative to the system `$HOME` directory, e.g. "~/foo/bar" -/// * relative to the configured `--home-dir` directory, e.g. "foo/bar" -pub fn expand_path(home_dir: &Path, path: &Path) -> PathBuf { - if path.starts_with("/") { - PathBuf::from(path) - } else if path.starts_with("~") { - expand_tilde(path) - } else { - expand_tilde(home_dir.join(path)) - } -} - -/// Expand paths that begin with "~" to `$HOME`. -pub fn expand_tilde>(path: P) -> PathBuf { - let p = path.as_ref().to_path_buf(); - if !p.starts_with("~") { - return p; - } - if p == Path::new("~") { - return dirs::home_dir().unwrap_or(p); - } - dirs::home_dir() - .map(|mut h| { - if h == Path::new("/") { - // `~/foo` becomes just `/foo` instead of `//foo` if `/` is home. - p.strip_prefix("~").unwrap().to_path_buf() - } else { - h.push(p.strip_prefix("~/").unwrap()); - h - } - }) - .unwrap_or(p) -} - #[cfg(test)] mod tests { use std::path::PathBuf; use serial_test::serial; - use super::expand_tilde; use super::Settings; fn try_parse_config(run_mode: &str) -> Result { @@ -376,22 +322,10 @@ mod tests { // Run these tests serially because they modify the environment. #[serial] mod env { - use crate::tests::try_parse_config; + use multiaddr::multiaddr; - /// Set some env vars, run a fallible piece of code, then unset the variables otherwise they would affect the next test. - fn with_env_vars(vars: Vec<(&str, &str)>, f: F) -> Result - where - F: FnOnce() -> Result, - { - for (k, v) in vars.iter() { - std::env::set_var(k, v); - } - let result = f(); - for (k, _) in vars { - std::env::remove_var(k); - } - result - } + use crate::tests::try_parse_config; + use crate::utils::tests::with_env_vars; #[test] fn parse_comma_separated() { @@ -418,14 +352,30 @@ mod tests { assert_eq!(settings.resolver.discovery.static_addresses.len(), 0); assert_eq!(settings.resolver.membership.static_subnets.len(), 0); } - } - #[test] - fn tilde_expands_to_home() { - let home = std::env::var("HOME").expect("should work on Linux"); - let home_project = PathBuf::from(format!("{}/.project", home)); - assert_eq!(expand_tilde("~/.project"), home_project); - assert_eq!(expand_tilde("/foo/bar"), PathBuf::from("/foo/bar")); - assert_eq!(expand_tilde("~foo/bar"), PathBuf::from("~foo/bar")); + #[test] + fn parse_with_interpolation() { + let settings = with_env_vars( + vec![ + ("FM_RESOLVER__DISCOVERY__STATIC_ADDRESSES", "/dns4/${SEED_1_HOST}/tcp/${SEED_1_PORT},/dns4/${SEED_2_HOST}/tcp/${SEED_2_PORT}"), + ("SEED_1_HOST", "foo.io"), + ("SEED_1_PORT", "1234"), + ("SEED_2_HOST", "bar.ai"), + ("SEED_2_PORT", "5678"), + ], + || try_parse_config(""), + ) + .unwrap(); + + assert_eq!(settings.resolver.discovery.static_addresses.len(), 2); + assert_eq!( + settings.resolver.discovery.static_addresses[0], + multiaddr!(Dns4("foo.io"), Tcp(1234u16)) + ); + assert_eq!( + settings.resolver.discovery.static_addresses[1], + multiaddr!(Dns4("bar.ai"), Tcp(5678u16)) + ); + } } } diff --git a/fendermint/app/settings/src/utils.rs b/fendermint/app/settings/src/utils.rs new file mode 100644 index 000000000..ff80ba64c --- /dev/null +++ b/fendermint/app/settings/src/utils.rs @@ -0,0 +1,191 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use config::{ConfigError, Source, Value, ValueKind}; +use lazy_static::lazy_static; +use regex::Regex; +use std::path::{Path, PathBuf}; + +#[macro_export] +macro_rules! home_relative { + // Using this inside something that has a `.home_dir()` function. + ($($name:ident),+) => { + $( + pub fn $name(&self) -> std::path::PathBuf { + $crate::utils::expand_path(&self.home_dir(), &self.$name) + } + )+ + }; + + // Using this outside something that requires a `home_dir` parameter to be passed to it. + ($settings:ty { $($name:ident),+ } ) => { + impl $settings { + $( + pub fn $name(&self, home_dir: &std::path::Path) -> std::path::PathBuf { + $crate::utils::expand_path(home_dir, &self.$name) + } + )+ + } + }; +} + +/// Expand a path which can either be : +/// * absolute, e.g. "/foo/bar" +/// * relative to the system `$HOME` directory, e.g. "~/foo/bar" +/// * relative to the configured `--home-dir` directory, e.g. "foo/bar" +pub fn expand_path(home_dir: &Path, path: &Path) -> PathBuf { + if path.starts_with("/") { + PathBuf::from(path) + } else if path.starts_with("~") { + expand_tilde(path) + } else { + expand_tilde(home_dir.join(path)) + } +} + +/// Expand paths that begin with "~" to `$HOME`. +pub fn expand_tilde>(path: P) -> PathBuf { + let p = path.as_ref().to_path_buf(); + if !p.starts_with("~") { + return p; + } + if p == Path::new("~") { + return dirs::home_dir().unwrap_or(p); + } + dirs::home_dir() + .map(|mut h| { + if h == Path::new("/") { + // `~/foo` becomes just `/foo` instead of `//foo` if `/` is home. + p.strip_prefix("~").unwrap().to_path_buf() + } else { + h.push(p.strip_prefix("~/").unwrap()); + h + } + }) + .unwrap_or(p) +} + +#[derive(Clone, Debug)] +pub struct EnvInterpol(pub T); + +impl Source for EnvInterpol { + fn clone_into_box(&self) -> Box { + Box::new(self.clone()) + } + + fn collect(&self) -> Result, ConfigError> { + let mut values = self.0.collect()?; + for value in values.values_mut() { + interpolate_values(value); + } + Ok(values) + } +} + +/// Find values in the string that can be interpolated, e.g. "${NOMAD_HOST_ADDRESS_cometbft_p2p}" +fn find_vars(value: &str) -> Vec<&str> { + lazy_static! { + /// Capture env variables like `${VARIABLE_NAME}` + static ref ENV_VAR_RE: Regex = Regex::new(r"\$\{([^}]+)\}").expect("env var regex parses"); + } + ENV_VAR_RE + .captures_iter(value) + .map(|c| c.extract()) + .map(|(_, [n])| n) + .collect() +} + +/// Find variables and replace them from the environment. +/// +/// Returns `None` if there are no env vars in the value. +fn interpolate_vars(value: &str) -> Option { + let keys = find_vars(value); + if keys.is_empty() { + return None; + } + let mut value = value.to_string(); + for k in keys { + if let Ok(v) = std::env::var(k) { + value = value.replace(&format!("${{{k}}}"), &v); + } + } + Some(value) +} + +/// Find strings which have env vars in them and do the interpolation. +/// +/// It does not change the kind of the values, ie. it doesn't try to parse +/// into primitives or arrays *after* the interpolation. It does recurse +/// into arrays, though, so if there are variables within array items, +/// they get replaced. +fn interpolate_values(value: &mut Value) { + match value.kind { + ValueKind::String(ref mut s) => { + if let Some(i) = interpolate_vars(s) { + // TODO: We could try to parse into primitive values, + // but the only reason we do it with `Environment` is to support list separators, + // otherwise it was fine with just strings, so I think we can skip this for now. + *s = i; + } + } + ValueKind::Array(ref mut vs) => { + for v in vs.iter_mut() { + interpolate_values(v); + } + } + // Leave anything else as it is. + _ => {} + } +} + +#[cfg(test)] +pub(crate) mod tests { + use std::path::PathBuf; + + use crate::utils::find_vars; + + use super::{expand_tilde, interpolate_vars}; + + /// Set some env vars, run a fallible piece of code, then unset the variables otherwise they would affect the next test. + pub fn with_env_vars(vars: Vec<(&str, &str)>, f: F) -> Result + where + F: FnOnce() -> Result, + { + for (k, v) in vars.iter() { + std::env::set_var(k, v); + } + let result = f(); + for (k, _) in vars { + std::env::remove_var(k); + } + result + } + + #[test] + fn tilde_expands_to_home() { + let home = std::env::var("HOME").expect("should work on Linux"); + let home_project = PathBuf::from(format!("{}/.project", home)); + assert_eq!(expand_tilde("~/.project"), home_project); + assert_eq!(expand_tilde("/foo/bar"), PathBuf::from("/foo/bar")); + assert_eq!(expand_tilde("~foo/bar"), PathBuf::from("~foo/bar")); + } + + #[test] + fn test_find_vars() { + assert_eq!( + find_vars("FOO_${NAME}_${NUMBER}_BAR"), + vec!["NAME", "NUMBER"] + ); + assert!(find_vars("FOO_${NAME").is_empty()); + assert!(find_vars("FOO_$NAME").is_empty()); + } + + #[test] + fn test_interpolate_vars() { + let s = "FOO_${NAME}_${NUMBER}_BAR"; + let i = with_env_vars::<_, _, ()>(vec![("NAME", "spam")], || Ok(interpolate_vars(s))) + .unwrap() + .expect("non empty vars"); + assert_eq!(i, "FOO_spam_${NUMBER}_BAR"); + } +} diff --git a/fendermint/app/src/cmd/mod.rs b/fendermint/app/src/cmd/mod.rs index d5a86306d..c1a521d5a 100644 --- a/fendermint/app/src/cmd/mod.rs +++ b/fendermint/app/src/cmd/mod.rs @@ -5,7 +5,7 @@ use crate::{ options::{Commands, Options}, - settings::{expand_tilde, Settings}, + settings::{utils::expand_tilde, Settings}, }; use anyhow::{anyhow, Context}; use async_trait::async_trait;