From 3715c2d44ef0f9b2eb957dded99f8dacfbf39901 Mon Sep 17 00:00:00 2001 From: Jarrett Tierney Date: Thu, 2 May 2024 13:21:29 -0700 Subject: [PATCH] logdog: switch to use apiclient cli to get settings This changes logdog to fetch settings using the apiclient. This creates two breaking changes. First logdog will now include sections such as 'os' in the log. This provides more information in the log. Secondly instead of removing the existence of secret settings their values are replaced with REDACTED. This allows the user to see the setting was there while maintaining the scrubbing of secrets. --- sources/Cargo.lock | 5 --- sources/logdog/Cargo.toml | 5 --- sources/logdog/src/error.rs | 19 +++----- sources/logdog/src/log_request.rs | 74 ++++++++++++++++++++----------- 4 files changed, 56 insertions(+), 47 deletions(-) diff --git a/sources/Cargo.lock b/sources/Cargo.lock index a9c503970b0..971e4a2153d 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -2584,14 +2584,9 @@ dependencies = [ name = "logdog" version = "0.1.0" dependencies = [ - "apiclient", - "bottlerocket-variant", - "constants", - "datastore", "flate2", "generate-readme", "glob", - "models", "reqwest", "serde_json", "shell-words", diff --git a/sources/logdog/Cargo.toml b/sources/logdog/Cargo.toml index 9fe5c6bbdca..307a6f5baf1 100644 --- a/sources/logdog/Cargo.toml +++ b/sources/logdog/Cargo.toml @@ -9,12 +9,8 @@ publish = false exclude = ["README.md"] [dependencies] -apiclient = { path = "../api/apiclient", version = "0.1" } -constants = { path = "../constants", version = "0.1" } -datastore = { path = "../api/datastore", version = "0.1" } flate2 = "1" glob = "0.3" -models = { path = "../models", version = "0.1" } reqwest = { version = "0.11", default-features = false, features = ["blocking", "rustls-tls-native-roots"] } serde_json = "1" shell-words = "1" @@ -26,5 +22,4 @@ url = "2" walkdir = "2" [build-dependencies] -bottlerocket-variant = { version = "0.1", path = "../bottlerocket-variant" } generate-readme = { version = "0.1", path = "../generate-readme" } diff --git a/sources/logdog/src/error.rs b/sources/logdog/src/error.rs index 677ef662ce8..4dfb282f13d 100644 --- a/sources/logdog/src/error.rs +++ b/sources/logdog/src/error.rs @@ -4,7 +4,6 @@ use std::io; use std::path::PathBuf; use crate::log_request::REQUESTS_DIR; -use datastore::{deserialization, serialization}; use reqwest::Url; use snafu::{Backtrace, Snafu}; @@ -12,13 +11,15 @@ use snafu::{Backtrace, Snafu}; #[snafu(visibility(pub(crate)))] #[allow(clippy::enum_variant_names)] pub(crate) enum Error { - #[snafu(display("Error calling Bottlerocket API '{}': {}", uri, source))] - ApiClient { - #[snafu(source(from(apiclient::Error, Box::new)))] - source: Box, - uri: String, + #[snafu(display("Error calling apiclient: {}", source))] + ApiCommandFailure { + #[snafu(source(from(std::io::Error, Box::new)))] + source: Box, }, + #[snafu(display("Error fetching settings via apiclient: {}", reason))] + ApiExecutionFailure { reason: String }, + #[snafu(display("Error creating the command stderr file '{}': {}", path.display(), source))] CommandErrFile { source: io::Error, @@ -67,9 +68,6 @@ pub(crate) enum Error { backtrace: Backtrace, }, - #[snafu(display("Error deserializing Settings: {} ", source))] - DeserializeSettings { source: deserialization::Error }, - #[snafu(display("Error creating the error file '{}': {}", path.display(), source))] ErrorFile { source: io::Error, @@ -167,9 +165,6 @@ pub(crate) enum Error { #[snafu(display("Cannot write to / as a file."))] RootAsFile { backtrace: Backtrace }, - #[snafu(display("Error serializing Settings: {} ", source))] - SerializeSettings { source: serialization::Error }, - #[snafu(display("Unable to deserialize Bottlerocket settings: {}", source))] SettingsJson { source: serde_json::Error }, diff --git a/sources/logdog/src/log_request.rs b/sources/logdog/src/log_request.rs index 557a5d2835d..e6c9c148e07 100644 --- a/sources/logdog/src/log_request.rs +++ b/sources/logdog/src/log_request.rs @@ -9,9 +9,7 @@ //! these provide the list of log requests that `logdog` will run. use crate::error::{self, Result}; -use datastore::deserialization::from_map; -use datastore::serialization::to_pairs; -use glob::{glob, Pattern}; +use glob::glob; use reqwest::blocking::{Client, Response}; use snafu::{ensure, OptionExt, ResultExt}; use std::collections::HashSet; @@ -28,15 +26,14 @@ pub(crate) const REQUESTS_DIR: &str = "/usr/share/logdog.d"; /// Patterns to filter from settings output. These follow the Unix shell style pattern outlined /// here: https://docs.rs/glob/0.3.0/glob/struct.Pattern.html. const SENSITIVE_SETTINGS_PATTERNS: &[&str] = &[ - "*.user-data", - "settings.kubernetes.bootstrap-token", + "/settings/kubernetes/bootstrap-token", // Can contain a username:password component - "settings.network.https-proxy", - "settings.kubernetes.server-key", - "settings.container-registry.credentials", + "/settings/network/https-proxy", + "/settings/kubernetes/server-key", + "/settings/container-registry/credentials", // Can be stored in settings.aws.credentials, but user can also add creds here - "settings.aws.config", - "settings.aws.credentials", + "/settings/aws/config", + "/settings/aws/credentials", ]; /// Returns the list of log requests to run by combining `VARIANT_REQUESTS` and `COMMON_REQUESTS`. @@ -166,24 +163,44 @@ where Ok(()) } +/// Strips user-data out of the settings model +fn strip_user_data(input: &mut serde_json::Value) { + let redacted = serde_json::Value::String("REDACTED".into()); + match input { + serde_json::Value::Object(map) => { + for (key, value) in map.iter_mut() { + if key == "user-data" { + *value = redacted.clone(); + } else { + strip_user_data(value) + } + } + } + serde_json::Value::Array(array) => { + for child in array { + strip_user_data(child) + } + } + _ => {} + } +} + /// Requests settings from the API, filters them, and writes the output to `tempdir` async fn handle_settings_request

(request: &LogRequest<'_>, tempdir: P) -> Result<()> where P: AsRef, { - let settings = get_settings().await?; - let mut settings_map = to_pairs(&settings).context(error::SerializeSettingsSnafu)?; + let mut settings = get_settings().await?; // Filter all settings that match any of the "sensitive" patterns + let redacted = serde_json::Value::String("REDACTED".into()); for pattern in SENSITIVE_SETTINGS_PATTERNS { - let pattern = - Pattern::new(pattern).context(error::ParseGlobPatternSnafu { pattern: *pattern })?; - settings_map.retain(|k, _| !pattern.matches(k.name().as_str())) + if let Some(found) = settings.pointer_mut(pattern) { + *found = redacted.clone(); + } } + strip_user_data(&mut settings); - // Serialize the map back to a `Settings` to remove the escaping so it writes nicely to file - let settings: model::Settings = - from_map(&settings_map).context(error::DeserializeSettingsSnafu)?; let outpath = tempdir.as_ref().join(request.filename); let outfile = File::create(&outpath).context(error::FileCreateSnafu { path: &outpath })?; serde_json::to_writer_pretty(&outfile, &settings) @@ -191,14 +208,21 @@ where Ok(()) } -/// Uses `apiclient` to request all settings from the apiserver and deserializes into a `Settings` -async fn get_settings() -> Result { - let uri = constants::API_SETTINGS_URI; - let (_status, response_body) = apiclient::raw_request(constants::API_SOCKET, uri, "GET", None) - .await - .context(error::ApiClientSnafu { uri })?; +/// Uses `apiclient` to request all settings from the apiserver and deserializes into a `json::Value` +async fn get_settings() -> Result { + let result = Command::new("/usr/bin/apiclient") + .arg("get") + .output() + .context(error::ApiCommandFailureSnafu)?; + + ensure!( + result.status.success(), + error::ApiExecutionFailureSnafu { + reason: String::from_utf8_lossy(&result.stderr) + } + ); - serde_json::from_str(&response_body).context(error::SettingsJsonSnafu) + serde_json::from_slice(result.stdout.as_slice()).context(error::SettingsJsonSnafu) } /// Runs an `exec` `LogRequest`'s `instructions` and writes its output to to `tempdir`.