Merge pull request #2953 from jarhodes314/bug/c8y-root-cert-upload

fix: Parse root certificate from either file or directory in tedge cert upload

.gitignore

@@ -59,3 +59,6 @@ tedge-private-key.pem
 
 # temporary changelog
 _CHANGELOG.md
+
+# 
+mutants.out*

Cargo.toml

@@ -120,6 +120,7 @@ rumqttc = "0.23"
 # TODO: used git rev version to fix `unknown feature stdsimd` error: replace with 0.20 version after the release
 rumqttd = { git = "https://github.com/bytebeamio/rumqtt", rev = "0767080715699c34d8fe90b843716ba5ec12f8b9" }
 rustls = "0.21.11"
+rustls-native-certs = "0.6.3"
 rustls-pemfile = "1.0.1"
 serde = "1.0"
 serde_ignored = "0.1"

clippy.toml

@@ -0,0 +1,12 @@
+disallowed-types = [
+    { path = "reqwest::ClientBuilder", reason = "Use `certificate::CloudRootCerts` type instead to take root_cert_path configurations into account" },
+    { path = "reqwest::blocking::ClientBuilder", reason = "Use `certificate::CloudRootCerts` type instead to take root_cert_path configurations into account" },
+]
+disallowed-methods = [
+    { path = "reqwest::Client::builder", reason = "Use `certificate::CloudRootCerts` type instead to take root_cert_path configurations into account" },
+    { path = "reqwest::blocking::Client::builder", reason = "Use `certificate::CloudRootCerts` type instead to take root_cert_path configurations into account" },
+    { path = "reqwest::blocking::Client::new", reason = "Use `certificate::CloudRootCerts` type instead to take root_cert_path configurations into account" },
+    { path = "reqwest::Client::new", reason = "Use `certificate::CloudRootCerts` type instead to take root_cert_path configurations into account" },
+    { path = "hyper::client::Client::new", reason = "Use Client::builder()" },
+    { path = "hyper_rustls::HttpsConnectorBuilder::with_native_roots", reason = "Use .with_tls_config(tedge_config.cloud_client_tls_config()) instead to use configured root certificate paths for the connected cloud" },
+]

crates/common/axum_tls/src/acceptor.rs

@@ -107,6 +107,7 @@ fn common_name<'a>(cert: Option<&'a (&[u8], X509Certificate)>) -> Option<&'a str
 }
 
 #[cfg(test)]
+#[allow(clippy::disallowed_methods)]
 mod tests {
     use super::*;
     use crate::ssl_config;

crates/common/axum_tls/src/files.rs

@@ -321,10 +321,7 @@ mod tests {
             let app = Router::new().route("/test", get(|| async { "it works!" }));
 
             let task = tokio::spawn(crate::start_tls_server(listener, config, app));
-            let client = reqwest::Client::builder()
-                .add_root_certificate(cert)
-                .build()
-                .unwrap();
+            let client = client_builder().add_root_certificate(cert).build().unwrap();
             assert_eq!(
                 client
                     .get(format!("https://localhost:{port}/test"))
@@ -339,6 +336,11 @@ mod tests {
             task.abort();
         }
 
+        #[allow(clippy::disallowed_methods, clippy::disallowed_types)]
+        fn client_builder() -> reqwest::ClientBuilder {
+            reqwest::Client::builder()
+        }
+
         fn listener() -> (u16, std::net::TcpListener) {
             let mut port = 3500;
             loop {

crates/common/certificate/Cargo.toml

@@ -8,18 +8,27 @@ license = { workspace = true }
 homepage = { workspace = true }
 repository = { workspace = true }
 
+[features]
+default = []
+reqwest-blocking = ["dep:reqwest", "reqwest/blocking"]
+reqwest = ["dep:reqwest"]
+
 [dependencies]
+anyhow = { workspace = true }
+camino = { workspace = true }
 rcgen = { workspace = true }
+reqwest = { workspace = true, optional = true }
 rustls = { workspace = true }
+rustls-native-certs = { workspace = true }
 rustls-pemfile = { workspace = true }
 sha-1 = { workspace = true }
 thiserror = { workspace = true }
 time = { workspace = true }
+tracing = { workspace = true }
 x509-parser = { workspace = true }
 zeroize = { workspace = true }
 
 [dev-dependencies]
-anyhow = { workspace = true }
 assert_matches = { workspace = true }
 base64 = { workspace = true }
 tempfile = { workspace = true }

crates/common/certificate/src/cloud_root_certificate.rs

@@ -0,0 +1,110 @@
+use anyhow::Context;
+use camino::Utf8Path;
+use camino::Utf8PathBuf;
+use reqwest::Certificate;
+use std::fs::File;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub struct CloudRootCerts {
+    certificates: Arc<[Certificate]>,
+}
+
+impl CloudRootCerts {
+    #[allow(clippy::disallowed_types)]
+    pub fn client_builder(&self) -> reqwest::ClientBuilder {
+        self.certificates
+            .iter()
+            .cloned()
+            .fold(reqwest::ClientBuilder::new(), |builder, cert| {
+                builder.add_root_certificate(cert)
+            })
+    }
+
+    #[allow(clippy::disallowed_types)]
+    pub fn client(&self) -> reqwest::Client {
+        self.client_builder()
+            .build()
+            .expect("Valid reqwest client builder configuration")
+    }
+
+    #[allow(clippy::disallowed_types)]
+    #[cfg(feature = "reqwest-blocking")]
+    pub fn blocking_client_builder(&self) -> reqwest::blocking::ClientBuilder {
+        self.certificates
+            .iter()
+            .cloned()
+            .fold(reqwest::blocking::ClientBuilder::new(), |builder, cert| {
+                builder.add_root_certificate(cert)
+            })
+    }
+
+    #[allow(clippy::disallowed_types)]
+    #[cfg(feature = "reqwest-blocking")]
+    pub fn blocking_client(&self) -> reqwest::blocking::Client {
+        self.blocking_client_builder()
+            .build()
+            .expect("Valid reqwest client builder configuration")
+    }
+}
+
+impl From<Arc<[Certificate]>> for CloudRootCerts {
+    fn from(certificates: Arc<[Certificate]>) -> Self {
+        Self { certificates }
+    }
+}
+
+impl From<[Certificate; 0]> for CloudRootCerts {
+    fn from(certificates: [Certificate; 0]) -> Self {
+        Self {
+            certificates: Arc::new(certificates),
+        }
+    }
+}
+
+/// Read a directory into a [RootCertStore]
+pub fn read_trust_store(ca_dir_or_file: &Utf8Path) -> anyhow::Result<Vec<Certificate>> {
+    let mut certs = Vec::new();
+    for path in iter_file_or_directory(ca_dir_or_file) {
+        let path =
+            path.with_context(|| format!("reading metadata for file at {ca_dir_or_file}"))?;
+
+        if path.is_dir() {
+            continue;
+        }
+
+        let mut pem_file = match File::open(&path).map(std::io::BufReader::new) {
+            Ok(pem_file) => pem_file,
+            err if path == ca_dir_or_file => {
+                err.with_context(|| format!("failed to read from path {path:?}"))?
+            }
+            Err(_other_unreadable_file) => continue,
+        };
+
+        let ders = rustls_pemfile::certs(&mut pem_file)
+            .with_context(|| format!("reading {path}"))?
+            .into_iter()
+            .map(|der| Certificate::from_der(&der).unwrap());
+        certs.extend(ders)
+    }
+
+    Ok(certs)
+}
+
+fn iter_file_or_directory(
+    possible_dir: &Utf8Path,
+) -> Box<dyn Iterator<Item = anyhow::Result<Utf8PathBuf>> + 'static> {
+    let path = possible_dir.to_path_buf();
+    if let Ok(dir) = possible_dir.read_dir_utf8() {
+        Box::new(dir.map(move |file| match file {
+            Ok(file) => {
+                let mut path = path.clone();
+                path.push(file.file_name());
+                Ok(path)
+            }
+            Err(e) => Err(e).with_context(|| format!("reading metadata for file in {path}")),
+        }))
+    } else {
+        Box::new([Ok(path)].into_iter())
+    }
+}

crates/common/certificate/src/lib.rs

@@ -9,6 +9,11 @@ use std::path::PathBuf;
 use time::Duration;
 use time::OffsetDateTime;
 use zeroize::Zeroizing;
+#[cfg(feature = "reqwest")]
+mod cloud_root_certificate;
+#[cfg(feature = "reqwest")]
+pub use cloud_root_certificate::*;
+
 pub mod device_id;
 pub mod parse_root_certificate;
 pub struct PemCertificate {

crates/common/certificate/src/parse_root_certificate.rs

@@ -28,6 +28,46 @@ pub fn create_tls_config(
         .with_client_auth_cert(cert_chain, pvt_key)?)
 }
 
+pub fn client_config_for_ca_certificates<P>(
+    root_certificates: impl IntoIterator<Item = P>,
+) -> Result<ClientConfig, std::io::Error>
+where
+    P: AsRef<Path>,
+{
+    let mut roots = RootCertStore::empty();
+    for cert_path in root_certificates {
+        rec_add_root_cert(&mut roots, cert_path.as_ref());
+    }
+
+    let (mut valid_count, mut invalid_count) = (0, 0);
+    for cert in rustls_native_certs::load_native_certs().expect("could not load platform certs") {
+        match roots.add(&Certificate(cert.0)) {
+            Ok(_) => valid_count += 1,
+            Err(err) => {
+                tracing::debug!("certificate parsing failed: {:?}", err);
+                invalid_count += 1
+            }
+        }
+    }
+    tracing::debug!(
+        "with_native_roots processed {} valid and {} invalid certs",
+        valid_count,
+        invalid_count
+    );
+    if roots.is_empty() {
+        tracing::debug!("no valid root CA certificates found");
+        Err(std::io::Error::new(
+            std::io::ErrorKind::NotFound,
+            format!("no valid root CA certificates found ({invalid_count} invalid)"),
+        ))?
+    }
+
+    Ok(ClientConfig::builder()
+        .with_safe_defaults()
+        .with_root_certificates(roots)
+        .with_no_client_auth())
+}
+
 pub fn add_certs_from_file(
     root_store: &mut RootCertStore,
     cert_file: impl AsRef<Path>,

crates/common/download/Cargo.toml

@@ -13,6 +13,7 @@ repository = { workspace = true }
 anyhow = { workspace = true, features = ["backtrace"] }
 axum_tls = { workspace = true, features = ["error-matching"] }
 backoff = { workspace = true }
+certificate = { workspace = true, features = ["reqwest"] }
 hyper = { workspace = true }
 log = { workspace = true }
 nix = { workspace = true }

crates/common/download/examples/simple_download.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use certificate::CloudRootCerts;
 use download::DownloadInfo;
 use download::Downloader;
 
@@ -12,7 +13,7 @@ async fn main() -> Result<()> {
 
     // Create downloader instance with desired file path and target directory.
     #[allow(deprecated)]
-    let downloader = Downloader::new("/tmp/test_download".into(), None);
+    let downloader = Downloader::new("/tmp/test_download".into(), None, CloudRootCerts::from([]));
 
     // Call `download` method to get data from url.
     downloader.download(&url_data).await?;