From aa0ccce7935f8de8cdb9a97a4aa8518e5042fedf Mon Sep 17 00:00:00 2001 From: Brian May Date: Tue, 26 Nov 2024 15:03:34 +1100 Subject: [PATCH] fix: rewrite caching API --- Cargo.lock | 410 +++++++++++++++++++++++++++++++++++++-------- Cargo.toml | 8 +- src/repos/cache.rs | 238 +++++++++++++++++--------- src/repos/hash.rs | 2 +- src/repos/helm.rs | 93 +++++++--- src/repos/local.rs | 158 +++++++++++++---- src/repos/meta.rs | 14 +- src/repos/mod.rs | 12 +- src/repos/oci.rs | 157 ++++++++++++----- 9 files changed, 827 insertions(+), 265 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37f32bc..c8a54f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,12 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - [[package]] name = "aho-corasick" version = "1.1.3" @@ -108,6 +102,106 @@ version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ca9a001c1e8ba5149f91a74362376cc6bc5b919d92d988668657bd570bdcec" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.3.1", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", +] + +[[package]] +name = "async-io" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" +dependencies = [ + "async-lock", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener 5.3.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" +dependencies = [ + "async-channel 2.3.1", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener 5.3.1", + "futures-lite", + "rustix", + "tracing", +] + [[package]] name = "async-recursion" version = "1.0.0" @@ -119,6 +213,71 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "async-signal" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-std" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c634475f29802fde2b8f0b505b1bd00dfe4df7d4a000f0b36f7671197d5c3615" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io", + "async-lock", + "async-process", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-tar" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a42f905d4f623faf634bbd1e001e84e0efc24694afa64be9ad239bf6ca49e1f8" +dependencies = [ + "async-std", + "filetime", + "libc", + "pin-project", + "redox_syscall 0.2.16", + "xattr", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.83" @@ -130,6 +289,12 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.1.0" @@ -173,7 +338,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide 0.6.2", + "miniz_oxide", "object", "rustc-demangle", ] @@ -240,6 +405,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" +dependencies = [ + "async-channel 2.3.1", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "bstr" version = "1.9.1" @@ -421,6 +599,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -447,13 +634,10 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.4.2" +name = "crossbeam-utils" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crossterm" @@ -688,6 +872,33 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener 5.3.1", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.2.0" @@ -706,16 +917,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "flate2" -version = "1.0.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" -dependencies = [ - "crc32fast", - "miniz_oxide 0.8.0", -] - [[package]] name = "fnv" version = "1.0.7" @@ -791,6 +992,19 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-lite" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-locks" version = "0.7.0" @@ -888,6 +1102,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.2" @@ -947,15 +1173,14 @@ name = "helmci" version = "2.0.0" dependencies = [ "anyhow", + "async-std", + "async-tar", "async-trait", - "bytes", "chrono", "clap", "crossterm", "docker_credential", - "flate2", "futures", - "futures-util", "hyper", "hyper-rustls", "hyper-util", @@ -974,11 +1199,8 @@ dependencies = [ "slack-morphism", "tabled", "tap", - "tar", - "tempfile", "thiserror 2.0.3", "tokio", - "tokio-util", "url", ] @@ -988,6 +1210,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -1403,6 +1631,15 @@ dependencies = [ "sha2", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1503,7 +1740,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.5.0", "libc", - "redox_syscall", + "redox_syscall 0.5.7", ] [[package]] @@ -1552,6 +1789,9 @@ name = "log" version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +dependencies = [ + "value-bag", +] [[package]] name = "lru" @@ -1599,22 +1839,13 @@ dependencies = [ "adler", ] -[[package]] -name = "miniz_oxide" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" -dependencies = [ - "adler2", -] - [[package]] name = "mio" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "log", "wasi", @@ -1764,6 +1995,12 @@ dependencies = [ "unicode-width 0.1.11", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1782,7 +2019,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.7", "smallvec", "windows-targets 0.52.6", ] @@ -1799,6 +2036,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "pin-project-lite" version = "0.2.12" @@ -1811,6 +2068,32 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "polling" +version = "3.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.4.0", + "pin-project-lite", + "rustix", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1995,6 +2278,15 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -2634,30 +2926,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "tar" -version = "0.4.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6" -dependencies = [ - "filetime", - "libc", - "xattr", -] - -[[package]] -name = "tempfile" -version = "3.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" -dependencies = [ - "cfg-if", - "fastrand", - "once_cell", - "rustix", - "windows-sys 0.59.0", -] - [[package]] name = "termcolor" version = "1.1.3" @@ -3020,6 +3288,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "value-bag" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef4c4aa54d5d05a279399bfa921ec387b7aba77caf7a682ae8d86785b8fdad2" + [[package]] name = "version_check" version = "0.9.5" @@ -3419,13 +3693,11 @@ checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] name = "xattr" -version = "1.3.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" dependencies = [ "libc", - "linux-raw-sys", - "rustix", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1a9b5dd..493624d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,9 +34,5 @@ thiserror = "2.0.3" sha256 = "1.5.0" docker_credential = "1.3.1" oci-client = { version = "0.14.0", default-features = false, features = [ "rustls-tls"] } -tempfile = "3.14.0" -bytes = "1.8.0" -flate2 = "1.0.35" -tar = "0.4.43" -tokio-util = "0.7.12" -futures-util = "0.3.31" +async-tar = "0.5.0" +async-std = "1.13.0" diff --git a/src/repos/cache.rs b/src/repos/cache.rs index f705054..a8748b1 100644 --- a/src/repos/cache.rs +++ b/src/repos/cache.rs @@ -1,36 +1,23 @@ use std::fmt::Debug; +use std::io::SeekFrom; use std::path::Path; use std::path::PathBuf; -use bytes::Bytes; -use futures::Stream; -use futures::StreamExt; +use super::hash::Sha256Hash; +use async_std::fs::File; +use async_std::io::SeekExt; +use async_std::io::WriteExt; +use tap::Pipe; use thiserror::Error; -use tokio::io::AsyncWriteExt; - -use super::charts::Chart; -use super::{hash::Sha256Hash, meta::Meta}; #[derive(Error, Debug)] pub enum Error { #[error("IO error {0}: {1}")] Io(PathBuf, std::io::Error), - #[error("Digest mismatch {0}: {1} != {2}")] - DigestMismatch(PathBuf, Sha256Hash, Sha256Hash), - // #[error("Invalid JSON {0}: {1}")] - // InvalidJson(PathBuf, serde_json::Error), -} - -#[derive(Error, Debug)] -pub enum StreamError { - #[error("IO error {0}: {1}")] - Io(PathBuf, std::io::Error), - #[error("Digest mismatch {0}: {1} != {2}")] - DigestMismatch(PathBuf, Sha256Hash, Sha256Hash), - // #[error("Invalid JSON {0}: {1}")] - // InvalidJson(PathBuf, serde_json::Error), - #[error("Stream error: {0}")] - Stream(PathBuf, E), + #[error("SHA256 hash mismatch {0}: {1} != {2}")] + Sha256HashMismatch(PathBuf, Sha256Hash, Sha256Hash), + #[error("File too big error {0}: expected {1} got {2}")] + TooBig(PathBuf, u64, u64), } #[derive(Debug)] @@ -38,87 +25,186 @@ pub struct Cache { pub path: std::path::PathBuf, } -impl Cache { - pub fn new(path: &Path) -> Self { +#[derive(Clone, Debug)] +pub(super) struct Key { + pub name: String, + pub sha256_hash: Sha256Hash, +} + +pub(super) struct CreateCacheEntry { + path: PathBuf, + temp_path: PathBuf, + file: File, + expected_size: Option, +} + +impl CreateCacheEntry { + async fn new( + path: PathBuf, + temp_path: PathBuf, + expected_size: Option, + ) -> Result { + let parent = temp_path.parent().ok_or_else(|| { + Error::Io( + temp_path.clone(), + std::io::Error::new(std::io::ErrorKind::NotFound, "No parent directory"), + ) + })?; + + tokio::fs::create_dir_all(&parent) + .await + .map_err(|e| Error::Io(parent.into(), e))?; + + let file = File::create(&temp_path) + .await + .map_err(|e| Error::Io(temp_path.clone(), e))?; + Self { - path: path.join(".charts"), + path, + temp_path, + file, + expected_size, } + .pipe(Ok) } - fn get_cache_file_path(&self, chart_name: &str, digest: &Sha256Hash) -> PathBuf { - self.path.join(chart_name).join(format!("{digest}.tgz")) + pub(super) async fn write_chunk(&mut self, chunk: &[u8]) -> Result<(), Error> { + self.file + .write_all(chunk) + .await + .map_err(|e| Error::Io(self.temp_path.clone(), e))?; + + if let Some(expected_size) = self.expected_size { + let file_len = self + .file + .seek(SeekFrom::End(0)) + .await + .map_err(|e| Error::Io(self.temp_path.clone(), e))?; + + if file_len > expected_size { + return Err(Error::TooBig( + self.temp_path.clone(), + expected_size, + file_len, + )); + } + } + + Ok(()) } - fn get_cache_temp_path(&self, chart_name: &str, digest: &Sha256Hash) -> PathBuf { - self.path.join(chart_name).join(format!("{digest}.tmp")) + // pub(super) const fn file(&self) -> &File { + // &self.file + // } + + pub(super) fn mut_file(&mut self) -> &mut File { + &mut self.file } - pub(super) async fn get_cache_entry(&self, meta: &Meta) -> Result, Error> { - let file_path = self.get_cache_file_path(&meta.name, &meta.sha256_hash); + fn get_cache_file_path(&self, key: &Key) -> PathBuf { + self.path + .join(&key.name) + .join(format!("{}.tgz", key.sha256_hash)) + } - if !file_path.exists() { - return Ok(None); - } + pub(super) async fn finalize(mut self, key: &Key) -> Result { + let final_path = self.get_cache_file_path(key); - let sha256_hash = Sha256Hash::from_file_async(&file_path) - .await - .map_err(|e| Error::Io(file_path.clone(), e))?; + self.flush().await?; - if meta.sha256_hash != sha256_hash { - return Err(Error::DigestMismatch( - file_path.clone(), - meta.sha256_hash.clone(), + let sha256_hash = self.calc_sha256_hash().await?; + if key.sha256_hash != sha256_hash { + return Err(Error::Sha256HashMismatch( + self.temp_path.clone(), + key.sha256_hash.clone(), sha256_hash, )); } - Ok(Some(Chart { - file_path, - meta: meta.clone(), - })) + let parent = final_path.parent().ok_or_else(|| { + Error::Io( + final_path.clone(), + std::io::Error::new(std::io::ErrorKind::NotFound, "No parent directory"), + ) + })?; + + tokio::fs::create_dir_all(&parent) + .await + .map_err(|e| Error::Io(parent.into(), e))?; + + tokio::fs::rename(&self.temp_path, &final_path) + .await + .map_err(|e| Error::Io(final_path.clone(), e))?; + + Ok(final_path.clone()) } - pub(super) async fn create_cache_entry( - &self, - meta: Meta, - mut stream: impl Stream> + Unpin + Send, - ) -> Result> { - let file_path = self.get_cache_file_path(&meta.name, &meta.sha256_hash); - let temp_path = self.get_cache_temp_path(&meta.name, &meta.sha256_hash); - - if let Some(parent) = temp_path.parent() { - std::fs::create_dir_all(parent).map_err(|e| StreamError::Io(temp_path.clone(), e))?; + pub(super) async fn flush(&mut self) -> Result<(), Error> { + self.file + .flush() + .await + .map_err(|e| Error::Io(self.temp_path.clone(), e)) + } + + pub(super) async fn calc_sha256_hash(&self) -> Result { + Sha256Hash::from_async_path(&self.temp_path) + .await + .map_err(|e| Error::Io(self.temp_path.clone(), e)) + } +} + +impl Drop for CreateCacheEntry { + fn drop(&mut self) { + if self.temp_path.exists() { + let _ = std::fs::remove_file(&self.temp_path); } + } +} - { - let mut file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| StreamError::Io(temp_path.clone(), e))?; +// Cache is not thread safe as creating entries will have race conditions. +unsafe impl Sync for Cache {} + +impl Cache { + pub fn new(path: &Path) -> Self { + Self { + path: path.join(".charts"), + } + } + + fn get_cache_file_path(&self, key: &Key) -> PathBuf { + self.path + .join(&key.name) + .join(format!("{}.tgz", key.sha256_hash)) + } - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|e| StreamError::Stream(temp_path.clone(), e))?; + pub(super) async fn get_cache_entry(&self, key: &Key) -> Result, Error> { + let file_path = self.get_cache_file_path(key); - file.write_all(&chunk) - .await - .map_err(|e| StreamError::Io(temp_path.clone(), e))?; - } + if !file_path.exists() { + return Ok(None); } - let sha256_hash = Sha256Hash::from_file_async(&temp_path) + let sha256_hash = Sha256Hash::from_async_path(&file_path) .await - .map_err(|e| StreamError::Io(temp_path.clone(), e))?; - if meta.sha256_hash != sha256_hash { - return Err(StreamError::DigestMismatch( - temp_path, - meta.sha256_hash.clone(), + .map_err(|e| Error::Io(file_path.clone(), e))?; + + if key.sha256_hash != sha256_hash { + return Err(Error::Sha256HashMismatch( + file_path.clone(), + key.sha256_hash.clone(), sha256_hash, )); } - tokio::fs::rename(&temp_path, &file_path) - .await - .map_err(|e| StreamError::Io(file_path.clone(), e))?; + Ok(Some(file_path)) + } - Ok(Chart { file_path, meta }) + pub(super) async fn create_cache_entry( + &self, + extension: &str, + expected_size: Option, + ) -> Result { + let temp_path = self.path.join(format!("tmp{extension}")); + CreateCacheEntry::new(self.path.clone(), temp_path, expected_size).await } } diff --git a/src/repos/hash.rs b/src/repos/hash.rs index 1ce08e1..adf2e4f 100644 --- a/src/repos/hash.rs +++ b/src/repos/hash.rs @@ -15,7 +15,7 @@ impl Sha256Hash { // Ok(Self(hash)) // } - pub async fn from_file_async(path: &std::path::Path) -> Result { + pub async fn from_async_path(path: &std::path::Path) -> Result { let hash = path.async_digest().await?; Ok(Self(hash)) } diff --git a/src/repos/helm.rs b/src/repos/helm.rs index e3b457a..e8d2055 100644 --- a/src/repos/helm.rs +++ b/src/repos/helm.rs @@ -1,9 +1,11 @@ use std::collections::HashMap; +use std::path::PathBuf; +use futures::StreamExt; use thiserror::Error; use url::Url; -use super::cache::{self, Cache}; +use super::cache::{self, Cache, Key}; use super::hash::Sha256Hash; use super::meta::RepoSpecific; use super::{charts::Chart, meta::Meta}; @@ -18,18 +20,20 @@ pub enum Error { Parse(#[from] serde_yml::Error), #[error("Failed to append url: {0}")] UrlJoin(#[from] AppendUrlError), - #[error("IO error: {0}")] - Io(#[from] std::io::Error), + #[error("File error: {0}")] + File(PathBuf, std::io::Error), #[error("Cache error: {0}")] Cache(#[from] cache::Error), - #[error("Cache streaming error: {0}")] - CacheStream(#[from] cache::StreamError), #[error("Chart not found")] ChartNotFound, #[error("Failed to parse version: {0}")] VersionParse(#[from] versions::Error), #[error("Chart missing digest")] MissingDigest, + #[error("Size mismatch {0}: expected {1}, but got {2}")] + SizeMismatch(PathBuf, u64, u64), + #[error("Reqwest error {0}: {1}")] + Reqwest(Url, reqwest::Error), } #[allow(dead_code)] @@ -48,33 +52,71 @@ struct Entry { } impl Entry { - async fn download_chart(&self, cache: &Cache) -> Result { - let Some(digest) = &self.digest else { + async fn download_chart( + &self, + cache: &Cache, + expected_size: Option, + ) -> Result { + let Some(sha256_hash) = &self.digest else { return Err(Error::MissingDigest); }; - let chart = Meta { + let key = Key { name: self.name.clone(), - version: self.version.clone(), - sha256_hash: digest.clone(), - created: Some(self.created), - repo: RepoSpecific::Helm, + sha256_hash: sha256_hash.clone(), }; - if let Some(entry) = cache.get_cache_entry(&chart).await? { - return Ok(entry); + if let Some(file) = cache.get_cache_entry(&key).await? { + self.file_to_chart(file, sha256_hash, expected_size)?; } let url = self.urls.first().ok_or(Error::ChartNotFound)?; - let stream = reqwest::get(url.as_str()) - .await? - .error_for_status()? - .bytes_stream(); + let mut file = cache.create_cache_entry(".tgz", expected_size).await?; + + { + let mut stream = reqwest::get(url.as_str()) + .await? + .error_for_status()? + .bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| Error::Reqwest(url.clone(), e))?; + file.write_chunk(&chunk).await?; + } + } + + let path = file.finalize(&key).await?; + self.file_to_chart(path, sha256_hash, expected_size) + } + + fn file_to_chart( + &self, + file_path: PathBuf, + sha256_hash: &Sha256Hash, + expected_size: Option, + ) -> Result { + let metadata = file_path + .metadata() + .map_err(|e| Error::File(file_path.clone(), e))?; + let size = metadata.len(); + + if let Some(expected_size) = expected_size { + if expected_size != size { + return Err(Error::SizeMismatch(file_path, expected_size, size)); + } + } + + let meta = Meta { + name: self.name.clone(), + version: self.version.clone(), + sha256_hash: sha256_hash.clone(), + created: Some(self.created), + repo: RepoSpecific::Helm, + size, + }; - let stream = stream; - let cache = cache.create_cache_entry(chart, stream).await?; + let chart = Chart { file_path, meta }; - Ok(cache) + Ok(chart) } } @@ -117,7 +159,7 @@ impl Repo { ) -> Result { let entry = self._get_by_version(name, version); if let Some(entry) = entry { - Ok(entry.download_chart(cache).await?) + Ok(entry.download_chart(cache, None).await?) } else { Err(Error::ChartNotFound) } @@ -126,7 +168,7 @@ impl Repo { pub async fn get_by_meta(&self, meta: &Meta, cache: &Cache) -> Result { let entry = self._get_by_sha256(&meta.name, &meta.sha256_hash); if let Some(entry) = entry { - Ok(entry.download_chart(cache).await?) + Ok(entry.download_chart(cache, Some(meta.size)).await?) } else { Err(Error::ChartNotFound) } @@ -230,7 +272,7 @@ generated: 2016-10-06T16:23:20.499029981-06:00 assert!(ingress.file_path.exists()); - let hash = Sha256Hash::from_file_async(&ingress.file_path) + let hash = Sha256Hash::from_async_path(&ingress.file_path) .await .unwrap(); assert_eq!(hash, hash); @@ -256,6 +298,7 @@ generated: 2016-10-06T16:23:20.499029981-06:00 sha256_hash: expected_hash.clone(), created: None, repo: RepoSpecific::Helm, + size: 58282, }; let cache = Cache::new(std::path::Path::new("tmp/helm_download_by_meta")); @@ -264,7 +307,7 @@ generated: 2016-10-06T16:23:20.499029981-06:00 assert_eq!(ingress.meta.version, "4.11.3"); assert_eq!(ingress.meta.sha256_hash, expected_hash); - let hash = Sha256Hash::from_file_async(&ingress.file_path) + let hash = Sha256Hash::from_async_path(&ingress.file_path) .await .unwrap(); assert_eq!(hash, hash); diff --git a/src/repos/local.rs b/src/repos/local.rs index 9dd097b..9d7a4d5 100644 --- a/src/repos/local.rs +++ b/src/repos/local.rs @@ -1,16 +1,10 @@ use std::fs::File; -use std::path::Path; +use std::path::{Path, PathBuf}; -use bytes::BytesMut; -use flate2::read::GzEncoder; -use flate2::Compression; -use futures::stream::TryStreamExt; -use futures_util::TryFutureExt; use serde::Deserialize; use thiserror::Error; -use tokio::pin; -use tokio_util::codec::{BytesCodec, FramedRead}; +use crate::repos::cache::Key; use crate::repos::meta::RepoSpecific; use super::cache::{self, Cache}; @@ -23,14 +17,14 @@ pub enum Error { Download(#[from] reqwest::Error), #[error("Failed to parse index: {0}")] Parse(#[from] serde_yml::Error), - #[error("IO error: {0}")] - Io(#[from] std::io::Error), + #[error("File error: {0}")] + File(PathBuf, std::io::Error), #[error("Cache error: {0}")] Cache(#[from] cache::Error), - #[error("Cache streaming error: {0}")] - CacheStream(#[from] cache::StreamError), #[error("Invalid JSON")] InvalidJson(#[from] serde_json::Error), + #[error("Size mismatch {0}: expected {1}, but got {2}")] + SizeMismatch(PathBuf, u64, u64), } #[allow(dead_code)] @@ -47,45 +41,102 @@ struct Entry { } impl Entry { - async fn download_chart(&self, path: &Path, cache: &Cache) -> Result { - // FIXME: management of temp file is yuck - let tempfile = Path::new("archive.tar.gz"); - { - let tar_gz = File::create(tempfile)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut tar = tar::Builder::new(enc); - tar.append_dir_all(format!("{}-{}", self.name, self.version), path)?; + async fn download_chart( + &self, + path: &Path, + cache: &Cache, + expected_size: Option, + expected_sha256_hash: Option, + ) -> Result { + if let Some(expected_sha256_hash) = &expected_sha256_hash { + let key = Key { + name: self.name.clone(), + sha256_hash: expected_sha256_hash.clone(), + }; + + if let Some(file_path) = cache.get_cache_entry(&key).await? { + return self.file_to_chart(file_path, expected_sha256_hash, expected_size); + } } - let sha256_hash = Sha256Hash::from_file_async(tempfile).await?; + let mut temp_file = cache.create_cache_entry(".tar", expected_size).await?; + + { + let file = temp_file.mut_file(); + // Could not get this to flush correctly + // let encoder = GzipEncoder::new(file); + let mut tar = async_tar::Builder::new(file); + tar.append_dir_all(format!("{}-{}", self.name, self.version), path) + .await + .map_err(|e| Error::File(path.to_path_buf(), e))?; + tar.finish() + .await + .map_err(|e| Error::File(path.to_path_buf(), e))?; + }; + + // we must flush the file before calculating the hash + temp_file.flush().await?; - let chart = Meta { + // // If we don't have an expected hash value, just use the real hash + let sha256_hash = match expected_sha256_hash { + Some(expected) => expected, + None => temp_file.calc_sha256_hash().await?, + }; + + let key = Key { name: self.name.clone(), - version: self.version.clone(), - sha256_hash, - created: Some(chrono::Utc::now().fixed_offset()), - repo: RepoSpecific::Local, + sha256_hash: sha256_hash.clone(), }; + let path = temp_file.finalize(&key).await?; + self.file_to_chart(path, &sha256_hash, expected_size) + } - if let Some(entry) = cache.get_cache_entry(&chart).await? { - return Ok(entry); + fn file_to_chart( + &self, + file_path: PathBuf, + sha256_hash: &Sha256Hash, + expected_size: Option, + ) -> Result { + let metadata = file_path + .metadata() + .map_err(|e| Error::File(file_path.clone(), e))?; + let size = metadata.len(); + + if let Some(expected_size) = expected_size { + if expected_size != size { + return Err(Error::SizeMismatch(file_path, expected_size, size)); + } } - let stream = tokio::fs::File::open(tempfile) - .map_ok(|file| FramedRead::new(file, BytesCodec::new()).map_ok(BytesMut::freeze)) - .try_flatten_stream(); + let meta = Meta { + name: self.name.clone(), + version: self.version.clone(), + sha256_hash: sha256_hash.clone(), + created: Some(chrono::Utc::now().fixed_offset()), + repo: RepoSpecific::Helm, + size, + }; - pin!(stream); + let chart = Chart { file_path, meta }; - let cache = cache.create_cache_entry(chart, stream).await?; - Ok(cache) + Ok(chart) } } pub async fn get_by_path(path: &Path, cache: &Cache) -> Result { let config_path = path.join("Chart.yaml"); - let entry: Entry = serde_yml::from_reader(File::open(config_path)?)?; - entry.download_chart(path, cache).await + let file = File::open(config_path.clone()).map_err(|e| Error::File(config_path, e))?; + let entry: Entry = serde_yml::from_reader(file)?; + entry.download_chart(path, cache, None, None).await +} + +pub async fn get_by_meta(path: &Path, cache: &Cache, meta: &Meta) -> Result { + let config_path = path.join("Chart.yaml"); + let file = File::open(config_path.clone()).map_err(|e| Error::File(config_path, e))?; + let entry: Entry = serde_yml::from_reader(file)?; + entry + .download_chart(path, cache, Some(meta.size), Some(meta.sha256_hash.clone())) + .await } #[cfg(test)] @@ -113,7 +164,40 @@ mod tests { assert!(ingress.file_path.exists()); - let hash = Sha256Hash::from_file_async(&ingress.file_path) + let hash = Sha256Hash::from_async_path(&ingress.file_path) + .await + .unwrap(); + assert_eq!(hash, hash); + } + + #[tokio::test] + #[ignore] + async fn test_get_by_meta() { + let expected_hash = + Sha256Hash::new("0a2fa83c8d432594612dde50eadfbd23bb795a0f017815dc8023fb8d9e40d30d"); + + let meta = Meta { + name: "ingress".to_string(), + version: "0.0.1".to_string(), + sha256_hash: expected_hash.clone(), + created: None, + repo: RepoSpecific::Helm, + size: 13312, + }; + + let cache = Cache::new(std::path::Path::new("tmp/local_get_by_meta")); + let ingress = get_by_meta( + Path::new("/home/brian/tree/ea/cloudcell/helm-charts/charts/ingress"), + &cache, + &meta, + ) + .await + .unwrap(); + assert_eq!(ingress.meta.name, "ingress"); + assert_eq!(ingress.meta.version, "0.0.1"); + assert_eq!(ingress.meta.sha256_hash, expected_hash); + + let hash = Sha256Hash::from_async_path(&ingress.file_path) .await .unwrap(); assert_eq!(hash, hash); diff --git a/src/repos/meta.rs b/src/repos/meta.rs index 90b699a..b8ebf59 100644 --- a/src/repos/meta.rs +++ b/src/repos/meta.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -use super::hash::Sha256Hash; +use super::{cache::Key, hash::Sha256Hash}; #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] pub struct Meta { @@ -9,12 +9,22 @@ pub struct Meta { pub sha256_hash: Sha256Hash, pub created: Option>, pub repo: RepoSpecific, + pub size: u64, } #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] #[serde(tag = "type")] pub enum RepoSpecific { Local, - Oci { digest: String }, + Oci { manifest_digest: String }, Helm, } + +impl Meta { + pub(super) fn get_cache_key(&self) -> Key { + Key { + name: self.name.clone(), + sha256_hash: self.sha256_hash.clone(), + } + } +} diff --git a/src/repos/mod.rs b/src/repos/mod.rs index 653e0a4..1c5ec7c 100644 --- a/src/repos/mod.rs +++ b/src/repos/mod.rs @@ -9,6 +9,7 @@ pub mod oci; use std::collections::HashMap; +use charts::Chart; use meta::Meta; use thiserror::Error; use url::Url; @@ -107,16 +108,21 @@ pub async fn download_by_meta( } } - if let Some(chart) = cache - .get_cache_entry(meta) + let key = meta.get_cache_key(); + if let Some(path) = cache + .get_cache_entry(&key) .await .map_err(|e| Error::Cache(chart.clone(), e))? { + let chart = Chart { + meta: meta.clone(), + file_path: path, + }; return Ok(chart); } match chart { - ChartReference::Local { path } => local::get_by_path(path, cache) + ChartReference::Local { path } => local::get_by_meta(path, cache, meta) .await .map_err(|e| Error::Local(path.to_string_lossy().to_string(), e)), ChartReference::Helm { repo_url, .. } => { diff --git a/src/repos/oci.rs b/src/repos/oci.rs index aa3aaae..5739903 100644 --- a/src/repos/oci.rs +++ b/src/repos/oci.rs @@ -1,4 +1,7 @@ +use std::path::PathBuf; + use docker_credential::{CredentialRetrievalError, DockerCredential}; +use futures::StreamExt; use oci_client::{manifest::OciManifest, secrets::RegistryAuth, Client, Reference}; use thiserror::Error; use url::Url; @@ -6,25 +9,23 @@ use url::Url; use crate::repos::{hash::Sha256Hash, meta::RepoSpecific}; use super::{ - cache::{self, Cache}, + cache::{self, Cache, Key}, charts::Chart, meta::Meta, }; #[derive(Error, Debug)] pub enum Error { - #[error("IO error: {0}")] - Io(#[from] std::io::Error), + #[error("File error: {0}")] + File(PathBuf, std::io::Error), + #[error("OCI IO error: {0}")] + OciIo(Box, std::io::Error), #[error("Cache error: {0}")] Cache(#[from] cache::Error), - #[error("Cache streaming error: {0}")] - CacheStream(#[from] cache::StreamError), #[error("Invalid OCI URL")] - InvalidOciUrl, - #[error("OCI parse error: {0}")] - OciParseClient(#[from] oci_client::ParseError), + InvalidOciUrl(Url), #[error("OCI error: {0}")] - OciDistribution(#[from] oci_client::errors::OciDistributionError), + OciDistribution(Box, oci_client::errors::OciDistributionError), #[error("Not an OCI image")] NotAnOciImage, #[error("Invalid JSON")] @@ -35,6 +36,8 @@ pub enum Error { CredentialRetrieval(#[from] CredentialRetrievalError), #[error("Unsupported docker credentials")] UnsupportedDockerCredentials, + #[error("Size mismatch {0}: expected {1}, but got {2}")] + SizeMismatch(PathBuf, u64, u64), } #[allow(dead_code)] @@ -46,10 +49,13 @@ pub struct Repo { impl Repo { pub fn download_index(url: &Url) -> Result { if url.scheme() != "oci" { - return Err(Error::InvalidOciUrl); + return Err(Error::InvalidOciUrl(url.clone())); } - let host = url.host_str().ok_or(Error::InvalidOciUrl)?.to_string(); + let host = url + .host_str() + .ok_or(Error::InvalidOciUrl(url.clone()))? + .to_string(); let path = String::from(url.path()); Ok(Repo { host, path }) @@ -61,12 +67,16 @@ impl Repo { cache: &Cache, name: &str, version: &str, + expected_size: Option, ) -> Result { let auth = build_auth(reference)?; let client_config = build_client_config(); let client = Client::new(client_config); - let (manifest, manifest_digest) = client.pull_manifest(reference, &auth).await?; + let (manifest, manifest_digest) = client + .pull_manifest(reference, &auth) + .await + .map_err(|e| Error::OciDistribution(Box::new(reference.clone()), e))?; let OciManifest::Image(manifest) = manifest else { return Err(Error::NotAnOciImage); @@ -79,45 +89,55 @@ impl Repo { let Some(hash) = layer.digest.strip_prefix("sha256:") else { return Err(Error::NotAnOciImage); }; + let sha256_hash = Sha256Hash::new(hash); let annotations = manifest.annotations; - // Cam't rely on annotations being present - // let Some(name) = annotations.get("org.opencontainers.image.title") else { - // return Err(Error::MissingAnnotation( - // "org.opencontainers.image.title".to_string(), - // )); - // }; - - // Cam't rely on annotations being present - // let Some(version) = annotations.get("org.opencontainers.image.version") else { - // return Err(Error::MissingAnnotation( - // "org.opencontainers.image.version".to_string(), - // )); - // }; - let created = annotations .as_ref() .and_then(|a| a.get("org.opencontainers.image.created")) .and_then(|date| date.parse().ok()); - let chart = Meta { + let key = Key { name: name.to_string(), - version: version.to_string(), - sha256_hash: Sha256Hash::new(hash), - created, - repo: RepoSpecific::Oci { - digest: manifest_digest, - }, + sha256_hash: sha256_hash.clone(), }; - if let Some(entry) = cache.get_cache_entry(&chart).await? { - return Ok(entry); + if let Some(file_path) = cache.get_cache_entry(&key).await? { + return file_to_chart( + file_path, + name, + version, + created, + sha256_hash, + expected_size, + manifest_digest, + ); + } + + let mut file = cache.create_cache_entry(".tgz", expected_size).await?; + + { + let mut stream = client + .pull_blob_stream(reference, &layer) + .await + .map_err(|e| Error::OciDistribution(Box::new(reference.clone()), e))?; + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| Error::OciIo(Box::new(reference.clone()), e))?; + file.write_chunk(&chunk).await?; + } } - let mut stream = client.pull_blob_stream(reference, &layer).await?; - let cache = cache.create_cache_entry(chart, &mut stream).await?; - Ok(cache) + let path = file.finalize(&key).await?; + file_to_chart( + path, + name, + version, + created, + sha256_hash, + expected_size, + manifest_digest, + ) } pub async fn get_by_version( @@ -129,23 +149,66 @@ impl Repo { let path = format!("{}/{}", self.path, name); let reference: Reference = Reference::with_tag(self.host.clone(), path, version.to_string()); - self.get_by_reference(&reference, cache, name, version) + self.get_by_reference(&reference, cache, name, version, None) .await } pub async fn get_by_meta(&self, meta: &Meta, cache: &Cache) -> Result { let path = format!("{}/{}", self.path, meta.name); - let RepoSpecific::Oci { digest } = &meta.repo else { + let RepoSpecific::Oci { + manifest_digest: digest, + } = &meta.repo + else { return Err(Error::NotAnOciImage); }; let reference: Reference = Reference::with_digest(self.host.clone(), path, digest.to_string()); - self.get_by_reference(&reference, cache, &meta.name, &meta.version) - .await + self.get_by_reference( + &reference, + cache, + &meta.name, + &meta.version, + Some(meta.size), + ) + .await } } +fn file_to_chart( + file_path: PathBuf, + name: &str, + version: &str, + created: Option>, + sha256_hash: Sha256Hash, + expected_size: Option, + manifest_digest: String, +) -> Result { + let metadata = file_path + .metadata() + .map_err(|e| Error::File(file_path.clone(), e))?; + let size = metadata.len(); + + if let Some(expected_size) = expected_size { + if expected_size != size { + return Err(Error::SizeMismatch(file_path, expected_size, size)); + } + } + + let meta = Meta { + name: name.to_string(), + version: version.to_string(), + sha256_hash, + created, + repo: RepoSpecific::Oci { manifest_digest }, + size, + }; + + let chart = Chart { file_path, meta }; + + Ok(chart) +} + fn build_auth(reference: &Reference) -> Result { let server = reference .resolve_registry() @@ -201,7 +264,7 @@ mod tests { assert!(karpenter.file_path.exists()); - let hash = Sha256Hash::from_file_async(&karpenter.file_path) + let hash = Sha256Hash::from_async_path(&karpenter.file_path) .await .unwrap(); assert_eq!(hash, expected_hash); @@ -221,9 +284,11 @@ mod tests { sha256_hash: expected_hash.clone(), created: None, repo: RepoSpecific::Oci { - digest: "sha256:98382d6406a3c85711269112fbb337c056d4debabaefb936db2d10137b58bd1b" - .to_string(), + manifest_digest: + "sha256:98382d6406a3c85711269112fbb337c056d4debabaefb936db2d10137b58bd1b" + .to_string(), }, + size: 36211, }; let cache = Cache::new(Path::new("tmp/oci_download_by_meta")); @@ -235,7 +300,7 @@ mod tests { assert!(karpenter.file_path.exists()); - let hash = Sha256Hash::from_file_async(&karpenter.file_path) + let hash = Sha256Hash::from_async_path(&karpenter.file_path) .await .unwrap(); assert_eq!(hash, expected_hash);