From 666c3ddeb93d9680c6e9ae4191dd2b161803af78 Mon Sep 17 00:00:00 2001 From: Jameel Al-Aziz Date: Tue, 19 Mar 2024 01:47:33 -0700 Subject: [PATCH 1/7] fix: Switch to bollard for docker API interaction The `shiplift` library which is currently used to interact with the docker API is unmaintained. Switch to `bollard` which is an actively maintained and fairly popular library. This fixes #537 Signed-off-by: Erdem Meydanli --- Cargo.lock | 630 ++++++++++++++++++++++-------------- enclave_build/Cargo.toml | 4 +- enclave_build/src/docker.rs | 143 ++++---- enclave_build/src/lib.rs | 5 +- src/common/mod.rs | 2 +- tools/Dockerfile | 4 +- 6 files changed, 478 insertions(+), 310 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3699174..ea06b67d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -60,7 +69,7 @@ dependencies = [ "serde_bytes", "serde_cbor", "serde_repr", - "serde_with", + "serde_with 1.12.0", ] [[package]] @@ -85,10 +94,19 @@ dependencies = [ ] [[package]] -name = "base64" -version = "0.13.1" +name = "backtrace" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] [[package]] name = "base64" @@ -140,6 +158,50 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83545367eb6428eb35c29cdec3a1f350fa8d6d9085d59a7d7bcb637f2e38db5a" +dependencies = [ + "base64", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-util", + "hyperlocal-next", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.44.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709d9aa1c37abb89d40f19f5d0ad6f0d88cb1581264e571c9350fc5bb89cf1c5" +dependencies = [ + "serde", + "serde_repr", + "serde_with 3.6.1", +] + [[package]] name = "build_const" version = "0.2.2" @@ -158,12 +220,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - [[package]] name = "bytes" version = "1.1.0" @@ -226,7 +282,7 @@ dependencies = [ "atty", "bitflags 1.3.2", "clap_lex", - "indexmap", + "indexmap 1.8.0", "strsim", "termcolor", "textwrap", @@ -292,9 +348,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -343,6 +399,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "digest" version = "0.9.0" @@ -360,7 +426,7 @@ checksum = "8d0fa3cd8dc96ada974e126a940d37d4079bbbe6a24aca15b1113c2f362441c5" dependencies = [ "cfg-if", "libc", - "socket2 0.5.4", + "socket2", "windows-sys 0.48.0", ] @@ -391,15 +457,17 @@ name = "enclave_build" version = "0.1.0" dependencies = [ "aws-nitro-enclaves-image-format", - "base64 0.21.4", + "base64", + "bollard", "clap", + "flate2", "futures", "log", "serde", "serde_json", "serde_yaml", "sha2", - "shiplift", + "tar", "tempfile", "tokio", "url", @@ -419,24 +487,19 @@ dependencies = [ ] [[package]] -name = "errno" -version = "0.3.0" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.45.0", -] +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "errno" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] @@ -450,25 +513,23 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.15" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.11", - "winapi", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", ] [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ - "cfg-if", "crc32fast", - "libc", "miniz_oxide", ] @@ -607,18 +668,6 @@ dependencies = [ "slab", ] -[[package]] -name = "futures_codec" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" -dependencies = [ - "bytes 0.5.6", - "futures", - "memchr", - "pin-project 0.4.29", -] - [[package]] name = "generic-array" version = "0.14.5" @@ -640,6 +689,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + [[package]] name = "glob" version = "0.3.0" @@ -658,6 +713,12 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -681,23 +742,35 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.6" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes 1.1.0", + "bytes", "fnv", "itoa", ] [[package]] name = "http-body" -version = "0.4.4" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ - "bytes 1.1.0", + "bytes", "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", "pin-project-lite", ] @@ -707,12 +780,6 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" - [[package]] name = "humantime" version = "2.1.0" @@ -721,56 +788,71 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.22" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abfba89e19b959ca163c7752ba59d737c1ceea53a5d31a149c805446fc958064" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" dependencies = [ - "bytes 1.1.0", + "bytes", "futures-channel", - "futures-core", "futures-util", "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] -name = "hyper-openssl" -version = "0.9.2" +name = "hyper-named-pipe" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ - "http", + "hex", "hyper", - "linked_hash_set", - "once_cell", - "openssl", - "openssl-sys", - "parking_lot", + "hyper-util", + "pin-project-lite", "tokio", - "tokio-openssl", - "tower-layer", + "tower-service", + "winapi", ] [[package]] -name = "hyperlocal" -version = "0.8.0" +name = "hyper-util" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ + "bytes", + "futures-channel", "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal-next" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf569d43fa9848e510358c07b80f4adf34084ddc28c6a4a651ee8474c070dcc" +dependencies = [ "hex", + "http-body-util", "hyper", - "pin-project 1.0.10", + "hyper-util", + "pin-project-lite", "tokio", + "tower-service", ] [[package]] @@ -824,7 +906,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.11.2", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", + "serde", ] [[package]] @@ -938,15 +1032,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" -[[package]] -name = "linked_hash_set" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -960,13 +1045,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" [[package]] -name = "lock_api" -version = "0.4.6" +name = "linux-raw-sys" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" -dependencies = [ - "scopeguard", -] +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "log" @@ -1001,12 +1083,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1015,12 +1091,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", - "autocfg", ] [[package]] @@ -1147,6 +1222,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.3.3" @@ -1210,6 +1291,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.10.0" @@ -1282,29 +1372,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "parking_lot" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.2.11", - "smallvec", - "windows-sys 0.32.0", -] - [[package]] name = "peeking_take_while" version = "0.1.2" @@ -1317,33 +1384,13 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" -[[package]] -name = "pin-project" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9615c18d31137579e9ff063499264ddc1278e7b1982757ebc111028c4d1dc909" -dependencies = [ - "pin-project-internal 0.4.29", -] - [[package]] name = "pin-project" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" dependencies = [ - "pin-project-internal 1.0.10", -] - -[[package]] -name = "pin-project-internal" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "044964427019eed9d49d9d5bbce6047ef18f37100ea400912a9fa4a3523ab12a" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "pin-project-internal", ] [[package]] @@ -1359,9 +1406,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1375,6 +1422,12 @@ version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -1441,18 +1494,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.11" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] @@ -1474,6 +1527,12 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1508,6 +1567,19 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", +] + [[package]] name = "rustversion" version = "1.0.6" @@ -1520,12 +1592,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - [[package]] name = "scratch" version = "1.0.5" @@ -1593,6 +1659,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_with" version = "1.12.0" @@ -1603,13 +1681,30 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.8.0", + "indexmap 2.2.5", + "serde", + "serde_derive", + "serde_json", + "time", +] + [[package]] name = "serde_yaml" version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ - "indexmap", + "indexmap 1.8.0", "ryu", "serde", "yaml-rust", @@ -1628,33 +1723,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "shiplift" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e468265908f45299c26571dad9a2e5cb3656eceb51cd58f1441cf61aa71aad6" -dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes 1.1.0", - "chrono", - "flate2", - "futures-util", - "futures_codec", - "hyper", - "hyper-openssl", - "hyperlocal", - "log", - "mime", - "openssl", - "pin-project 1.0.10", - "serde", - "serde_json", - "tar", - "tokio", - "url", -] - [[package]] name = "shlex" version = "1.3.0" @@ -1688,28 +1756,18 @@ checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" [[package]] name = "smallvec" -version = "1.8.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1748,9 +1806,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.38" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" dependencies = [ "filetime", "libc", @@ -1814,6 +1872,37 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "time" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.5.1" @@ -1831,29 +1920,48 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.28.2" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ - "autocfg", + "backtrace", + "bytes", "libc", "mio", "num_cpus", "pin-project-lite", - "socket2 0.4.9", + "socket2", "windows-sys 0.48.0", ] [[package]] -name = "tokio-openssl" -version = "0.6.3" +name = "tokio-util" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", "futures-util", - "openssl", - "openssl-sys", + "pin-project", + "pin-project-lite", "tokio", + "tower-layer", + "tower-service", + "tracing", ] [[package]] @@ -1875,6 +1983,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ "cfg-if", + "log", "pin-project-lite", "tracing-core", ] @@ -2099,19 +2208,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" -dependencies = [ - "windows_aarch64_msvc 0.32.0", - "windows_i686_gnu 0.32.0", - "windows_i686_msvc 0.32.0", - "windows_x86_64_gnu 0.32.0", - "windows_x86_64_msvc 0.32.0", -] - [[package]] name = "windows-sys" version = "0.45.0" @@ -2130,6 +2226,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + [[package]] name = "windows-targets" version = "0.42.1" @@ -2160,6 +2265,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.1" @@ -2173,10 +2293,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "windows_aarch64_msvc" -version = "0.32.0" +name = "windows_aarch64_gnullvm" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -2191,10 +2311,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "windows_i686_gnu" -version = "0.32.0" +name = "windows_aarch64_msvc" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -2209,10 +2329,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "windows_i686_msvc" -version = "0.32.0" +name = "windows_i686_gnu" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -2227,10 +2347,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] -name = "windows_x86_64_gnu" -version = "0.32.0" +name = "windows_i686_msvc" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -2244,6 +2364,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" @@ -2257,10 +2383,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "windows_x86_64_msvc" -version = "0.32.0" +name = "windows_x86_64_gnullvm" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -2274,13 +2400,21 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + [[package]] name = "xattr" -version = "0.2.2" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", + "linux-raw-sys 0.4.13", + "rustix 0.38.31", ] [[package]] diff --git a/enclave_build/Cargo.toml b/enclave_build/Cargo.toml index 9ed9bb5c..e991a49f 100644 --- a/enclave_build/Cargo.toml +++ b/enclave_build/Cargo.toml @@ -8,11 +8,11 @@ rust-version = "1.68" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +bollard = "0.16.0" clap = "3.2" serde = { version = "1.0", features = ["derive"] } serde_yaml = "0.8" serde_json = "1.0" -shiplift = "0.7" tempfile = "3.5.0" tokio = { version = "1.27", features = ["rt-multi-thread"] } base64 = "0.21" @@ -22,3 +22,5 @@ sha2 = "0.9.5" futures = "0.3.28" aws-nitro-enclaves-image-format = "0.2" +tar = "0.4.40" +flate2 = "1.0.28" diff --git a/enclave_build/src/docker.rs b/enclave_build/src/docker.rs index 31436ca6..297afd32 100644 --- a/enclave_build/src/docker.rs +++ b/enclave_build/src/docker.rs @@ -3,11 +3,13 @@ use crate::docker::DockerError::CredentialsError; use base64::{engine::general_purpose, Engine as _}; +use bollard::auth::DockerCredentials; +use bollard::image::{BuildImageOptions, CreateImageOptions}; +use bollard::Docker; +use flate2::{write::GzEncoder, Compression}; use futures::stream::StreamExt; use log::{debug, error, info}; use serde_json::{json, Value}; -use shiplift::RegistryAuth; -use shiplift::{BuildOptions, Docker, PullOptions}; use std::fs::File; use std::io::Write; use std::path::Path; @@ -21,6 +23,7 @@ pub const DOCKER_ARCH_AMD64: &str = "amd64"; #[derive(Debug, PartialEq, Eq)] pub enum DockerError { + ConnectionError, BuildError, InspectError, PullError, @@ -38,20 +41,25 @@ pub struct DockerUtil { impl DockerUtil { /// Constructor that takes as argument a tag for the docker image to be used - pub fn new(docker_image: String) -> Self { + pub fn new(docker_image: String) -> Result { let mut docker_image = docker_image; if !docker_image.contains(':') { docker_image.push_str(":latest"); } - DockerUtil { - // DOCKER_HOST environment variable is parsed inside - // if docker daemon address needs to be substituted. - // By default it tries to connect to 'unix:///var/run/docker.sock' - docker: Docker::new(), + // DOCKER_HOST environment variable is parsed inside + // if docker daemon address needs to be substituted. + // By default, it tries to connect to 'unix:///var/run/docker.sock' + let docker = Docker::connect_with_defaults().map_err(|e| { + error!("{:?}", e); + DockerError::ConnectionError + })?; + + Ok(DockerUtil { + docker, docker_image, - } + }) } /// Returns the credentials by reading ${HOME}/.docker/config.json or ${DOCKER_CONFIG} @@ -60,7 +68,7 @@ impl DockerUtil { /// we are parsing it correctly, so the parsing mechanism had been infered by /// reading a config.json created by: // Docker version 19.03.2 - fn get_credentials(&self) -> Result { + fn get_credentials(&self) -> Result { let image = self.docker_image.clone(); let host = if let Ok(uri) = Url::parse(&image) { uri.host().map(|s| s.to_string()) @@ -109,10 +117,11 @@ impl DockerUtil { if let Some(index) = decoded.rfind(':') { let (user, after_user) = decoded.split_at(index); let (_, password) = after_user.split_at(1); - return Ok(RegistryAuth::builder() - .username(user) - .password(password) - .build()); + return Ok(DockerCredentials { + username: Some(user.to_string()), + password: Some(password.to_string()), + ..Default::default() + }); } } } @@ -160,46 +169,40 @@ impl DockerUtil { let act = async { // Check if the Docker image is locally available. // If available, early exit. - if self - .docker - .images() - .get(&self.docker_image) - .inspect() - .await - .is_ok() - { + if self.docker.inspect_image(&self.docker_image).await.is_ok() { eprintln!("Using the locally available Docker image..."); return Ok(()); } - let mut pull_options_builder = PullOptions::builder(); - pull_options_builder.image(&self.docker_image); + let create_image_options = CreateImageOptions { + from_image: self.docker_image.clone(), + ..Default::default() + }; - match self.get_credentials() { - Ok(auth) => { - pull_options_builder.auth(auth); - } + let credentials = match self.get_credentials() { + Ok(auth) => Some(auth), // It is not mandatory to have the credentials set, but this is // the most likely reason for failure when pulling, so log the // error. Err(err) => { debug!("WARNING!! Credential could not be set {:?}", err); + None } }; - let mut stream = self.docker.images().pull(&pull_options_builder.build()); + let mut stream = + self.docker + .create_image(Some(create_image_options), None, credentials); loop { if let Some(item) = stream.next().await { match item { Ok(output) => { - let msg = &output; - - if let Some(err_msg) = msg.get("error") { - error!("{:?}", err_msg.clone()); + if let Some(err_msg) = &output.error { + error!("{:?}", err_msg); break Err(DockerError::PullError); } else { - info!("{}", msg); + info!("{:?}", output); } } Err(e) => { @@ -221,24 +224,36 @@ impl DockerUtil { /// Build an image locally, with the tag provided in constructor, using a /// directory that contains a Dockerfile pub fn build_image(&self, dockerfile_dir: String) -> Result<(), DockerError> { - let act = async { - let mut stream = self.docker.images().build( - &BuildOptions::builder(dockerfile_dir) - .tag(self.docker_image.clone()) - .build(), + let mut archive = tar::Builder::new(GzEncoder::new(Vec::default(), Compression::best())); + archive.append_dir_all(".", &dockerfile_dir).map_err(|e| { + error!("{:?}", e); + DockerError::BuildError + })?; + let bytes = archive.into_inner().and_then(|c| c.finish()).map_err(|e| { + error!("{:?}", e); + DockerError::BuildError + })?; + + let act = async move { + let mut stream = self.docker.build_image( + BuildImageOptions { + dockerfile: "Dockerfile".to_string(), + t: self.docker_image.clone(), + ..Default::default() + }, + None, + Some(bytes.into()), ); loop { if let Some(item) = stream.next().await { match item { Ok(output) => { - let msg = &output; - - if let Some(err_msg) = msg.get("error") { + if let Some(err_msg) = &output.error { error!("{:?}", err_msg.clone()); break Err(DockerError::BuildError); } else { - info!("{}", msg); + info!("{:?}", output); } } Err(e) => { @@ -260,7 +275,7 @@ impl DockerUtil { /// Inspect docker image and return its description as a json String pub fn inspect_image(&self) -> Result { let act = async { - match self.docker.images().get(&self.docker_image).inspect().await { + match self.docker.inspect_image(&self.docker_image).await { Ok(image) => Ok(json!(image)), Err(e) => { error!("{:?}", e); @@ -276,8 +291,11 @@ impl DockerUtil { fn extract_image(&self) -> Result<(Vec, Vec), DockerError> { // First try to find CMD parameters (together with potential ENV bindings) let act_cmd = async { - match self.docker.images().get(&self.docker_image).inspect().await { - Ok(image) => image.config.cmd.ok_or(DockerError::UnsupportedEntryPoint), + match self.docker.inspect_image(&self.docker_image).await { + Ok(image) => image + .config + .and_then(|c| c.cmd) + .ok_or(DockerError::UnsupportedEntryPoint), Err(e) => { error!("{:?}", e); Err(DockerError::InspectError) @@ -285,8 +303,11 @@ impl DockerUtil { } }; let act_env = async { - match self.docker.images().get(&self.docker_image).inspect().await { - Ok(image) => image.config.env.ok_or(DockerError::UnsupportedEntryPoint), + match self.docker.inspect_image(&self.docker_image).await { + Ok(image) => image + .config + .and_then(|c| c.env) + .ok_or(DockerError::UnsupportedEntryPoint), Err(e) => { error!("{:?}", e); Err(DockerError::InspectError) @@ -304,10 +325,10 @@ impl DockerUtil { // If no CMD instructions are found, try to locate an ENTRYPOINT command if check_cmd_runtime.is_err() || check_env_runtime.is_err() { let act_entrypoint = async { - match self.docker.images().get(&self.docker_image).inspect().await { + match self.docker.inspect_image(&self.docker_image).await { Ok(image) => image .config - .entrypoint + .and_then(|c| c.entrypoint) .ok_or(DockerError::UnsupportedEntryPoint), Err(e) => { error!("{:?}", e); @@ -325,10 +346,15 @@ impl DockerUtil { } let act = async { - match self.docker.images().get(&self.docker_image).inspect().await { + match self.docker.inspect_image(&self.docker_image).await { Ok(image) => Ok(( - image.config.entrypoint.unwrap(), - image.config.env.ok_or_else(Vec::::new).unwrap(), + image.config.clone().unwrap().entrypoint.unwrap(), + image + .config + .unwrap() + .env + .ok_or_else(Vec::::new) + .unwrap(), )), Err(e) => { error!("{:?}", e); @@ -343,8 +369,11 @@ impl DockerUtil { } let act = async { - match self.docker.images().get(&self.docker_image).inspect().await { - Ok(image) => Ok((image.config.cmd.unwrap(), image.config.env.unwrap())), + match self.docker.inspect_image(&self.docker_image).await { + Ok(image) => Ok(( + image.config.clone().unwrap().cmd.unwrap(), + image.config.unwrap().env.unwrap(), + )), Err(e) => { error!("{:?}", e); Err(DockerError::InspectError) @@ -372,8 +401,8 @@ impl DockerUtil { /// Fetch architecture information from an image pub fn architecture(&self) -> Result { let arch = async { - match self.docker.images().get(&self.docker_image).inspect().await { - Ok(image) => Ok(image.architecture), + match self.docker.inspect_image(&self.docker_image).await { + Ok(image) => Ok(image.architecture.unwrap_or_default()), Err(e) => { error!("{:?}", e); Err(DockerError::InspectError) diff --git a/enclave_build/src/lib.rs b/enclave_build/src/lib.rs index d8d41a19..12d9ccf5 100644 --- a/enclave_build/src/lib.rs +++ b/enclave_build/src/lib.rs @@ -74,7 +74,10 @@ impl<'a> Docker2Eif<'a> { metadata_path: Option, build_info: EifBuildInfo, ) -> Result { - let docker = DockerUtil::new(docker_image.clone()); + let docker = DockerUtil::new(docker_image.clone()).map_err(|e| { + eprintln!("Docker error: {e:?}"); + Docker2EifError::DockerError + })?; if !Path::new(&init_path).is_file() { return Err(Docker2EifError::InitPathError); diff --git a/src/common/mod.rs b/src/common/mod.rs index e50099ac..48ac0983 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -220,7 +220,7 @@ pub enum EnclaveProcessReply { } /// Struct that is passed along the backtrace and accumulates error messages. -#[derive(Debug, Default, PartialEq)] +#[derive(Debug, Default, PartialEq, Eq)] pub struct NitroCliFailure { /// Main action which was attempted and failed. pub action: String, diff --git a/tools/Dockerfile b/tools/Dockerfile index 344f2a8c..fed947b3 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -36,8 +36,8 @@ RUN source $HOME/.cargo/env && \ rustup toolchain install ${RUST_VERSION}-${ARCH}-unknown-linux-gnu && \ rustup default ${RUST_VERSION}-${ARCH}-unknown-linux-gnu && \ rustup target add --toolchain ${RUST_VERSION} ${ARCH}-unknown-linux-musl && \ - cargo install cargo-audit --version 0.16.0 --locked && \ - cargo install cargo-about --version 0.5.1 --locked + cargo install cargo-audit --version 0.17.6 --locked && \ + cargo install cargo-about --version 0.5.6 --locked # Install docker for nitro-cli build-enclave runs RUN apt-get update && \ From 8624807d4ac956941f4039682f2a400b4f0d3085 Mon Sep 17 00:00:00 2001 From: Erdem Meydanli Date: Tue, 19 Mar 2024 22:06:02 +0000 Subject: [PATCH 2/7] enclave_build: Add more tests Add additional simple tests to docker.rs for better test coverage. Signed-off-by: Erdem Meydanli --- enclave_build/src/docker.rs | 73 ++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/enclave_build/src/docker.rs b/enclave_build/src/docker.rs index 297afd32..3eafcf12 100644 --- a/enclave_build/src/docker.rs +++ b/enclave_build/src/docker.rs @@ -430,14 +430,15 @@ fn write_config(config: Vec) -> Result { #[cfg(test)] mod tests { use super::*; - use std::io::Read; + use base64::engine::general_purpose; + use std::{env, io::Read}; /// Test extracted configuration is as expected #[test] fn test_config() { let docker = DockerUtil::new(String::from("public.ecr.aws/aws-nitro-enclaves/hello:v1")); - let (cmd_file, env_file) = docker.load().unwrap(); + let (cmd_file, env_file) = docker.unwrap().load().unwrap(); let mut cmd_file = File::open(cmd_file.path()).unwrap(); let mut env_file = File::open(env_file.path()).unwrap(); @@ -453,4 +454,72 @@ mod tests { HELLO=Hello from the enclave side!\n" ); } + + #[test] + fn test_new() { + let docker = DockerUtil::new(String::from("alpine")).unwrap(); + assert_eq!(docker.docker_image, "alpine:latest"); + let docker = DockerUtil::new(String::from("nginx:1.19")).unwrap(); + assert_eq!(docker.docker_image, "nginx:1.19"); + } + + #[test] + fn test_get_credentials() { + let test_user = "test_user"; + let test_password = "test_password"; + let auth = format!("{}:{}", test_user, test_password); + let encoded_auth = general_purpose::STANDARD.encode(auth); + let config = format!( + r#"{{ + "auths": {{ + "https://public.ecr.aws/aws-nitro-enclaves/hello/v1/": {{ + "auth": "{}" + }}, + "https://registry.example.com": {{ + "auth": "b3RoZXJfdXNlcjpvdGhlcl9wYXNzd29yZA==" + }} + }} + }}"#, + encoded_auth + ); + + // Create a temporary file + let mut temp_file = NamedTempFile::new().expect("Failed to create temporary file."); + + // Write the config to the temporary file + write!(temp_file, "{}", config).expect("Failed to write to temporary file."); + + // Set the DOCKER_CONFIG environment variable to point to the temporary file's path + let temp_file_path = temp_file.path().to_string_lossy().to_string(); + env::set_var("DOCKER_CONFIG", temp_file_path); + + let docker = + DockerUtil::new(String::from("public.ecr.aws/aws-nitro-enclaves/hello:v1")).unwrap(); + let creds = docker.get_credentials().unwrap(); + assert_eq!(creds.username, Some(test_user.to_string())); + assert_eq!(creds.password, Some(test_password.to_string())); + + temp_file.close().unwrap(); + } + + #[test] + fn test_architecture() { + #[cfg(target_arch = "x86_64")] + { + let docker = + DockerUtil::new(String::from("public.ecr.aws/aws-nitro-enclaves/hello:v1")) + .unwrap(); + docker.pull_image().unwrap(); + let arch = docker.architecture().unwrap(); + assert_eq!(arch, "amd64"); + } + + #[cfg(target_arch = "aarch64")] + { + let docker = DockerUtil::new(String::from("arm64v8/alpine")).unwrap(); + docker.pull_image().unwrap(); + let arch = docker.architecture().unwrap(); + assert_eq!(arch, "arm64"); + } + } } From bc88a3936c50f71ebfaae4acb47dc1c4dc84d085 Mon Sep 17 00:00:00 2001 From: Erdem Meydanli Date: Thu, 21 Mar 2024 21:03:42 +0000 Subject: [PATCH 3/7] enclave_build: Extract inspect method Extract inspect method in docker.rs to eliminate repetitive code. Signed-off-by: Erdem Meydanli --- enclave_build/src/docker.rs | 155 +++++++++--------------------------- 1 file changed, 38 insertions(+), 117 deletions(-) diff --git a/enclave_build/src/docker.rs b/enclave_build/src/docker.rs index 3eafcf12..4098b17d 100644 --- a/enclave_build/src/docker.rs +++ b/enclave_build/src/docker.rs @@ -5,6 +5,7 @@ use crate::docker::DockerError::CredentialsError; use base64::{engine::general_purpose, Engine as _}; use bollard::auth::DockerCredentials; use bollard::image::{BuildImageOptions, CreateImageOptions}; +use bollard::secret::ImageInspect; use bollard::Docker; use flate2::{write::GzEncoder, Compression}; use futures::stream::StreamExt; @@ -166,14 +167,14 @@ impl DockerUtil { /// Pull the image, with the tag provided in constructor, from the Docker registry pub fn pull_image(&self) -> Result<(), DockerError> { - let act = async { - // Check if the Docker image is locally available. - // If available, early exit. - if self.docker.inspect_image(&self.docker_image).await.is_ok() { - eprintln!("Using the locally available Docker image..."); - return Ok(()); - } + // Check if the Docker image is locally available. + // If available, early exit. + if self.inspect().is_ok() { + eprintln!("Using the locally available Docker image..."); + return Ok(()); + } + let act = async { let create_image_options = CreateImageOptions { from_image: self.docker_image.clone(), ..Default::default() @@ -272,118 +273,49 @@ impl DockerUtil { runtime.block_on(act) } - /// Inspect docker image and return its description as a json String - pub fn inspect_image(&self) -> Result { - let act = async { - match self.docker.inspect_image(&self.docker_image).await { - Ok(image) => Ok(json!(image)), + fn inspect(&self) -> Result { + let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; + let image_future = self.docker.inspect_image(&self.docker_image); + + runtime.block_on(async { + match image_future.await { + Ok(image) => Ok(image), Err(e) => { error!("{:?}", e); Err(DockerError::InspectError) } } - }; + }) + } - let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; - runtime.block_on(act) + /// Inspect docker image and return its description as a json String + pub fn inspect_image(&self) -> Result { + match self.inspect() { + Ok(image) => Ok(json!(image)), + Err(e) => { + error!("{:?}", e); + Err(DockerError::InspectError) + } + } } fn extract_image(&self) -> Result<(Vec, Vec), DockerError> { // First try to find CMD parameters (together with potential ENV bindings) - let act_cmd = async { - match self.docker.inspect_image(&self.docker_image).await { - Ok(image) => image - .config - .and_then(|c| c.cmd) - .ok_or(DockerError::UnsupportedEntryPoint), - Err(e) => { - error!("{:?}", e); - Err(DockerError::InspectError) - } - } - }; - let act_env = async { - match self.docker.inspect_image(&self.docker_image).await { - Ok(image) => image - .config - .and_then(|c| c.env) - .ok_or(DockerError::UnsupportedEntryPoint), - Err(e) => { - error!("{:?}", e); - Err(DockerError::InspectError) - } - } - }; + let image = self.inspect()?; + let config = image.config.ok_or(DockerError::UnsupportedEntryPoint)?; - let check_cmd_runtime = Runtime::new() - .map_err(|_| DockerError::RuntimeError)? - .block_on(act_cmd); - let check_env_runtime = Runtime::new() - .map_err(|_| DockerError::RuntimeError)? - .block_on(act_env); + if let Some(cmd) = &config.cmd { + let env = config.env.unwrap_or_default(); + return Ok((cmd.clone(), env)); + } // If no CMD instructions are found, try to locate an ENTRYPOINT command - if check_cmd_runtime.is_err() || check_env_runtime.is_err() { - let act_entrypoint = async { - match self.docker.inspect_image(&self.docker_image).await { - Ok(image) => image - .config - .and_then(|c| c.entrypoint) - .ok_or(DockerError::UnsupportedEntryPoint), - Err(e) => { - error!("{:?}", e); - Err(DockerError::InspectError) - } - } - }; - - let check_entrypoint_runtime = Runtime::new() - .map_err(|_| DockerError::RuntimeError)? - .block_on(act_entrypoint); - - if check_entrypoint_runtime.is_err() { - return Err(DockerError::UnsupportedEntryPoint); - } - - let act = async { - match self.docker.inspect_image(&self.docker_image).await { - Ok(image) => Ok(( - image.config.clone().unwrap().entrypoint.unwrap(), - image - .config - .unwrap() - .env - .ok_or_else(Vec::::new) - .unwrap(), - )), - Err(e) => { - error!("{:?}", e); - Err(DockerError::InspectError) - } - } - }; - - let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; - - return runtime.block_on(act); + if let Some(entrypoint) = &config.entrypoint { + let env = config.env.unwrap_or_default(); + return Ok((entrypoint.clone(), env)); } - let act = async { - match self.docker.inspect_image(&self.docker_image).await { - Ok(image) => Ok(( - image.config.clone().unwrap().cmd.unwrap(), - image.config.unwrap().env.unwrap(), - )), - Err(e) => { - error!("{:?}", e); - Err(DockerError::InspectError) - } - } - }; - - let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; - - runtime.block_on(act) + Err(DockerError::UnsupportedEntryPoint) } /// The main function of this struct. This needs to be called in order to @@ -400,19 +332,8 @@ impl DockerUtil { /// Fetch architecture information from an image pub fn architecture(&self) -> Result { - let arch = async { - match self.docker.inspect_image(&self.docker_image).await { - Ok(image) => Ok(image.architecture.unwrap_or_default()), - Err(e) => { - error!("{:?}", e); - Err(DockerError::InspectError) - } - } - }; - - let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; - - runtime.block_on(arch) + let image = self.inspect()?; + Ok(image.architecture.unwrap_or_default()) } } From 949e22e4cd350823b253d4d69a16f20d88c9305c Mon Sep 17 00:00:00 2001 From: Erdem Meydanli Date: Thu, 21 Mar 2024 21:13:52 +0000 Subject: [PATCH 4/7] enclave_build: Extract parse_docker_host method Extract parse_docker_host method from get_credentials for reusability. Signed-off-by: Erdem Meydanli --- enclave_build/src/docker.rs | 95 +++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/enclave_build/src/docker.rs b/enclave_build/src/docker.rs index 4098b17d..dd225abc 100644 --- a/enclave_build/src/docker.rs +++ b/enclave_build/src/docker.rs @@ -63,67 +63,70 @@ impl DockerUtil { }) } - /// Returns the credentials by reading ${HOME}/.docker/config.json or ${DOCKER_CONFIG} - /// - /// config.json doesn't seem to have a schema that we could use to validate - /// we are parsing it correctly, so the parsing mechanism had been infered by - /// reading a config.json created by: - // Docker version 19.03.2 - fn get_credentials(&self) -> Result { - let image = self.docker_image.clone(); - let host = if let Ok(uri) = Url::parse(&image) { + fn parse_docker_host(docker_image: &str) -> Option { + if let Ok(uri) = Url::parse(docker_image) { uri.host().map(|s| s.to_string()) } else { // Some Docker URIs don't have the protocol included, so just use // a dummy one to trick Url that it's a properly defined Uri. - let uri = format!("dummy://{image}"); + let uri = format!("dummy://{docker_image}"); if let Ok(uri) = Url::parse(&uri) { uri.host().map(|s| s.to_string()) } else { None } + } + } + /// Returns the credentials by reading ${HOME}/.docker/config.json or ${DOCKER_CONFIG} + /// + /// config.json doesn't seem to have a schema that we could use to validate + /// we are parsing it correctly, so the parsing mechanism had been infered by + /// reading a config.json created by: + // Docker version 19.03.2 + fn get_credentials(&self) -> Result { + let host = match Self::parse_docker_host(&self.docker_image) { + Some(host) => host, + None => return Err(CredentialsError("Invalid docker image URI!".to_string())), }; - if let Some(registry_domain) = host { - let config_file = self.get_config_file()?; - - let config_json: serde_json::Value = serde_json::from_reader(&config_file) - .map_err(|err| CredentialsError(format!("JSON was not well-formatted: {err}")))?; - - let auths = config_json.get("auths").ok_or_else(|| { - CredentialsError("Could not find auths key in config JSON".to_string()) - })?; + let config_file = self.get_config_file()?; - if let Value::Object(auths) = auths { - for (registry_name, registry_auths) in auths.iter() { - if !registry_name.to_string().contains(®istry_domain) { - continue; - } + let config_json: serde_json::Value = serde_json::from_reader(&config_file) + .map_err(|err| CredentialsError(format!("JSON was not well-formatted: {err}")))?; - let auth = registry_auths - .get("auth") - .ok_or_else(|| { - CredentialsError("Could not find auth key in config JSON".to_string()) - })? - .to_string(); + let auths = config_json.get("auths").ok_or_else(|| { + CredentialsError("Could not find auths key in config JSON".to_string()) + })?; - let auth = auth.replace('"', ""); - let decoded = general_purpose::STANDARD.decode(auth).map_err(|err| { - CredentialsError(format!("Invalid Base64 encoding for auth: {err}")) - })?; - let decoded = std::str::from_utf8(&decoded).map_err(|err| { - CredentialsError(format!("Invalid utf8 encoding for auth: {err}")) - })?; + if let Value::Object(auths) = auths { + for (registry_name, registry_auths) in auths.iter() { + if !registry_name.to_string().contains(&host) { + continue; + } - if let Some(index) = decoded.rfind(':') { - let (user, after_user) = decoded.split_at(index); - let (_, password) = after_user.split_at(1); - return Ok(DockerCredentials { - username: Some(user.to_string()), - password: Some(password.to_string()), - ..Default::default() - }); - } + let auth = registry_auths + .get("auth") + .ok_or_else(|| { + CredentialsError("Could not find auth key in config JSON".to_string()) + })? + .to_string(); + + let auth = auth.replace('"', ""); + let decoded = general_purpose::STANDARD.decode(auth).map_err(|err| { + CredentialsError(format!("Invalid Base64 encoding for auth: {err}")) + })?; + let decoded = std::str::from_utf8(&decoded).map_err(|err| { + CredentialsError(format!("Invalid utf8 encoding for auth: {err}")) + })?; + + if let Some(index) = decoded.rfind(':') { + let (user, after_user) = decoded.split_at(index); + let (_, password) = after_user.split_at(1); + return Ok(DockerCredentials { + username: Some(user.to_string()), + password: Some(password.to_string()), + ..Default::default() + }); } } } From 7b0bbdde4956d3cc029b637dd8ff6361d8119537 Mon Sep 17 00:00:00 2001 From: Erdem Meydanli Date: Thu, 21 Mar 2024 21:20:26 +0000 Subject: [PATCH 5/7] enclave_build: Extract build_tarball method Extract build_tarball method from build_image in docker.rs for reusability. Signed-off-by: Erdem Meydanli --- enclave_build/src/docker.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/enclave_build/src/docker.rs b/enclave_build/src/docker.rs index dd225abc..0598e342 100644 --- a/enclave_build/src/docker.rs +++ b/enclave_build/src/docker.rs @@ -225,19 +225,24 @@ impl DockerUtil { runtime.block_on(act) } - /// Build an image locally, with the tag provided in constructor, using a - /// directory that contains a Dockerfile - pub fn build_image(&self, dockerfile_dir: String) -> Result<(), DockerError> { - let mut archive = tar::Builder::new(GzEncoder::new(Vec::default(), Compression::best())); + fn build_tarball(dockerfile_dir: String) -> Result, DockerError> { + let encoder = GzEncoder::new(Vec::default(), Compression::best()); + let mut archive = tar::Builder::new(encoder); + archive.append_dir_all(".", &dockerfile_dir).map_err(|e| { error!("{:?}", e); DockerError::BuildError })?; - let bytes = archive.into_inner().and_then(|c| c.finish()).map_err(|e| { + + archive.into_inner().and_then(|c| c.finish()).map_err(|e| { error!("{:?}", e); DockerError::BuildError - })?; + }) + } + /// Build an image locally, with the tag provided in constructor, using a + /// directory that contains a Dockerfile + pub fn build_image(&self, dockerfile_dir: String) -> Result<(), DockerError> { let act = async move { let mut stream = self.docker.build_image( BuildImageOptions { @@ -246,7 +251,7 @@ impl DockerUtil { ..Default::default() }, None, - Some(bytes.into()), + Some(Self::build_tarball(dockerfile_dir)?.into()), ); loop { From d7504553bd5e59933950be7f4efdb397ad9b4cc3 Mon Sep 17 00:00:00 2001 From: Erdem Meydanli Date: Thu, 21 Mar 2024 21:26:52 +0000 Subject: [PATCH 6/7] enclave_build: Refactor docker.rs for consistent Runtime creation Create Runtime instances at the beginning of method blocks in docker.rs for consistency. Signed-off-by: Erdem Meydanli --- enclave_build/src/docker.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/enclave_build/src/docker.rs b/enclave_build/src/docker.rs index 0598e342..55358738 100644 --- a/enclave_build/src/docker.rs +++ b/enclave_build/src/docker.rs @@ -177,7 +177,9 @@ impl DockerUtil { return Ok(()); } - let act = async { + let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; + + runtime.block_on(async { let create_image_options = CreateImageOptions { from_image: self.docker_image.clone(), ..Default::default() @@ -218,11 +220,7 @@ impl DockerUtil { break Ok(()); } } - }; - - let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; - - runtime.block_on(act) + }) } fn build_tarball(dockerfile_dir: String) -> Result, DockerError> { @@ -243,7 +241,9 @@ impl DockerUtil { /// Build an image locally, with the tag provided in constructor, using a /// directory that contains a Dockerfile pub fn build_image(&self, dockerfile_dir: String) -> Result<(), DockerError> { - let act = async move { + let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; + + runtime.block_on(async move { let mut stream = self.docker.build_image( BuildImageOptions { dockerfile: "Dockerfile".to_string(), @@ -274,11 +274,7 @@ impl DockerUtil { break Ok(()); } } - }; - - let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; - - runtime.block_on(act) + }) } fn inspect(&self) -> Result { From 66ff6fd2b6ca1645afe90799e6e9580c7d3fec4f Mon Sep 17 00:00:00 2001 From: Erdem Meydanli Date: Thu, 21 Mar 2024 22:44:34 +0000 Subject: [PATCH 7/7] enclave_build: Extract stream output handling Extract stream output handling code into handle_stream_output function and move to utils.rs Signed-off-by: Erdem Meydanli --- enclave_build/src/docker.rs | 56 ++++++------------------------------- enclave_build/src/lib.rs | 1 + enclave_build/src/utils.rs | 53 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 47 deletions(-) create mode 100644 enclave_build/src/utils.rs diff --git a/enclave_build/src/docker.rs b/enclave_build/src/docker.rs index 55358738..df3fbbfc 100644 --- a/enclave_build/src/docker.rs +++ b/enclave_build/src/docker.rs @@ -2,14 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 use crate::docker::DockerError::CredentialsError; +use crate::utils::handle_stream_output; use base64::{engine::general_purpose, Engine as _}; use bollard::auth::DockerCredentials; use bollard::image::{BuildImageOptions, CreateImageOptions}; use bollard::secret::ImageInspect; use bollard::Docker; use flate2::{write::GzEncoder, Compression}; -use futures::stream::StreamExt; -use log::{debug, error, info}; +use log::{debug, error}; use serde_json::{json, Value}; use std::fs::File; use std::io::Write; @@ -196,30 +196,11 @@ impl DockerUtil { } }; - let mut stream = - self.docker - .create_image(Some(create_image_options), None, credentials); - - loop { - if let Some(item) = stream.next().await { - match item { - Ok(output) => { - if let Some(err_msg) = &output.error { - error!("{:?}", err_msg); - break Err(DockerError::PullError); - } else { - info!("{:?}", output); - } - } - Err(e) => { - error!("{:?}", e); - break Err(DockerError::PullError); - } - } - } else { - break Ok(()); - } - } + let stream = self + .docker + .create_image(Some(create_image_options), None, credentials); + + handle_stream_output(stream, DockerError::PullError).await }) } @@ -244,7 +225,7 @@ impl DockerUtil { let runtime = Runtime::new().map_err(|_| DockerError::RuntimeError)?; runtime.block_on(async move { - let mut stream = self.docker.build_image( + let stream = self.docker.build_image( BuildImageOptions { dockerfile: "Dockerfile".to_string(), t: self.docker_image.clone(), @@ -254,26 +235,7 @@ impl DockerUtil { Some(Self::build_tarball(dockerfile_dir)?.into()), ); - loop { - if let Some(item) = stream.next().await { - match item { - Ok(output) => { - if let Some(err_msg) = &output.error { - error!("{:?}", err_msg.clone()); - break Err(DockerError::BuildError); - } else { - info!("{:?}", output); - } - } - Err(e) => { - error!("{:?}", e); - break Err(DockerError::BuildError); - } - } - } else { - break Ok(()); - } - } + handle_stream_output(stream, DockerError::BuildError).await }) } diff --git a/enclave_build/src/lib.rs b/enclave_build/src/lib.rs index 12d9ccf5..66069b48 100644 --- a/enclave_build/src/lib.rs +++ b/enclave_build/src/lib.rs @@ -7,6 +7,7 @@ use std::path::Path; use std::process::Command; mod docker; +mod utils; mod yaml_generator; use aws_nitro_enclaves_image_format::defs::{EifBuildInfo, EifIdentityInfo, EIF_HDR_ARCH_ARM64}; diff --git a/enclave_build/src/utils.rs b/enclave_build/src/utils.rs new file mode 100644 index 00000000..530ab7bb --- /dev/null +++ b/enclave_build/src/utils.rs @@ -0,0 +1,53 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use bollard::errors::Error; +use bollard::secret::{BuildInfo, CreateImageInfo}; +use futures::stream::StreamExt; +use futures::Stream; +use log::{error, info}; + +pub trait StreamItem { + fn error(&self) -> Option; +} + +// Implement StreamItem for CreateImageInfo +impl StreamItem for CreateImageInfo { + fn error(&self) -> Option { + self.error.clone() + } +} + +// Implement StreamItem for BuildInfo +impl StreamItem for BuildInfo { + fn error(&self) -> Option { + self.error.clone() + } +} + +pub async fn handle_stream_output( + mut stream: impl Stream> + Unpin, + error_type: U, +) -> Result<(), U> +where + T: StreamItem + std::fmt::Debug, +{ + while let Some(item) = stream.next().await { + match item { + Ok(output) => { + if let Some(err_msg) = output.error() { + error!("{:?}", err_msg); + return Err(error_type); + } else { + info!("{:?}", output); + } + } + Err(e) => { + error!("{:?}", e); + return Err(error_type); + } + } + } + + Ok(()) +}