From cbd57e31989d96c44d3a81d8293d0b8707adafe3 Mon Sep 17 00:00:00 2001 From: "Worker Pants (Pantsbuild GitHub Automation Bot)" <133242086+WorkerPants@users.noreply.github.com> Date: Mon, 24 Feb 2025 19:22:01 -0800 Subject: [PATCH] feat(ghac): Bump opendal to support ghac v2 (Cherry-pick of #21994) (#21997) Github is planning to sunset the legacy services before March 1st. This PR updates OpenDAL to version 0.52.0, which automatically supports GHAC v2. The ghac v2 requires using azblob to upload data internally, so we need to add some extra dependencies in this PR. Perhaps we can consider enabling both s3 and azblob support in the future since all dependencies have already been included. Refer to https://github.com/apache/opendal/issues/5620 for more details. Signed-off-by: Xuanwo Co-authored-by: Xuanwo Co-authored-by: Huon Wilson --- docs/notes/2.25.x.md | 2 +- src/rust/engine/Cargo.lock | 77 +++++++++++++++++-- src/rust/engine/Cargo.toml | 2 +- .../src/action_cache_tests.rs | 2 +- .../remote_provider_opendal/src/lib.rs | 2 +- 5 files changed, 76 insertions(+), 9 deletions(-) diff --git a/docs/notes/2.25.x.md b/docs/notes/2.25.x.md index 4beebd6810f..a45f61c0c8b 100644 --- a/docs/notes/2.25.x.md +++ b/docs/notes/2.25.x.md @@ -32,7 +32,7 @@ one in `remote_store_headers` or `remote_execution_headers`. Pants now supports the `{chroot}` replacement marker in remote execution contexts. (With local and Docker execution, the `{chroot}` marker is replaced with the absolute path of the sandbox directory if it appears in program arguments or environment variables. Pants will do the same as well in remote execution contexts. This requires `/bin/bash` to be available on the remote execution server.) -The OpenDAL library powering the Github Actions cache backend has been updated, picking up some bug fixes for Github Enterprise Server instances using AWS S3 as backing storage for the Github Actions cache. +The OpenDAL library powering the Github Actions cache backend has been updated, picking up support for the new GitHub Actions cache v2, and some bug fixes for Github Enterprise Server instances using AWS S3 as backing storage for the Github Actions cache. ### New Options System diff --git a/src/rust/engine/Cargo.lock b/src/rust/engine/Cargo.lock index 6ec5529d14d..01daa69c397 100644 --- a/src/rust/engine/Cargo.lock +++ b/src/rust/engine/Cargo.lock @@ -583,9 +583,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -593,7 +593,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.0", + "windows-targets 0.52.5", ] [[package]] @@ -744,6 +744,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" version = "0.9.1" @@ -993,7 +999,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -1510,6 +1518,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghac" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10bd5b898cac1a4de4a882a754b2ccaafead449348cfb420b48cd5c00ffd08b" +dependencies = [ + "prost", +] + [[package]] name = "gimli" version = "0.27.3" @@ -1712,6 +1729,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -2520,9 +2546,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opendal" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9dcfa7a3615e3c60eb662ed6b46b6f244cf2658098f593c0c0915430b3a268" +checksum = "a55c840b5a6ad96106d6c0612fabb8f35a5ace826e0474fc55ebda33042b8d33" dependencies = [ "anyhow", "async-trait", @@ -2532,15 +2558,19 @@ dependencies = [ "chrono", "futures", "getrandom 0.2.8", + "ghac", "http", "log", "md-5", "once_cell", "percent-encoding", + "prost", "quick-xml", + "reqsign", "reqwest", "serde", "serde_json", + "sha2", "tokio", "uuid", ] @@ -3426,6 +3456,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "reqsign" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb0075a66c8bfbf4cc8b70dca166e722e1f55a3ea9250ecbb85f4d92a5f64149" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "chrono", + "form_urlencoded", + "getrandom 0.2.8", + "hex", + "hmac", + "home", + "http", + "log", + "percent-encoding", + "rand", + "reqwest", + "serde", + "serde_json", + "sha1", + "sha2", +] + [[package]] name = "reqwest" version = "0.12.5" @@ -3834,6 +3890,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.7" diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml index c9cc0294575..539d793ccc4 100644 --- a/src/rust/engine/Cargo.toml +++ b/src/rust/engine/Cargo.toml @@ -275,7 +275,7 @@ notify = { git = "https://github.com/pantsbuild/notify", rev = "276af0f3c5f300bf num_cpus = "1" num_enum = "0.5" once_cell = "1.20" -opendal = { version = "0.51.1", default-features = false, features = [ +opendal = { version = "0.52.0", default-features = false, features = [ "services-memory", "services-fs", "services-ghac", diff --git a/src/rust/engine/remote_provider/remote_provider_opendal/src/action_cache_tests.rs b/src/rust/engine/remote_provider/remote_provider_opendal/src/action_cache_tests.rs index 7d2560237c7..27933b80db8 100644 --- a/src/rust/engine/remote_provider/remote_provider_opendal/src/action_cache_tests.rs +++ b/src/rust/engine/remote_provider/remote_provider_opendal/src/action_cache_tests.rs @@ -51,7 +51,7 @@ async fn write_test_data(provider: &Provider, digest: Digest, data: remexec::Act .operator .write(&test_path(digest), data.to_bytes()) .await - .unwrap() + .unwrap(); } #[tokio::test] diff --git a/src/rust/engine/remote_provider/remote_provider_opendal/src/lib.rs b/src/rust/engine/remote_provider/remote_provider_opendal/src/lib.rs index d7a978504c8..a67fa75d671 100644 --- a/src/rust/engine/remote_provider/remote_provider_opendal/src/lib.rs +++ b/src/rust/engine/remote_provider/remote_provider_opendal/src/lib.rs @@ -211,7 +211,7 @@ impl ByteStoreProvider for Provider { let path = self.path(digest.hash); match self.operator.write(&path, bytes).await { - Ok(()) => Ok(()), + Ok(_) => Ok(()), // The item already exists, i.e. these bytes have already been stored. For example, // concurrent executions that are caching the same bytes. This makes the assumption that // which ever execution won the race to create the item successfully finishes the write, and