From dbcb6e16bc7c538d8b6fe1d22ce584a331391afb Mon Sep 17 00:00:00 2001 From: kjuulh Date: Thu, 2 Mar 2023 22:22:59 +0100 Subject: [PATCH] feat: working git events storage --- Cargo.lock | 200 ++++++++++++++++-- crates/gitevents_sdk/Cargo.toml | 1 + .../examples/generic_repo/main.rs | 44 ++++ crates/gitevents_sdk/src/git/generic.rs | 103 ++++++++- 4 files changed, 325 insertions(+), 23 deletions(-) create mode 100644 crates/gitevents_sdk/examples/generic_repo/main.rs diff --git a/Cargo.lock b/Cargo.lock index 51ec8ac..9489652 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,6 +51,9 @@ name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -151,6 +154,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.26" @@ -251,6 +263,21 @@ dependencies = [ "wasi", ] +[[package]] +name = "git2" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "gitevents_sdk" version = "0.1.0" @@ -258,6 +285,7 @@ dependencies = [ "async-trait", "eyre", "futures", + "git2", "tokio", "tokio-cron-scheduler", "tracing", @@ -299,6 +327,16 @@ dependencies = [ "cxx-build", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indenter" version = "0.3.3" @@ -311,6 +349,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.61" @@ -332,6 +379,46 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +[[package]] +name = "libgit2-sys" +version = "0.14.2+1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f3d95f6b51075fe9810a7ae22c7095f12b98005ab364d8544797a825ce946a4" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b094a36eb4b8b8c8a7b4b8ae43b2944502be3e59cd87687595cf6b0a71b3f4ca" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "link-cplusplus" version = "1.0.8" @@ -390,7 +477,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -459,6 +546,35 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "111.25.1+1.1.1t" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ef9a9cc6ea7d9d5e7c4a913dc4b48d0e359eddf01af1dfec96ba7064b4aba10" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +dependencies = [ + "autocfg", + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" @@ -485,9 +601,15 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-sys", ] +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -500,6 +622,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + [[package]] name = "proc-macro2" version = "1.0.51" @@ -659,11 +787,26 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -676,7 +819,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -804,18 +947,44 @@ dependencies = [ "syn", ] +[[package]] +name = "unicode-bidi" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" + [[package]] name = "unicode-ident" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-width" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "uuid" version = "1.3.0" @@ -831,6 +1000,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -922,21 +1097,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - [[package]] name = "windows-sys" version = "0.45.0" diff --git a/crates/gitevents_sdk/Cargo.toml b/crates/gitevents_sdk/Cargo.toml index 6c1640f..c4b85fc 100644 --- a/crates/gitevents_sdk/Cargo.toml +++ b/crates/gitevents_sdk/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" async-trait = "0.1.64" eyre = "0.6.8" futures = "0.3.26" +git2 = { version = "0.16.1", features = ["vendored-libgit2", "vendored-openssl"] } tokio = { version = "1.25.0", features = ["full"] } tokio-cron-scheduler = { version = "0.9.4", features = ["signal"] } tracing = { version = "0.1.37", features = ["log", "async-await"] } diff --git a/crates/gitevents_sdk/examples/generic_repo/main.rs b/crates/gitevents_sdk/examples/generic_repo/main.rs new file mode 100644 index 0000000..2cfe508 --- /dev/null +++ b/crates/gitevents_sdk/examples/generic_repo/main.rs @@ -0,0 +1,44 @@ +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use gitevents_sdk::{ + cron::SchedulerOpts, + events::{EventHandler, EventRequest, EventResponse}, +}; +use tracing::Level; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + tracing_subscriber::fmt() + .pretty() + .with_max_level(Level::TRACE) + .init(); + + gitevents_sdk::builder::Builder::new() + .set_generic_git_url("kjuulh/gitevents") + .set_scheduler_opts(&SchedulerOpts { + // Duration must not be lower than 1 second, otherwise async runtime won't proceed + duration: Duration::from_secs(1), + }) + .action(|_req| async move { Ok(EventResponse {}) }) + .action(other_action) + .add_handler(Arc::new(TestHandler {})) + .execute() + .await?; + + Ok(()) +} + +async fn other_action(_req: EventRequest) -> eyre::Result { + Ok(EventResponse {}) +} + +pub struct TestHandler; + +#[async_trait] +impl EventHandler for TestHandler { + async fn handle(&self, _req: EventRequest) -> eyre::Result { + Ok(EventResponse {}) + } +} diff --git a/crates/gitevents_sdk/src/git/generic.rs b/crates/gitevents_sdk/src/git/generic.rs index 816ff64..6c6c8d4 100644 --- a/crates/gitevents_sdk/src/git/generic.rs +++ b/crates/gitevents_sdk/src/git/generic.rs @@ -2,7 +2,9 @@ use std::process::Stdio; use std::sync::Arc; use async_trait::async_trait; +use git2::Repository; use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::sync::Mutex; use crate::storage::volatile::VolatileStorage; use crate::storage::DynStorage; @@ -12,6 +14,7 @@ use super::{GitEvent, GitProvider}; pub struct GitGeneric { url: String, storage: DynStorage, + progress: Mutex>, } impl GitGeneric { @@ -19,6 +22,7 @@ impl GitGeneric { Self { url: url.into(), storage: Arc::new(VolatileStorage::new()), + progress: Mutex::new(None), } } } @@ -27,8 +31,68 @@ impl GitGeneric { impl GitProvider for GitGeneric { async fn listen(&mut self) -> eyre::Result> { match self.storage.exists().await? { - Some(_path) => { - todo!("update repo"); + Some(path) => { + let mut cmd = tokio::process::Command::new("git") + .args(&["pull"]) + .current_dir(&path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let stdout = cmd + .stdout + .take() + .ok_or(eyre::anyhow!("failed to capture stdout of cmd"))?; + let stderr = cmd + .stderr + .take() + .ok_or(eyre::anyhow!("failed to capture stdout of cmd"))?; + let mut reader = BufReader::new(stdout).lines(); + let mut errreader = BufReader::new(stderr).lines(); + + tokio::spawn(async move { + while let Ok(Some(line)) = reader.next_line().await { + tracing::debug!(line = line, "out: git pull"); + } + }); + + tokio::spawn(async move { + while let Ok(Some(line)) = errreader.next_line().await { + tracing::debug!(line = line, "err: git pull"); + } + }); + + let status = cmd.wait().await?.to_string(); + tracing::debug!(status = status, "git pull finished"); + + let mut p = self.progress.lock().await; + match p.as_mut() { + Some(p) => { + let repo = Repository::open(path)?; + let head = repo.head()?.target().unwrap(); + let mut revwalk = repo.revwalk()?; + revwalk.set_sorting(git2::Sort::NONE)?; //| git2::Sort::REVERSE)?; + let start = git2::Oid::from_str(p)?; + revwalk.hide(start)?; + revwalk.push(head)?; + + if let Some(rev) = revwalk.next() { + let revstr = rev?.to_string(); + tracing::trace!(progress = &revstr, "storing progress"); + dbg!(&revstr); + *p = revstr; + + return Ok(Some(GitEvent {})); + } + } + None => { + eyre::bail!( + "inconsistency found, object should not already have progress stored" + ); + } + } + + Ok(None) } None => { let path = self.storage.allocate().await?; @@ -70,7 +134,21 @@ impl GitProvider for GitGeneric { let status = cmd.wait().await?.to_string(); tracing::debug!(status = status, "git clone finished"); - //TODO: Store progress + let mut p = self.progress.lock().await; + match p.as_mut() { + Some(_) => eyre::bail!( + "inconsistency found, object should not already have progress stored" + ), + None => { + let repo = Repository::open(path)?; + let head = repo.head()?; + let revstr = head.target().unwrap().to_string(); + tracing::trace!(progress = &revstr, "storing progress"); + *p = Some(revstr); + } + } + + drop(p); Ok(None) } @@ -103,6 +181,12 @@ mod tests { git_commit_all(&tempdir, "initial file").await.unwrap(); + let mut file_path2 = tempdir.clone(); + file_path2.push("readme2.md"); + write(file_path2, "Some file").unwrap(); + + git_commit_all(&tempdir, "next commit").await.unwrap(); + let mut git = GitGeneric::new(tempdir.to_str().unwrap()); let event = git.listen().await.unwrap(); @@ -110,6 +194,19 @@ mod tests { assert!(logs_contain("git clone finished")); assert!(logs_contain("err: git clone")); + let mut file_path3 = tempdir.clone(); + file_path3.push("readme3.md"); + write(file_path3, "Some file").unwrap(); + + git_commit_all(&tempdir, "next commit 3").await.unwrap(); + + let event = git.listen().await.unwrap(); + + assert!(event.is_some()); + assert!(logs_contain("git pull finished")); + assert!(logs_contain("err: git pull")); + assert!(logs_contain("storing progress123")); + remove_dir_all(tempdir).await.unwrap(); }