From e801a958610d70c8d5823c9cda23daafab6ef326 Mon Sep 17 00:00:00 2001 From: edgar Date: Wed, 25 Dec 2024 23:25:53 +0100 Subject: [PATCH 1/3] implement hessian response --- crates/kornia-imgproc/src/features/mod.rs | 2 + .../kornia-imgproc/src/features/responses.rs | 63 +++++++++++++++ crates/kornia-imgproc/src/lib.rs | 3 + examples/features/Cargo.toml | 12 +++ examples/features/README.md | 21 +++++ examples/features/src/main.rs | 78 +++++++++++++++++++ 6 files changed, 179 insertions(+) create mode 100644 crates/kornia-imgproc/src/features/mod.rs create mode 100644 crates/kornia-imgproc/src/features/responses.rs create mode 100644 examples/features/Cargo.toml create mode 100644 examples/features/README.md create mode 100644 examples/features/src/main.rs diff --git a/crates/kornia-imgproc/src/features/mod.rs b/crates/kornia-imgproc/src/features/mod.rs new file mode 100644 index 00000000..5ffed59f --- /dev/null +++ b/crates/kornia-imgproc/src/features/mod.rs @@ -0,0 +1,2 @@ +mod responses; +pub use responses::*; diff --git a/crates/kornia-imgproc/src/features/responses.rs b/crates/kornia-imgproc/src/features/responses.rs new file mode 100644 index 00000000..2f5fa6a8 --- /dev/null +++ b/crates/kornia-imgproc/src/features/responses.rs @@ -0,0 +1,63 @@ +use kornia_image::{Image, ImageError}; + +use rayon::prelude::*; + +/// Compute the Hessian response of an image. +pub fn hessian_response(src: &Image, dst: &mut Image) -> Result<(), ImageError> { + if src.size() != dst.size() { + return Err(ImageError::InvalidImageSize( + src.cols(), + src.rows(), + dst.cols(), + dst.rows(), + )); + } + + let src_data = src.as_slice(); + + dst.as_slice_mut() + .par_chunks_exact_mut(src.cols()) + .enumerate() + .for_each(|(row_idx, row_chunk)| { + if row_idx == 0 || row_idx == src.rows() - 1 { + // skip the first and last row + return; + } + + let row_offset = row_idx * src.cols(); + + row_chunk + .iter_mut() + .enumerate() + .for_each(|(col_idx, dst_pixel)| { + if col_idx == 0 || col_idx == src.cols() - 1 { + // skip the first and last column + return; + } + + let current_idx = row_offset + col_idx; + let prev_row_idx = current_idx - src.cols(); + let next_row_idx = current_idx + src.cols(); + + let v11 = src_data[prev_row_idx - 1]; + let v12 = src_data[prev_row_idx]; + let v13 = src_data[prev_row_idx + 1]; + let v21 = src_data[current_idx - 1]; + let v22 = src_data[current_idx]; + let v23 = src_data[current_idx + 1]; + let v31 = src_data[next_row_idx - 1]; + let v32 = src_data[next_row_idx]; + let v33 = src_data[next_row_idx + 1]; + + let dxx = v21 - 2.0 * v22 + v23; + let dyy = v12 - 2.0 * v22 + v32; + let dxy = 0.25 * (v31 - v11 - v33 + v13); + + let det = dxx * dyy - dxy * dxy; + + *dst_pixel = det; + }); + }); + + Ok(()) +} diff --git a/crates/kornia-imgproc/src/lib.rs b/crates/kornia-imgproc/src/lib.rs index 8cf8d5eb..919c1c14 100644 --- a/crates/kornia-imgproc/src/lib.rs +++ b/crates/kornia-imgproc/src/lib.rs @@ -22,6 +22,9 @@ pub mod draw; /// image enhancement module. pub mod enhance; +/// feature detection module. +pub mod features; + /// image flipping module. pub mod flip; diff --git a/examples/features/Cargo.toml b/examples/features/Cargo.toml new file mode 100644 index 00000000..98f1dca1 --- /dev/null +++ b/examples/features/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "features" +version = "0.1.0" +authors = ["Edgar Riba "] +license = "Apache-2.0" +edition = "2021" +publish = false + +[dependencies] +ctrlc = "3.4.4" +kornia = { workspace = true, features = ["gstreamer"] } +rerun = { workspace = true } diff --git a/examples/features/README.md b/examples/features/README.md new file mode 100644 index 00000000..89b95027 --- /dev/null +++ b/examples/features/README.md @@ -0,0 +1,21 @@ +An example showing how to use the webcam with the `kornia_io` module with the ability to cancel the feed after a certain amount of time. This example will display the webcam feed in a [`rerun`](https://github.com/rerun-io/rerun) window. + +NOTE: This example requires the gstremer backend to be enabled. To enable the gstreamer backend, use the `gstreamer` feature flag when building the `kornia` crate and its dependencies. + +```bash +Usage: webcam [OPTIONS] + +Options: + -c, --camera-id [default: 0] + -f, --fps [default: 30] + -d, --duration + -h, --help Print help +``` + +Example: + +```bash +cargo run --bin webcam --release -- --camera-id 0 --duration 5 --fps 30 +``` + +![Screenshot from 2024-08-28 18-33-56](https://github.com/user-attachments/assets/783619e4-4867-48bc-b7d2-d32a133e4f5a) diff --git a/examples/features/src/main.rs b/examples/features/src/main.rs new file mode 100644 index 00000000..14e7f1ba --- /dev/null +++ b/examples/features/src/main.rs @@ -0,0 +1,78 @@ +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +use kornia::{ + image::{ops, Image}, + imgproc, + io::stream::V4L2CameraConfig, +}; + +fn main() -> Result<(), Box> { + // start the recording stream + let rec = rerun::RecordingStreamBuilder::new("Kornia Webcapture App").spawn()?; + + // image size + let size = [640, 480].into(); + + // create a webcam capture object with camera id 0 + // and force the image size to 640x480 + let mut webcam = V4L2CameraConfig::new().with_size(size).build()?; + + // start the background pipeline + webcam.start()?; + + // create a cancel token to stop the webcam capture + let cancel_token = Arc::new(AtomicBool::new(false)); + + ctrlc::set_handler({ + let cancel_token = cancel_token.clone(); + move || { + println!("Received Ctrl-C signal. Sending cancel signal !!"); + cancel_token.store(true, Ordering::SeqCst); + } + })?; + + // preallocate images + let mut img_f32 = Image::from_size_val(size, 0f32)?; + let mut gray = Image::from_size_val(size, 0f32)?; + let mut hessian = Image::from_size_val(size, 0f32)?; + + // start grabbing frames from the camera + while !cancel_token.load(Ordering::SeqCst) { + let Some(img) = webcam.grab()? else { + continue; + }; + + // convert to grayscale + ops::cast_and_scale(&img, &mut img_f32, 1. / 255.)?; + imgproc::color::gray_from_rgb(&img_f32, &mut gray)?; + + // compute the hessian response + imgproc::features::hessian_response(&gray, &mut hessian)?; + + // log the image + rec.log_static( + "image", + &rerun::Image::from_elements(img.as_slice(), img.size().into(), rerun::ColorModel::RGB), + )?; + + // log the hessian response + rec.log_static( + "hessian", + &rerun::Image::from_elements( + hessian.as_slice(), + hessian.size().into(), + rerun::ColorModel::L, + ), + )?; + } + + // NOTE: this is important to close the webcam properly, otherwise the app will hang + webcam.close()?; + + println!("Finished recording. Closing app."); + + Ok(()) +} From 3b66852379610d8131b47bcebaa12a0c2886f299 Mon Sep 17 00:00:00 2001 From: edgar Date: Thu, 26 Dec 2024 12:20:35 +0100 Subject: [PATCH 2/3] improve example --- .../{features/responses.rs => features.rs} | 42 +++++++++++++++++++ crates/kornia-imgproc/src/features/mod.rs | 2 - examples/features/README.md | 18 +------- examples/features/src/main.rs | 12 ++++-- 4 files changed, 52 insertions(+), 22 deletions(-) rename crates/kornia-imgproc/src/{features/responses.rs => features.rs} (64%) delete mode 100644 crates/kornia-imgproc/src/features/mod.rs diff --git a/crates/kornia-imgproc/src/features/responses.rs b/crates/kornia-imgproc/src/features.rs similarity index 64% rename from crates/kornia-imgproc/src/features/responses.rs rename to crates/kornia-imgproc/src/features.rs index 2f5fa6a8..6aaa3645 100644 --- a/crates/kornia-imgproc/src/features/responses.rs +++ b/crates/kornia-imgproc/src/features.rs @@ -3,6 +3,12 @@ use kornia_image::{Image, ImageError}; use rayon::prelude::*; /// Compute the Hessian response of an image. +/// +/// The Hessian response is computed as the absolute value of the determinant of the Hessian matrix. +/// +/// Args: +/// src: The source image with shape (H, W). +/// dst: The destination image with shape (H, W). pub fn hessian_response(src: &Image, dst: &mut Image) -> Result<(), ImageError> { if src.size() != dst.size() { return Err(ImageError::InvalidImageSize( @@ -61,3 +67,39 @@ pub fn hessian_response(src: &Image, dst: &mut Image) -> Result< Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hessian_response() -> Result<(), ImageError> { + #[rustfmt::skip] + let src = Image::from_size_slice( + [5, 5].into(), + &[ + 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 1.0, 1.0, 1.0, 0.0, + 0.0, 1.0, 0.0, 1.0, 0.0, + 0.0, 1.0, 1.0, 1.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, + ], + )?; + + let mut dst = Image::from_size_val([5, 5].into(), 0.0)?; + hessian_response(&src, &mut dst)?; + + #[rustfmt::skip] + assert_eq!( + dst.as_slice(), + &[ + 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 1.0, 0.0, 1.0, 0.0, + 0.0, 0.0, 4.0, 0.0, 0.0, + 0.0, 1.0, 0.0, 1.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, + ] + ); + Ok(()) + } +} diff --git a/crates/kornia-imgproc/src/features/mod.rs b/crates/kornia-imgproc/src/features/mod.rs deleted file mode 100644 index 5ffed59f..00000000 --- a/crates/kornia-imgproc/src/features/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod responses; -pub use responses::*; diff --git a/examples/features/README.md b/examples/features/README.md index 89b95027..e73c5ca2 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -1,21 +1,7 @@ -An example showing how to use the webcam with the `kornia_io` module with the ability to cancel the feed after a certain amount of time. This example will display the webcam feed in a [`rerun`](https://github.com/rerun-io/rerun) window. - -NOTE: This example requires the gstremer backend to be enabled. To enable the gstreamer backend, use the `gstreamer` feature flag when building the `kornia` crate and its dependencies. - -```bash -Usage: webcam [OPTIONS] - -Options: - -c, --camera-id [default: 0] - -f, --fps [default: 30] - -d, --duration - -h, --help Print help -``` +An example showing how to compute the corners of an image using the Hessian response. Example: ```bash -cargo run --bin webcam --release -- --camera-id 0 --duration 5 --fps 30 +cargo run --bin features --release ``` - -![Screenshot from 2024-08-28 18-33-56](https://github.com/user-attachments/assets/783619e4-4867-48bc-b7d2-d32a133e4f5a) diff --git a/examples/features/src/main.rs b/examples/features/src/main.rs index 14e7f1ba..2e13e709 100644 --- a/examples/features/src/main.rs +++ b/examples/features/src/main.rs @@ -38,6 +38,7 @@ fn main() -> Result<(), Box> { let mut img_f32 = Image::from_size_val(size, 0f32)?; let mut gray = Image::from_size_val(size, 0f32)?; let mut hessian = Image::from_size_val(size, 0f32)?; + let mut corners = Image::from_size_val(size, 0f32)?; // start grabbing frames from the camera while !cancel_token.load(Ordering::SeqCst) { @@ -52,18 +53,21 @@ fn main() -> Result<(), Box> { // compute the hessian response imgproc::features::hessian_response(&gray, &mut hessian)?; + // compute the corners + imgproc::threshold::threshold_binary(&hessian, &mut corners, 0.01, 1.0)?; + // log the image rec.log_static( "image", &rerun::Image::from_elements(img.as_slice(), img.size().into(), rerun::ColorModel::RGB), )?; - // log the hessian response + // log the corners rec.log_static( - "hessian", + "corners", &rerun::Image::from_elements( - hessian.as_slice(), - hessian.size().into(), + corners.as_slice(), + corners.size().into(), rerun::ColorModel::L, ), )?; From 6954a3ad5358a575e6a73a6e5763b9dad5c378a5 Mon Sep 17 00:00:00 2001 From: Edgar Riba Date: Thu, 26 Dec 2024 12:24:55 +0100 Subject: [PATCH 3/3] add picture in the readme --- examples/features/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/features/README.md b/examples/features/README.md index e73c5ca2..e9cb8414 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -5,3 +5,5 @@ Example: ```bash cargo run --bin features --release ``` + +![Screenshot from 2024-12-26 12-23-18](https://github.com/user-attachments/assets/aab6d989-41a8-4ad8-ac5f-857eb3ae76a0)