Extract shm stats code into common crate

Also change log level to INFO, as it's really useful for performance tuning, and would get lost in DEBUG messages...
LiberTEM · Mar 22, 2023 · ef773ae · ef773ae
1 parent 5e529a8
commit ef773ae
Show file tree

Hide file tree

Showing 11 changed files with 166 additions and 130 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,6 +5,7 @@ members = [
     "ipc_test",
     "playegui",
     "bs-sys",
+    "stats",
 ]
 
 [profile.release]

diff --git a/libertem_asi_tpx3/Cargo.toml b/libertem_asi_tpx3/Cargo.toml
@@ -2,7 +2,7 @@
 name = "libertem-asi-tpx3"
 authors = ["Alexander Clausen <[email protected]>"]
 license = "MIT"
-version = "0.2.6"
+version = "0.2.7"
 edition = "2021"
 readme = "README.md"
 
@@ -22,6 +22,7 @@ pyo3 = { version = "0.17", features = ["abi3-py37"] }
 serde = { version = "1.0.143", features = ["derive"] }
 uuid = { version = "1.1.2", features = ["v4", "fast-rng"] }
 ipc-test = { path = "../ipc_test" }
+stats = { path = "../stats" }
 zerocopy = "0.6.1"
 
 [dev-dependencies]

diff --git a/libertem_asi_tpx3/src/main_py.rs b/libertem_asi_tpx3/src/main_py.rs
@@ -12,7 +12,6 @@ use crate::{
     exceptions::{ConnectionError, TimeoutError},
     headers::AcquisitionStart,
     receiver::{ReceiverStatus, ResultMsg, TPXReceiver},
-    stats::Stats,
 };
 
 use ipc_test::SharedSlabAllocator;
@@ -21,6 +20,7 @@ use pyo3::{
     exceptions::{self, PyRuntimeError},
     prelude::*,
 };
+use stats::Stats;
 
 #[pymodule]
 fn libertem_asi_tpx3(py: Python, m: &PyModule) -> PyResult<()> {
@@ -146,6 +146,7 @@ impl<'a, 'b, 'c, 'd> ChunkIterator<'a, 'b, 'c, 'd> {
                 }
                 Some(ResultMsg::End { frame_stack }) => {
                     self.stats.log_stats();
+                    self.stats.reset();
                     return Ok(Some(frame_stack));
                 }
                 Some(ResultMsg::FrameStack { frame_stack }) => {
@@ -302,6 +303,7 @@ impl ASITpx3Connection {
     // for example `wait_for_arm`, which calls `allow_threads`...
     fn close(&mut self) {
         self.stats.log_stats();
+        self.stats.reset();
         self.close_impl();
     }
 
@@ -314,7 +316,7 @@ impl ASITpx3Connection {
         )?;
         iter.get_next_stack_impl(py, max_size).map(|maybe_stack| {
             if let Some(frame_stack) = &maybe_stack {
-                self.stats.count_chunk_stack(frame_stack);
+                self.stats.count_stats_item(frame_stack);
             }
             maybe_stack
         })
@@ -323,6 +325,7 @@ impl ASITpx3Connection {
     fn log_shm_stats(&self) {
         let free = self.local_shm.num_slots_free();
         let total = self.local_shm.num_slots_total();
+        self.stats.log_stats();
         info!("shm stats free/total: {}/{}", free, total);
     }
 }
diff --git a/libertem_asi_tpx3/src/stats.rs b/libertem_asi_tpx3/src/stats.rs
@@ -1,74 +1,31 @@
-use log::debug;
+use stats::GetStats;
 
 use crate::chunk_stack::ChunkStackHandle;
 
-pub struct Stats {
-    /// total number of bytes (compressed) that have flown through the system
-    payload_size_sum: usize,
-
-    /// maximum size of compressed frames seen
-    frame_size_max: usize,
-
-    /// minimum size of compressed frames seen
-    frame_size_min: usize,
-
-    /// sum of the size of the slots used
-    slots_size_sum: usize,
-
-    /// number of frames seen
-    num_frames: usize,
-
-    /// number of times a frame stack was split
-    split_count: usize,
-}
-
-impl Stats {
-    pub fn new() -> Self {
-        Self {
-            payload_size_sum: 0,
-            slots_size_sum: 0,
-            frame_size_max: 0,
-            frame_size_min: usize::MAX,
-            num_frames: 0,
-            split_count: 0,
-        }
+impl GetStats for ChunkStackHandle {
+    fn payload_size(&self) -> usize {
+        self.payload_size()
     }
 
-    pub fn count_chunk_stack(&mut self, chunk_stack: &ChunkStackHandle) {
-        self.payload_size_sum += chunk_stack.payload_size();
-        self.slots_size_sum += chunk_stack.slot_size();
-        self.frame_size_max = self.frame_size_max.max(
-            chunk_stack
-                .get_layout()
-                .iter()
-                .max_by_key(|fm| fm.data_length_bytes)
-                .map_or(self.frame_size_max, |fm| fm.data_length_bytes),
-        );
-        self.frame_size_min = self.frame_size_min.min(
-            chunk_stack
-                .get_layout()
-                .iter()
-                .min_by_key(|fm| fm.data_length_bytes)
-                .map_or(self.frame_size_min, |fm| fm.data_length_bytes),
-        );
-        self.num_frames += chunk_stack.len() as usize;
+    fn slot_size(&self) -> usize {
+        self.slot_size()
     }
 
-    pub fn count_split(&mut self) {
-        self.split_count += 1;
+    fn max_frame_size(&self, old_max: usize) -> usize {
+        self.get_layout()
+            .iter()
+            .max_by_key(|fm| fm.data_length_bytes)
+            .map_or(old_max, |fm| fm.data_length_bytes)
     }
 
-    pub fn log_stats(&self) {
-        let efficiency = self.payload_size_sum as f32 / self.slots_size_sum as f32;
-        debug!(
-            "Stats: frames seen: {}, total payload size: {}, total slot size used: {}, min chunk size: {}, max chunk size: {}, splits: {}, shm efficiency: {}",
-            self.num_frames, self.payload_size_sum, self.slots_size_sum, self.frame_size_min, self.frame_size_max, self.split_count, efficiency,
-        );
+    fn min_frame_size(&self, old_min: usize) -> usize {
+        self.get_layout()
+            .iter()
+            .min_by_key(|fm| fm.data_length_bytes)
+            .map_or(old_min, |fm| fm.data_length_bytes)
     }
-}
 
-impl Default for Stats {
-    fn default() -> Self {
-        Self::new()
+    fn num_frames(&self) -> usize {
+        self.len() as usize
     }
 }
diff --git a/libertem_dectris/Cargo.toml b/libertem_dectris/Cargo.toml
@@ -2,7 +2,7 @@
 name = "libertem-dectris"
 authors = ["Alexander Clausen <[email protected]>"]
 license = "MIT"
-version = "0.2.6"
+version = "0.2.7"
 edition = "2021"
 readme = "README.md"
 
@@ -32,6 +32,7 @@ spin_sleep = "1.1.1"
 uuid = { version = "1.1.2", features = ["v4", "fast-rng"] }
 zmq = { version = "0.10.0", features = [] }
 ipc-test = { path = "../ipc_test" }
+stats = { path = "../stats" }
 nix = "0.26.1"
 lz4 = "1.24.0"
 zerocopy = "0.6.1"

diff --git a/libertem_dectris/README.md b/libertem_dectris/README.md
@@ -81,6 +81,11 @@ finally:
 
 ## Changelog
 
+### v0.2.7
+
+- Log more details in `DectrisConnection.log_shm_stats` and change log level to
+  `INFO`
+
 ### v0.2.4 - v0.2.6
 
 - Updated examples and CI configuration

diff --git a/libertem_dectris/src/dectris_py.rs b/libertem_dectris/src/dectris_py.rs
@@ -15,7 +15,6 @@ use crate::{
     frame_stack::FrameStackHandle,
     receiver::{DectrisReceiver, ReceiverStatus, ResultMsg},
     sim::DectrisSim,
-    stats::Stats,
 };
 
 use ipc_test::SharedSlabAllocator;
@@ -24,6 +23,7 @@ use pyo3::{
     exceptions::{self, PyRuntimeError},
     prelude::*,
 };
+use stats::Stats;
 
 #[pymodule]
 fn libertem_dectris(py: Python, m: &PyModule) -> PyResult<()> {
@@ -150,6 +150,7 @@ impl<'a, 'b, 'c, 'd> FrameChunkedIterator<'a, 'b, 'c, 'd> {
                 }
                 Some(ResultMsg::End { frame_stack }) => {
                     self.stats.log_stats();
+                    self.stats.reset();
                     return Ok(Some(frame_stack));
                 }
                 Some(ResultMsg::FrameStack { frame_stack }) => {
@@ -304,6 +305,7 @@ impl DectrisConnection {
 
     fn close(mut slf: PyRefMut<Self>) {
         slf.stats.log_stats();
+        slf.stats.reset();
         slf.close_impl();
     }
 
@@ -320,7 +322,7 @@ impl DectrisConnection {
         )?;
         iter.get_next_stack_impl(py, max_size).map(|maybe_stack| {
             if let Some(frame_stack) = &maybe_stack {
-                self.stats.count_frame_stack(frame_stack);
+                self.stats.count_stats_item(frame_stack);
             }
             maybe_stack
         })
@@ -329,6 +331,7 @@ impl DectrisConnection {
     fn log_shm_stats(&self) {
         let free = self.local_shm.num_slots_free();
         let total = self.local_shm.num_slots_total();
+        self.stats.log_stats();
         info!("shm stats free/total: {}/{}", free, total);
     }
 }
diff --git a/libertem_dectris/src/stats.rs b/libertem_dectris/src/stats.rs
@@ -1,74 +1,31 @@
-use log::debug;
+use stats::GetStats;
 
 use crate::frame_stack::FrameStackHandle;
 
-pub struct Stats {
-    /// total number of bytes (compressed) that have flown through the system
-    payload_size_sum: usize,
-
-    /// maximum size of compressed frames seen
-    frame_size_max: usize,
-
-    /// minimum size of compressed frames seen
-    frame_size_min: usize,
-
-    /// sum of the size of the slots used
-    slots_size_sum: usize,
-
-    /// number of frames seen
-    num_frames: usize,
-
-    /// number of times a frame stack was split
-    split_count: usize,
-}
-
-impl Stats {
-    pub fn new() -> Self {
-        Self {
-            payload_size_sum: 0,
-            slots_size_sum: 0,
-            frame_size_max: 0,
-            frame_size_min: usize::MAX,
-            num_frames: 0,
-            split_count: 0,
-        }
+impl GetStats for FrameStackHandle {
+    fn payload_size(&self) -> usize {
+        self.payload_size()
     }
 
-    pub fn count_frame_stack(&mut self, frame_stack: &FrameStackHandle) {
-        self.payload_size_sum += frame_stack.payload_size();
-        self.slots_size_sum += frame_stack.slot_size();
-        self.frame_size_max = self.frame_size_max.max(
-            frame_stack
-                .get_meta()
-                .iter()
-                .max_by_key(|fm| fm.data_length_bytes)
-                .map_or(self.frame_size_max, |fm| fm.data_length_bytes),
-        );
-        self.frame_size_min = self.frame_size_min.min(
-            frame_stack
-                .get_meta()
-                .iter()
-                .min_by_key(|fm| fm.data_length_bytes)
-                .map_or(self.frame_size_min, |fm| fm.data_length_bytes),
-        );
-        self.num_frames += frame_stack.len();
+    fn slot_size(&self) -> usize {
+        self.slot_size()
     }
 
-    pub fn count_split(&mut self) {
-        self.split_count += 1;
+    fn max_frame_size(&self, old_max: usize) -> usize {
+        self.get_meta()
+            .iter()
+            .max_by_key(|fm| fm.data_length_bytes)
+            .map_or(old_max, |fm| fm.data_length_bytes)
     }
 
-    pub fn log_stats(&self) {
-        let efficiency = self.payload_size_sum as f32 / self.slots_size_sum as f32;
-        debug!(
-            "Stats: frames seen: {}, total payload size: {}, total slot size used: {}, min frame size: {}, max frame size: {}, splits: {}, shm efficiency: {}",
-            self.num_frames, self.payload_size_sum, self.slots_size_sum, self.frame_size_min, self.frame_size_max, self.split_count, efficiency,
-        );
+    fn min_frame_size(&self, old_min: usize) -> usize {
+        self.get_meta()
+            .iter()
+            .min_by_key(|fm| fm.data_length_bytes)
+            .map_or(old_min, |fm| fm.data_length_bytes)
     }
-}
 
-impl Default for Stats {
-    fn default() -> Self {
-        Self::new()
+    fn num_frames(&self) -> usize {
+        self.len()
     }
 }
diff --git a/stats/Cargo.toml b/stats/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+name = "stats"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4.17"