Merge pull request #2869 from Ruadhri17/apt-plugin-exclude-pattern

feat(sm-plugin): filter packages output list with exclude/include pattern

crates/common/tedge_config/src/tedge_config_cli/tedge_config.rs

@@ -751,7 +751,15 @@ define_tedge_config! {
 
             /// The maximum number of software packages reported for each type of software package
             #[tedge_config(example = "1000", default(value = 1000u32))]
-            max_packages: u32
+            max_packages: u32,
+
+            /// The filtering criterion, in form of regex, that is used to filter packages list output
+            #[tedge_config(example = "^(tedge|c8y).*")]
+            include: String,
+
+            /// The filtering criterion, in form of regex, that is used to filter out packages from the output list
+            #[tedge_config(example = "^(glibc|lib|kernel-|iptables-module).*")]
+            exclude: String,
         }
     },
 
@@ -1160,6 +1168,8 @@ mod tests {
     #[test_case::test_case("mqtt.external.certfile")]
     #[test_case::test_case("mqtt.external.keyfile")]
     #[test_case::test_case("software.plugin.default")]
+    #[test_case::test_case("software.plugin.exclude")]
+    #[test_case::test_case("software.plugin.include")]
     #[test_case::test_case("software.plugin.max_packages")]
     #[test_case::test_case("tmp.path")]
     #[test_case::test_case("logs.path")]

crates/core/plugin_sm/src/plugin.rs

@@ -2,6 +2,7 @@ use async_trait::async_trait;
 use csv::ReaderBuilder;
 use download::Downloader;
 use logged_command::LoggedCommand;
+use regex::Regex;
 use reqwest::Identity;
 use serde::Deserialize;
 use std::error::Error;
@@ -239,6 +240,8 @@ pub struct ExternalPluginCommand {
     pub path: PathBuf,
     pub sudo: SudoCommandBuilder,
     pub max_packages: u32,
+    exclude: Option<String>,
+    include: Option<String>,
     identity: Option<Identity>,
 }
 
@@ -248,13 +251,17 @@ impl ExternalPluginCommand {
         path: impl Into<PathBuf>,
         sudo: SudoCommandBuilder,
         max_packages: u32,
+        exclude: Option<String>,
+        include: Option<String>,
         identity: Option<Identity>,
     ) -> ExternalPluginCommand {
         ExternalPluginCommand {
             name: name.into(),
             path: path.into(),
             sudo,
             max_packages,
+            exclude,
+            include,
             identity,
         }
     }
@@ -461,13 +468,37 @@ impl Plugin for ExternalPluginCommand {
         let command = self.command(LIST, None)?;
         let output = self.execute(command, logger).await?;
         if output.status.success() {
+            let filtered_output = match (&self.exclude, &self.include) {
+                (None, None) => output.stdout,
+                _ => {
+                    // If no exclude pattern is given, exclude everything (except what matches the include pattern)
+                    let exclude_filter =
+                        Regex::new(self.exclude.as_ref().unwrap_or(&r".*".to_string()))?;
+                    // If no include pattern is given, include nothing (except what doesn't match the exclude pattern)
+                    let include_filter =
+                        Regex::new(self.include.as_ref().unwrap_or(&r"^$".to_string()))?;
+
+                    output
+                        .stdout
+                        .split_inclusive(|c| *c == b'\n')
+                        .filter_map(|line| std::str::from_utf8(line).ok())
+                        .filter(|line| {
+                            line.split_once('\t').is_some_and(|(name, _)| {
+                                include_filter.is_match(name) || !exclude_filter.is_match(name)
+                            })
+                        })
+                        .flat_map(|line| line.as_bytes().to_vec())
+                        .collect()
+                }
+            };
+
             // If max_packages is set to an invalid value, use 0 which represents all
             // of the content, don't bother filtering the content when all of it will
             // be included anyway
             let max_packages = usize::try_from(self.max_packages).unwrap_or(0);
             let last_char = match max_packages {
                 0 => 0,
-                _ => String::from_utf8(output.stdout.as_slice().to_vec())
+                _ => String::from_utf8(filtered_output.as_slice().to_vec())
                     .unwrap_or_default()
                     .char_indices()
                     .filter(|(_, c)| *c == '\n')
@@ -479,8 +510,8 @@ impl Plugin for ExternalPluginCommand {
             Ok(deserialize_module_info(
                 self.name.clone(),
                 match last_char {
-                    0 => &output.stdout[..],
-                    _ => &output.stdout[..=last_char],
+                    0 => &filtered_output[..],
+                    _ => &filtered_output[..=last_char],
                 },
             )?)
         } else {

crates/core/plugin_sm/src/plugin_manager.rs

@@ -207,6 +207,8 @@ impl ExternalPlugins {
                             &path,
                             self.sudo.clone(),
                             config.software.plugin.max_packages,
+                            config.software.plugin.exclude.or_none().cloned(),
+                            config.software.plugin.include.or_none().cloned(),
                             identity,
                         );
                         self.plugin_map.insert(plugin_name.into(), plugin);

crates/core/plugin_sm/tests/plugin.rs

@@ -67,6 +67,8 @@ mod tests {
             &dummy_plugin_path,
             SudoCommandBuilder::enabled(false),
             config.software.plugin.max_packages,
+            None,
+            None,
             config.http.client.auth.identity()?,
         );
         assert_eq!(plugin.name, "test");
@@ -86,6 +88,8 @@ mod tests {
             SudoCommandBuilder::enabled(false),
             100,
             None,
+            None,
+            None,
         );
 
         let module = SoftwareModule {
@@ -116,6 +120,8 @@ mod tests {
             SudoCommandBuilder::enabled(false),
             100,
             None,
+            None,
+            None,
         );
 
         // Create test module with name `test2`.
@@ -152,6 +158,8 @@ mod tests {
             SudoCommandBuilder::enabled(false),
             100,
             None,
+            None,
+            None,
         );
 
         // Create software module without an explicit type.

crates/core/tedge_api/Cargo.toml

@@ -16,6 +16,7 @@ download = { workspace = true }
 json-writer = { workspace = true }
 log = { workspace = true }
 mqtt_channel = { workspace = true }
+regex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 shell-words = { workspace = true }
@@ -35,7 +36,6 @@ assert_matches = { workspace = true }
 clock = { workspace = true }
 maplit = { workspace = true }
 mockall = { workspace = true }
-regex = { workspace = true }
 tempfile = { workspace = true }
 test-case = { workspace = true }
 time = { workspace = true, features = ["macros"] }

crates/core/tedge_api/src/error.rs

@@ -100,6 +100,9 @@ pub enum SoftwareError {
 
     #[error("CSV error: {reason:?}")]
     FromCSV { reason: String },
+
+    #[error("Regex error: {reason:?}")]
+    RegexError { reason: String },
 }
 
 impl From<serde_json::Error> for SoftwareError {
@@ -125,3 +128,11 @@ impl From<csv::Error> for SoftwareError {
         }
     }
 }
+
+impl From<regex::Error> for SoftwareError {
+    fn from(err: regex::Error) -> Self {
+        SoftwareError::RegexError {
+            reason: format!("{}", err),
+        }
+    }
+}

docs/src/references/agent/software-management.md

@@ -367,8 +367,14 @@ For each type of software package supported on the device must be provided a spe
 
 `tedge-agent` behavior on `software_update` commands can be configured with `tedge config`.
 
-- `software.plugin.default` set the default software plugin to be used for software management on the device. 
-- `software.plugin.max_packages` set the maximum number of software packages reported for each type of software package.
+- `software.plugin.default` sets the default software plugin to be used for software management on the device. 
+- `software.plugin.max_packages` sets the maximum number of software packages reported for each type of software package.
+- `software.plugin.exclude` sets the filtering criterion that excludes software packages from the output list if they match the pattern.
+- `software.plugin.include` sets the filtering criterion that includes software packages in the output list if they match the pattern.
+
+:::info
+Include pattern takes precedence over exclude pattern, so when both are used at the same time, the software list will exclude packages according to the pattern but keep the exceptions covered by the include pattern.
+:::
 
 ## Custom implementation
 

tests/RobotFramework/tests/cumulocity/software_management/sm-plugin.robot

@@ -74,6 +74,51 @@ sm-plugins download files from Cumulocity
     ${downloaded}=    Execute Command    cat /tmp/dummy3/installed_dummy-software
     Should Be Equal    ${downloaded}    Testing a thing
 
+Filter packages list using include pattern
+    Execute Command    sudo tedge config set software.plugin.include "^dummy1-[0-1]00[2-4]$"
+    Execute Command    sudo tedge config set software.plugin.max_packages 0
+    Connect Mapper    c8y
+    Device Should Exist    ${DEVICE_SN}
+    ${software}=    Device Should Have Installed Software
+    ...    {"name": "dummy1-0002", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-0003", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-0004", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-1002", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-1003", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-1004", "version": "1.0.0", "softwareType": "dummy1"}
+    Length Should Be    ${software}    6
+
+Filter packages list using exclude pattern
+    Execute Command    sudo tedge config set software.plugin.exclude "^dummy[1-2]-\\d+(0|2|4|6|8)$"
+    Execute Command    sudo tedge config set software.plugin.max_packages 0
+    Connect Mapper    c8y
+    Device Should Exist    ${DEVICE_SN}
+    ${software}=    Device Should Have Installed Software
+    ...    {"name": "dummy1-0001", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-0003", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-0005", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-1499", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy2-0001", "version": "1.0.0", "softwareType": "dummy2"}
+    ...    {"name": "dummy2-0003", "version": "1.0.0", "softwareType": "dummy2"}
+    ...    {"name": "dummy2-0005", "version": "1.0.0", "softwareType": "dummy2"}
+    ...    {"name": "dummy2-1499", "version": "1.0.0", "softwareType": "dummy2"}
+    Length Should Be    ${software}    1500
+
+Filter packages list using both patterns
+    Execute Command    sudo tedge config set software.plugin.exclude "^(dummy1.*)"
+    Execute Command    sudo tedge config set software.plugin.include "^(dummy1-\\d(|3|6|9)00)$"
+    Execute Command    sudo tedge config set software.plugin.max_packages 0
+    Connect Mapper    c8y
+    Device Should Exist    ${DEVICE_SN}
+    ${software}=    Device Should Have Installed Software
+    ...    {"name": "dummy1-0300", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-0600", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-0900", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy1-1300", "version": "1.0.0", "softwareType": "dummy1"}
+    ...    {"name": "dummy2-0001", "version": "1.0.0", "softwareType": "dummy2"}
+    ...    {"name": "dummy2-1500", "version": "1.0.0", "softwareType": "dummy2"}
+    Length Should Be    ${software}    1504
+
 *** Keywords ***
 Custom Setup
     ${DEVICE_SN}=    Setup    skip_bootstrap=${True}