Add new alert source file in support of periodic truenas_verify runs.

Update audit setup to generate an 'initial' version tagged truenas_verify log.

Fix truenas_verify audit setup.

Use ThreadedAlertSource

Fix up import formatting.

Removed setup_truenas_verify as an API endpoint.  Moved to an audit utility.
Converted truenas_verify call to middleware async 'run'.
Small Flake8 fix:  json was imported but not used.

Convert to run_in_thread module call.

Move from once per hour to once per day.

src/middlewared/middlewared/alert/source/truenas_verify.py

@@ -0,0 +1,32 @@
+from datetime import timedelta
+import subprocess
+
+from middlewared.alert.base import AlertClass, AlertCategory, Alert, AlertLevel, ThreadedAlertSource
+from middlewared.alert.schedule import IntervalSchedule
+
+
+# --------------- Monitored Alerts ----------------
+class TrueNASVerifyServiceChangeDetectionAlertClass(AlertClass):
+    category = AlertCategory.AUDIT
+    level = AlertLevel.ERROR
+    title = "TrueNAS Verify Service: Changes detected in root file system."
+    text = "%(verrs)s"
+
+
+class TrueNASVerifyServiceChangeDetectionAlertSource(ThreadedAlertSource):
+    '''
+    Periodic verification of root file system
+    '''
+    schedule = IntervalSchedule(timedelta(hours=24))
+    run_on_backup_node = False
+
+    def check_sync(self):
+        # Capture the results in syslog
+        res = subprocess.run(['truenas_verify', 'syslog'], capture_output=True, text=True)
+        if res.returncode:
+            errmsg = f"{res.stderr}  See syslog for details."
+            return Alert(
+                TrueNASVerifyServiceChangeDetectionAlertClass,
+                {'verrs': errmsg},
+                key=None
+            )

src/middlewared/middlewared/plugins/audit/audit.py

@@ -1,7 +1,6 @@
 import asyncio
 import csv
 import errno
-import json
 import middlewared.sqlalchemy as sa
 import os
 import shutil
@@ -22,6 +21,7 @@
     AUDITED_SERVICES,
     parse_query_filters,
     requires_python_filtering,
+    setup_truenas_verify,
 )
 from .schema.middleware import AUDIT_EVENT_MIDDLEWARE_JSON_SCHEMAS, AUDIT_EVENT_MIDDLEWARE_PARAM_SET
 from .schema.smb import AUDIT_EVENT_SMB_JSON_SCHEMAS, AUDIT_EVENT_SMB_PARAM_SET
@@ -598,6 +598,18 @@ async def setup(self):
             await self.middleware.call('alert.oneshot_create', 'AuditSetup', None)
             self.logger.error('Failed to apply auditing dataset configuration.', exc_info=True)
 
+        # Generate the initial truenas_verify file
+        try:
+            current_version = await self.middleware.call('system.version')
+            rc, err = await setup_truenas_verify(self.middleware, current_version)
+            if rc:
+                self.logger.error(
+                    'Unexpected result from truenas_verify initial setup. '
+                    'rc=%d, error=%s', rc, err
+                )
+        except Exception:
+            self.logger.error('Error detected in truenas_verify setup.', exe_info=True)
+
     @private
     @filterable
     async def json_schemas(self, filters, options):

src/middlewared/middlewared/plugins/audit/utils.py

@@ -1,10 +1,12 @@
-import middlewared.sqlalchemy as sa
 import os
 
 from sqlalchemy import Table
 from sqlalchemy.orm import declarative_base
 from .schema.common import AuditEventParam
 
+import middlewared.sqlalchemy as sa
+from truenas_verify import mtree_verify
+
 AUDIT_DATASET_PATH = '/audit'
 AUDITED_SERVICES = [('MIDDLEWARE', 0.1), ('SMB', 0.1), ('SUDO', 0.1), ('SYSTEM', 0.1)]
 AUDIT_TABLE_PREFIX = 'audit_'
@@ -195,3 +197,17 @@ def requires_python_filtering(
 
 
 AUDIT_TABLES = {svc[0]: generate_audit_table(*svc) for svc in AUDITED_SERVICES}
+
+
+async def setup_truenas_verify(middleware, sysver: str) -> tuple:
+    """
+    Called by audit setup to generate the initial truenas_verify
+    file for an updated or initial TrueNAS version.
+    """
+    verify_res = await middleware.run_in_thread(mtree_verify.do_verify, ['init', sysver])
+
+    err = ""
+    if verify_res.stderr:
+        err = verify_res.stderr.decode()
+
+    return (verify_res.returncode, err)