NAS-133206 / 25.10 / Convert boot.* to versioned API (#15524)

* add NotRequired default value

* unit test

* add alias test case

* add expose_secrets test case

* ForUpdateMetaclass unit test

* use new model

* Move unit tests to github CI

* validator has to be recursive

* fix test_excluded_field

* recursion actually not necessary

* add roles, convert boot.format

* missing comma

* use BOOT_ENV_WRITE

* update roles

* fix usage of NotRequired

* do NOT set new roles on private endpoints

* move to 25.10

* don't version private endpoints

* move BOOT_POOL_NAME_VALID

* don't import from boot

src/middlewared/middlewared/api/v25_10_0/__init__.py

@@ -9,6 +9,7 @@
 from .app_ix_volume import *  # noqa
 from .app_registry import *  # noqa
 from .auth import *  # noqa
+from .boot import *  # noqa
 from .boot_environments import *  # noqa
 from .catalog import *  # noqa
 from .cloud_backup import *  # noqa

src/middlewared/middlewared/api/v25_10_0/boot.py

@@ -0,0 +1,64 @@
+from pydantic import Field, PositiveInt
+
+from middlewared.api.base import BaseModel
+
+
+__all__ = [
+    "BootGetDisksArgs", "BootGetDisksResult", "BootAttachArgs", "BootAttachResult", "BootDetachArgs",
+    "BootDetachResult", "BootReplaceArgs", "BootReplaceResult", "BootScrubArgs", "BootScrubResult",
+    "BootSetScrubIntervalArgs", "BootSetScrubIntervalResult",
+]
+
+
+class BootAttachOptions(BaseModel):
+    expand: bool = False
+
+
+class BootGetDisksArgs(BaseModel):
+    pass
+
+
+class BootGetDisksResult(BaseModel):
+    result: list[str]
+
+
+class BootAttachArgs(BaseModel):
+    dev: str
+    options: BootAttachOptions = Field(default_factory=BootAttachOptions)
+
+
+class BootAttachResult(BaseModel):
+    result: None
+
+
+class BootDetachArgs(BaseModel):
+    dev: str
+
+
+class BootDetachResult(BaseModel):
+    result: None
+
+
+class BootReplaceArgs(BaseModel):
+    label: str
+    dev: str
+
+
+class BootReplaceResult(BaseModel):
+    result: None
+
+
+class BootScrubArgs(BaseModel):
+    pass
+
+
+class BootScrubResult(BaseModel):
+    result: None
+
+
+class BootSetScrubIntervalArgs(BaseModel):
+    interval: PositiveInt
+
+
+class BootSetScrubIntervalResult(BaseModel):
+    result: PositiveInt

src/middlewared/middlewared/plugins/boot.py

@@ -1,16 +1,36 @@
 import asyncio
 import os
 
-from contextlib import asynccontextmanager
-from middlewared.schema import accepts, Bool, Dict, Int, List, Str, returns, Patch
+from pydantic import Field
+
+from middlewared.api import api_method
+from middlewared.api.base import BaseModel
+from middlewared.api.current import (
+    BootGetDisksArgs, BootGetDisksResult, BootAttachArgs, BootAttachResult, BootDetachArgs,
+    BootDetachResult, BootReplaceArgs, BootReplaceResult, BootScrubArgs, BootScrubResult,
+    BootSetScrubIntervalArgs, BootSetScrubIntervalResult
+)
+from middlewared.schema import accepts, returns, Patch
 from middlewared.service import CallError, Service, job, private
-from middlewared.utils import run
+from middlewared.utils import run, BOOT_POOL_NAME_VALID
 from middlewared.utils.disks import valid_zfs_partition_uuids
-from middlewared.validators import Range
+
 
 BOOT_ATTACH_REPLACE_LOCK = 'boot_attach_replace'
 BOOT_POOL_NAME = BOOT_POOL_DISKS = None
-BOOT_POOL_NAME_VALID = ['freenas-boot', 'boot-pool']
+
+
+class BootUpdateInitramfsOptions(BaseModel):
+    database: str | None = None
+    force: bool = False
+
+
+class BootUpdateInitramfsArgs(BaseModel):
+    options: BootUpdateInitramfsOptions = Field(default_factory=BootUpdateInitramfsOptions)
+
+
+class BootUpdateInitramfsResult(BaseModel):
+    result: bool
 
 
 class BootService(Service):
@@ -62,8 +82,7 @@ async def clear_disks_cache(self):
         global BOOT_POOL_DISKS
         BOOT_POOL_DISKS = None
 
-    @accepts(roles=['READONLY_ADMIN'])
-    @returns(List('disks', items=[Str('disk')]))
+    @api_method(BootGetDisksArgs, BootGetDisksResult, roles=['DISK_READ'])
     async def get_disks(self):
         """
         Returns disks of the boot pool.
@@ -81,14 +100,7 @@ async def get_boot_type(self):
         # https://wiki.debian.org/UEFI
         return 'EFI' if os.path.exists('/sys/firmware/efi') else 'BIOS'
 
-    @accepts(
-        Str('dev'),
-        Dict(
-            'options',
-            Bool('expand', default=False),
-        ),
-    )
-    @returns()
+    @api_method(BootAttachArgs, BootAttachResult, roles=['DISK_WRITE'])
     @job(lock=BOOT_ATTACH_REPLACE_LOCK)
     async def attach(self, job, dev, options):
         """
@@ -138,8 +150,7 @@ async def attach(self, job, dev, options):
         await self.middleware.call('zfs.pool.online', BOOT_POOL_NAME, zfs_dev_part['name'], True)
         await self.update_initramfs()
 
-    @accepts(Str('dev'))
-    @returns()
+    @api_method(BootDetachArgs, BootDetachResult, roles=['DISK_WRITE'])
     async def detach(self, dev):
         """
         Detach given `dev` from boot pool.
@@ -148,8 +159,7 @@ async def detach(self, dev):
         await self.middleware.call('zfs.pool.detach', BOOT_POOL_NAME, dev, {'clear_label': True})
         await self.update_initramfs()
 
-    @accepts(Str('label'), Str('dev'))
-    @returns()
+    @api_method(BootReplaceArgs, BootReplaceResult, roles=['DISK_WRITE'])
     @job(lock=BOOT_ATTACH_REPLACE_LOCK)
     async def replace(self, job, label, dev):
         """
@@ -187,8 +197,7 @@ async def replace(self, job, label, dev):
         await self.middleware.call('boot.install_loader', dev)
         await self.update_initramfs()
 
-    @accepts()
-    @returns()
+    @api_method(BootScrubArgs, BootScrubResult, roles=['BOOT_ENV_WRITE'])
     @job(lock='boot_scrub')
     async def scrub(self, job):
         """
@@ -197,10 +206,7 @@ async def scrub(self, job):
         subjob = await self.middleware.call('pool.scrub.scrub', BOOT_POOL_NAME)
         return await job.wrap(subjob)
 
-    @accepts(
-        Int('interval', validators=[Range(min_=1)])
-    )
-    @returns(Int('interval'))
+    @api_method(BootSetScrubIntervalArgs, BootSetScrubIntervalResult, roles=['BOOT_ENV_WRITE'])
     async def set_scrub_interval(self, interval):
         """
         Set Automatic Scrub Interval value in days.
@@ -213,12 +219,7 @@ async def set_scrub_interval(self, interval):
         )
         return interval
 
-    @accepts(Dict(
-        'options',
-        Str('database', default=None, null=True),
-        Bool('force', default=False),
-    ))
-    @private
+    @api_method(BootUpdateInitramfsArgs, BootUpdateInitramfsResult, private=True)
     async def update_initramfs(self, options):
         """
         Returns true if initramfs was updated and false otherwise.

src/middlewared/middlewared/plugins/boot_/format.py

@@ -1,19 +1,30 @@
-from middlewared.schema import accepts, Dict, Int, Str
-from middlewared.service import CallError, private, Service
+from typing import Literal
+
+from pydantic import Field
+
+from middlewared.api import api_method
+from middlewared.api.base import BaseModel, NotRequired
+from middlewared.service import CallError, Service
 from middlewared.utils import run
 
 
+class BootFormatOptions(BaseModel):
+    size: int = NotRequired
+    legacy_schema: Literal["BIOS_ONLY", "EFI_ONLY", None] = None
+
+
+class BootFormatArgs(BaseModel):
+    dev: str
+    options: BootFormatOptions = Field(default_factory=BootFormatOptions)
+
+
+class BootFormatResult(BaseModel):
+    result: None
+
+
 class BootService(Service):
 
-    @accepts(
-        Str('dev'),
-        Dict(
-            'options',
-            Int('size'),
-            Str('legacy_schema', enum=[None, 'BIOS_ONLY', 'EFI_ONLY'], null=True, default=None),
-        )
-    )
-    @private
+    @api_method(BootFormatArgs, BootFormatResult, private=True)
     async def format(self, dev, options):
         """
         Format a given disk `dev` using the appropriate partition layout

src/middlewared/middlewared/plugins/pool_/dataset.py

@@ -3,16 +3,15 @@
 import os
 
 import middlewared.sqlalchemy as sa
-from middlewared.plugins.boot import BOOT_POOL_NAME_VALID
 from middlewared.plugins.zfs_.exceptions import ZFSSetPropertyError
 from middlewared.plugins.zfs_.validation_utils import validate_dataset_name
 from middlewared.schema import (
     accepts, Any, Attribute, EnumMixin, Bool, Dict, Int, List, NOT_PROVIDED, Patch, Ref, returns, Str
 )
 from middlewared.service import (
-    CallError, CRUDService, filterable, InstanceNotFound, item_method, job, private, ValidationErrors
+    CallError, CRUDService, filterable, InstanceNotFound, item_method, private, ValidationErrors
 )
-from middlewared.utils import filter_list
+from middlewared.utils import filter_list, BOOT_POOL_NAME_VALID
 from middlewared.validators import Exact, Match, Or, Range
 
 from .utils import (

src/middlewared/middlewared/plugins/pool_/pool.py

@@ -5,10 +5,10 @@
 
 from fenced.fence import ExitCode as FencedExitCodes
 
-from middlewared.plugins.boot import BOOT_POOL_NAME_VALID
 from middlewared.plugins.zfs_.validation_utils import validate_pool_name
 from middlewared.schema import Bool, Dict, Int, List, Patch, Str
 from middlewared.service import accepts, CallError, CRUDService, job, private, returns, ValidationErrors
+from middlewared.utils import BOOT_POOL_NAME_VALID
 from middlewared.utils.size import format_size
 from middlewared.validators import Range
 

src/middlewared/middlewared/plugins/sysdataset.py

@@ -11,13 +11,12 @@
 
 import middlewared.sqlalchemy as sa
 
-from middlewared.plugins.boot import BOOT_POOL_NAME_VALID
 from middlewared.plugins.system_dataset.hierarchy import get_system_dataset_spec
 from middlewared.plugins.system_dataset.utils import SYSDATASET_PATH
 from middlewared.schema import accepts, Bool, Dict, Int, returns, Str
 from middlewared.service import CallError, ConfigService, ValidationErrors, job, private
 from middlewared.service_exception import InstanceNotFound
-from middlewared.utils import filter_list, MIDDLEWARE_RUN_DIR
+from middlewared.utils import filter_list, MIDDLEWARE_RUN_DIR, BOOT_POOL_NAME_VALID
 from middlewared.utils.directoryservices.constants import DSStatus, DSType
 from middlewared.utils.size import format_size
 from middlewared.utils.tdb import close_sysdataset_tdb_handles

src/middlewared/middlewared/plugins/virt/global.py

@@ -14,8 +14,7 @@
 from middlewared.service import job, private
 from middlewared.service import ConfigService, ValidationErrors
 from middlewared.service_exception import CallError
-from middlewared.utils import run
-from middlewared.plugins.boot import BOOT_POOL_NAME_VALID
+from middlewared.utils import run, BOOT_POOL_NAME_VALID
 
 from .utils import Status, incus_call
 if TYPE_CHECKING:

src/middlewared/middlewared/plugins/zfs_/pool_actions.py

@@ -3,10 +3,9 @@
 import subprocess
 import functools
 
-from middlewared.plugins.boot import BOOT_POOL_NAME_VALID
-from middlewared.schema import accepts, Bool, Dict, Int, Str
+from middlewared.schema import accepts, Bool, Dict, Str
 from middlewared.service import CallError, Service
-from middlewared.validators import Range
+from middlewared.utils import BOOT_POOL_NAME_VALID
 
 from .pool_utils import find_vdev, SEARCH_PATHS
 

src/middlewared/middlewared/plugins/zfs_/utils.py

@@ -4,14 +4,12 @@
 import os
 import re
 
+from middlewared.plugins.audit.utils import AUDIT_DEFAULT_FILL_CRITICAL, AUDIT_DEFAULT_FILL_WARNING
 from middlewared.service_exception import CallError, MatchNotFound
+from middlewared.utils import BOOT_POOL_NAME_VALID
 from middlewared.utils.filesystem.constants import ZFSCTL
 from middlewared.utils.mount import getmntinfo
 from middlewared.utils.path import is_child
-from middlewared.plugins.audit.utils import (
-    AUDIT_DEFAULT_FILL_CRITICAL, AUDIT_DEFAULT_FILL_WARNING
-)
-from middlewared.plugins.boot import BOOT_POOL_NAME_VALID
 from middlewared.utils.tdb import (
     get_tdb_handle,
     TDBBatchAction,

src/middlewared/middlewared/plugins/zfs_/zfs_events.py

@@ -5,7 +5,6 @@
 from middlewared.alert.base import (
     Alert, AlertCategory, AlertClass, AlertLevel, OneShotAlertClass, SimpleOneShotAlertClass
 )
-from middlewared.plugins.boot import BOOT_POOL_NAME
 from middlewared.utils.threading import start_daemon_thread
 
 CACHE_POOLS_STATUSES = 'system.system_health_pools'
@@ -158,7 +157,7 @@ async def zfs_events(middleware, data):
         if pool_name:
             await middleware.call('cache.pop', 'VolumeStatusAlerts')
 
-            if pool_name == BOOT_POOL_NAME:
+            if pool_name == await middleware.call('boot.pool_name'):
                 # a change was made to the boot drive, so let's clear
                 # the disk mapping for this pool
                 await middleware.call('boot.clear_disks_cache')

src/middlewared/middlewared/utils/__init__.py

@@ -36,6 +36,7 @@ class ProductNames:
 MIDDLEWARE_RUN_DIR = '/var/run/middleware'
 MIDDLEWARE_STARTED_SENTINEL_PATH = f'{MIDDLEWARE_RUN_DIR}/middlewared-started'
 BOOTREADY = f'{MIDDLEWARE_RUN_DIR}/.bootready'
+BOOT_POOL_NAME_VALID = ['freenas-boot', 'boot-pool']
 MANIFEST_FILE = '/data/manifest.json'
 BRAND = ProductName.PRODUCT_NAME
 NULLS_FIRST = 'nulls_first:'