From cecff19aace3be25a9bbe1e4d72d3a696c83f101 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Wed, 22 Jan 2025 15:04:00 -0500 Subject: [PATCH 01/20] add NotRequired default value --- src/middlewared/middlewared/api/base/model.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/middlewared/middlewared/api/base/model.py b/src/middlewared/middlewared/api/base/model.py index 43875f7253faa..6b98e2ea5f306 100644 --- a/src/middlewared/middlewared/api/base/model.py +++ b/src/middlewared/middlewared/api/base/model.py @@ -13,7 +13,11 @@ __all__ = ["BaseModel", "ForUpdateMetaclass", "query_result", "query_result_item", - "single_argument_args", "single_argument_result"] + "single_argument_args", "single_argument_result", "NotRequired"] + + +NotRequired = undefined +"""Use as the default value for fields that may be excluded from the model.""" class BaseModel(PydanticBaseModel): @@ -38,6 +42,19 @@ def dump(t): "cannot be a member of an Optional or a Union, please make the whole field Private." ) + @model_serializer(mode="wrap") + def serialize_basemodel(self, serializer): + def remove_undefined(d): + if isinstance(d, dict): + return { + k: remove_undefined(v) + for k, v in d.items() + if v is not undefined + } + return d + + return remove_undefined(serializer(self)) + def model_dump( self, *, @@ -51,7 +68,7 @@ def model_dump( exclude_none: bool = False, round_trip: bool = False, warnings: bool | typing.Literal['none', 'warn', 'error'] = True, - serialize_as_any: bool = False, + serialize_as_any: bool = True, # so that nested models set to `NotRequired` do not serialize ) -> dict[str, typing.Any]: return self.__pydantic_serializer__.to_python( self, From 28237fa90fd730d605b9fea0bad8aa7b039be4a6 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Wed, 22 Jan 2025 17:17:50 -0500 Subject: [PATCH 02/20] unit test --- tests/unit/test_api.py | 70 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 745cabf252a86..40b9a8607dfea 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -1,6 +1,6 @@ from pydantic import Field -from middlewared.api.base import BaseModel +from middlewared.api.base import BaseModel, NotRequired from middlewared.api.base.handler.result import serialize_result @@ -17,3 +17,71 @@ class AliasModelResult(BaseModel): dump = serialize_result(AliasModelResult, result, False) assert dump == {'field1': 1, 'field2': 'two', 'field3': False} + + +def test_not_required(): + class NestedModel(BaseModel): + a: int = NotRequired + + class NotRequiredModel(BaseModel): + b: int + c: int = 3 + d: int = NotRequired + e: NestedModel + f: NestedModel = Field(default_factory=NestedModel) + g: NestedModel = NotRequired + h: list[NestedModel] = NotRequired + + test_cases = ( + ( + {"b": 2, "e": {}}, + {"b": 2, "c": 3, "e": {}, "f": {}} + ), + ( + {"b": 2, "e": {"a": 1}}, + {"b": 2, "c": 3, "e": {"a": 1}, "f": {}} + ), + ( + {"b": 2, "c": -3, "e": {}}, + {"b": 2, "c": -3, "e": {}, "f": {}} + ), + ( + {"b": 2, "d": 4, "e": {}}, + {"b": 2, "c": 3, "d": 4, "e": {}, "f": {}} + ), + ( + {"b": 2, "e": {}, "f": {}}, + {"b": 2, "c": 3, "e": {}, "f": {}} + ), + ( + {"b": 2, "e": {}, "f": {"a": 1}}, + {"b": 2, "c": 3, "e": {}, "f": {"a": 1}} + ), + ( + {"b": 2, "e": {}, "g": {}}, + {"b": 2, "c": 3, "e": {}, "f": {}, "g": {}} + ), + ( + {"b": 2, "e": {}, "g": {"a": 1}}, + {"b": 2, "c": 3, "e": {}, "f": {}, "g": {"a": 1}} + ), + ( + {"b": 2, "e": {}, "h": []}, + {"b": 2, "c": 3, "e": {}, "f": {}, "h": []} + ), + ( + {"b": 2, "e": {}, "h": [{}]}, + {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{}]} + ), + ( + {"b": 2, "e": {}, "h": [{"a": 1}]}, + {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{"a": 1}]} + ), + ( + {"b": 2, "e": {}, "h": [{"a": 1}, {}]}, + {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{"a": 1}, {}]} + ), + ) + for args, dump in test_cases: + result = NotRequiredModel(**args).model_dump(warnings=False) + assert result == dump, (args, dump, result) From 78b43f15324ebfbe8ad2e587567ff7fb55c132f2 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Thu, 23 Jan 2025 10:37:58 -0500 Subject: [PATCH 03/20] add alias test case --- tests/unit/test_api.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 40b9a8607dfea..1ea48ea76b35b 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -29,13 +29,17 @@ class NotRequiredModel(BaseModel): d: int = NotRequired e: NestedModel f: NestedModel = Field(default_factory=NestedModel) + # default_factory must be used here g: NestedModel = NotRequired h: list[NestedModel] = NotRequired + i_: int = Field(alias="i", default=NotRequired) test_cases = ( ( {"b": 2, "e": {}}, + # args passed to NotRequiredModel {"b": 2, "c": 3, "e": {}, "f": {}} + # expected result for model_dump ), ( {"b": 2, "e": {"a": 1}}, @@ -81,7 +85,11 @@ class NotRequiredModel(BaseModel): {"b": 2, "e": {}, "h": [{"a": 1}, {}]}, {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{"a": 1}, {}]} ), + ( + {"b": 2, "e": {}, "i": 4}, + {"b": 2, "c": 3, "e": {}, "f": {}, "i": 4} + ), ) for args, dump in test_cases: - result = NotRequiredModel(**args).model_dump(warnings=False) + result = NotRequiredModel(**args).model_dump(warnings=False, by_alias=True) assert result == dump, (args, dump, result) From 081e2bd0b5f0d26f32f23b51ae25874cb1fda8e6 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Thu, 23 Jan 2025 10:46:04 -0500 Subject: [PATCH 04/20] add expose_secrets test case --- tests/unit/test_api.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 1ea48ea76b35b..67bfc5c95f22c 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -1,4 +1,4 @@ -from pydantic import Field +from pydantic import Field, Secret from middlewared.api.base import BaseModel, NotRequired from middlewared.api.base.handler.result import serialize_result @@ -33,6 +33,7 @@ class NotRequiredModel(BaseModel): g: NestedModel = NotRequired h: list[NestedModel] = NotRequired i_: int = Field(alias="i", default=NotRequired) + j: Secret[int] = NotRequired test_cases = ( ( @@ -89,7 +90,11 @@ class NotRequiredModel(BaseModel): {"b": 2, "e": {}, "i": 4}, {"b": 2, "c": 3, "e": {}, "f": {}, "i": 4} ), + ( + {"b": 2, "e": {}, "j": 4}, + {"b": 2, "c": 3, "e": {}, "f": {}, "j": 4} + ), ) for args, dump in test_cases: - result = NotRequiredModel(**args).model_dump(warnings=False, by_alias=True) + result = NotRequiredModel(**args).model_dump(context={"expose_secrets": True}, warnings=False, by_alias=True) assert result == dump, (args, dump, result) From 570e1255c147d399b798f4dab941780b4c4b1bb8 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Thu, 23 Jan 2025 11:19:38 -0500 Subject: [PATCH 05/20] ForUpdateMetaclass unit test --- tests/unit/test_api.py | 52 ++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index 67bfc5c95f22c..cdea5177d10f0 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -1,22 +1,32 @@ from pydantic import Field, Secret -from middlewared.api.base import BaseModel, NotRequired -from middlewared.api.base.handler.result import serialize_result +from middlewared.api.base import BaseModel, NotRequired, ForUpdateMetaclass + + +def check_serialization(test_model, test_cases): + for args, dump in test_cases: + result = test_model(**args).model_dump(context={"expose_secrets": True}, warnings=False, by_alias=True) + assert result == dump, (args, dump, result) def test_dump_by_alias(): class AliasModel(BaseModel): - field1_: int = Field(..., alias='field1') + field1_: int = Field(alias='field1') field2: str field3_: bool = Field(alias='field3', default=False) class AliasModelResult(BaseModel): result: AliasModel - result = {'field1': 1, 'field2': 'two'} - dump = serialize_result(AliasModelResult, result, False) - - assert dump == {'field1': 1, 'field2': 'two', 'field3': False} + test_cases = ( + ( + {"result": {'field1': 1, 'field2': 'two'}}, + # args passed to AliasModelResult + {"result": {'field1': 1, 'field2': 'two', 'field3': False}} + # expected result for model_dump + ), + ) + check_serialization(AliasModelResult, test_cases) def test_not_required(): @@ -38,9 +48,7 @@ class NotRequiredModel(BaseModel): test_cases = ( ( {"b": 2, "e": {}}, - # args passed to NotRequiredModel {"b": 2, "c": 3, "e": {}, "f": {}} - # expected result for model_dump ), ( {"b": 2, "e": {"a": 1}}, @@ -95,6 +103,26 @@ class NotRequiredModel(BaseModel): {"b": 2, "c": 3, "e": {}, "f": {}, "j": 4} ), ) - for args, dump in test_cases: - result = NotRequiredModel(**args).model_dump(context={"expose_secrets": True}, warnings=False, by_alias=True) - assert result == dump, (args, dump, result) + check_serialization(NotRequiredModel, test_cases) + + +def test_update_metaclass(): + class NestedModel(BaseModel): + a: int + + class UpdateModel(BaseModel, metaclass=ForUpdateMetaclass): + b: int + c: NestedModel + + test_cases = ( + ( + {}, {} + ), + ( + {"b": 2}, {"b": 2} + ), + ( + {"c": {"a": 1}}, {"c": {"a": 1}} + ), + ) + check_serialization(UpdateModel, test_cases) From 0ba801332d474e00e8b266a755db8a5ffc448f18 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Fri, 24 Jan 2025 10:15:45 -0500 Subject: [PATCH 06/20] use new model --- src/middlewared/middlewared/api/base/model.py | 33 +++++++++---------- tests/unit/test_api.py | 8 ++--- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/middlewared/middlewared/api/base/model.py b/src/middlewared/middlewared/api/base/model.py index 6b98e2ea5f306..5d0d7b9780abd 100644 --- a/src/middlewared/middlewared/api/base/model.py +++ b/src/middlewared/middlewared/api/base/model.py @@ -13,11 +13,7 @@ __all__ = ["BaseModel", "ForUpdateMetaclass", "query_result", "query_result_item", - "single_argument_args", "single_argument_result", "NotRequired"] - - -NotRequired = undefined -"""Use as the default value for fields that may be excluded from the model.""" + "single_argument_args", "single_argument_result", "NotRequired", "NotRequiredModel"] class BaseModel(PydanticBaseModel): @@ -42,19 +38,6 @@ def dump(t): "cannot be a member of an Optional or a Union, please make the whole field Private." ) - @model_serializer(mode="wrap") - def serialize_basemodel(self, serializer): - def remove_undefined(d): - if isinstance(d, dict): - return { - k: remove_undefined(v) - for k, v in d.items() - if v is not undefined - } - return d - - return remove_undefined(serializer(self)) - def model_dump( self, *, @@ -161,6 +144,20 @@ def serialize_model(self, serializer): } +class NotRequiredModel(BaseModel): + @model_serializer(mode="wrap") + def serialize_model(self, serializer): + return { + k: v + for k, v in serializer(self).items() + if v is not undefined + } + + +NotRequired = undefined +"""Use as the default value for fields that may be excluded from the model.""" + + def _field_for_update(field): new = copy.deepcopy(field) new.default = undefined diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index cdea5177d10f0..17b5a91744df6 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -1,6 +1,6 @@ from pydantic import Field, Secret -from middlewared.api.base import BaseModel, NotRequired, ForUpdateMetaclass +from middlewared.api.base import BaseModel, NotRequired, ForUpdateMetaclass, NotRequiredModel def check_serialization(test_model, test_cases): @@ -30,10 +30,10 @@ class AliasModelResult(BaseModel): def test_not_required(): - class NestedModel(BaseModel): + class NestedModel(NotRequiredModel): a: int = NotRequired - class NotRequiredModel(BaseModel): + class NotRequiredTestModel(NotRequiredModel): b: int c: int = 3 d: int = NotRequired @@ -103,7 +103,7 @@ class NotRequiredModel(BaseModel): {"b": 2, "c": 3, "e": {}, "f": {}, "j": 4} ), ) - check_serialization(NotRequiredModel, test_cases) + check_serialization(NotRequiredTestModel, test_cases) def test_update_metaclass(): From 134def43ee40b4b113bce10d990d22e3f72e7990 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Mon, 27 Jan 2025 10:48:04 -0500 Subject: [PATCH 07/20] Move unit tests to github CI --- .../pytest/unit/api/base/test_excluded.py | 112 ++++++++++++++- .../unit/api/handler/result/test_alias.py | 17 +++ tests/unit/test_api.py | 128 ------------------ 3 files changed, 126 insertions(+), 131 deletions(-) create mode 100644 src/middlewared/middlewared/pytest/unit/api/handler/result/test_alias.py delete mode 100644 tests/unit/test_api.py diff --git a/src/middlewared/middlewared/pytest/unit/api/base/test_excluded.py b/src/middlewared/middlewared/pytest/unit/api/base/test_excluded.py index 29f54934f3e89..aeb84caf93121 100644 --- a/src/middlewared/middlewared/pytest/unit/api/base/test_excluded.py +++ b/src/middlewared/middlewared/pytest/unit/api/base/test_excluded.py @@ -1,7 +1,8 @@ +from pydantic import Field, Secret import pytest -from middlewared.api.base import BaseModel, Excluded, excluded_field -from middlewared.api.base.handler.accept import accept_params +from middlewared.api.base import BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NotRequired, NotRequiredModel +from middlewared.api.base.handler.accept import accept_params, validate_model from middlewared.service_exception import ValidationErrors @@ -18,9 +19,114 @@ class CreateArgs(BaseModel): data: CreateObject +def check_serialization(test_model, test_cases): + for args, dump in test_cases: + result = validate_model(test_model, args) + assert result == dump, (args, dump, result) + + def test_excluded_field(): with pytest.raises(ValidationErrors) as ve: - accept_params(CreateObject, [{"id": 1, "name": "Ivan"}]) + accept_params(CreateArgs, [{"id": 1, "name": "Ivan"}]) assert ve.value.errors[0].attribute == "id" assert ve.value.errors[0].errmsg == "Extra inputs are not permitted" + + +def test_not_required(): + class NestedModel(NotRequiredModel): + a: int = NotRequired + + class NotRequiredTestModel(NotRequiredModel): + b: int + c: int = 3 + d: int = NotRequired + e: NestedModel + f: NestedModel = Field(default_factory=NestedModel) + # default_factory must be used here + g: NestedModel = NotRequired + h: list[NestedModel] = NotRequired + i_: int = Field(alias="i", default=NotRequired) + j: Secret[int] = NotRequired + + test_cases = ( + ( + {"b": 2, "e": {}}, + {"b": 2, "c": 3, "e": {}, "f": {}} + ), + ( + {"b": 2, "e": {"a": 1}}, + {"b": 2, "c": 3, "e": {"a": 1}, "f": {}} + ), + ( + {"b": 2, "c": -3, "e": {}}, + {"b": 2, "c": -3, "e": {}, "f": {}} + ), + ( + {"b": 2, "d": 4, "e": {}}, + {"b": 2, "c": 3, "d": 4, "e": {}, "f": {}} + ), + ( + {"b": 2, "e": {}, "f": {}}, + {"b": 2, "c": 3, "e": {}, "f": {}} + ), + ( + {"b": 2, "e": {}, "f": {"a": 1}}, + {"b": 2, "c": 3, "e": {}, "f": {"a": 1}} + ), + ( + {"b": 2, "e": {}, "g": {}}, + {"b": 2, "c": 3, "e": {}, "f": {}, "g": {}} + ), + ( + {"b": 2, "e": {}, "g": {"a": 1}}, + {"b": 2, "c": 3, "e": {}, "f": {}, "g": {"a": 1}} + ), + ( + {"b": 2, "e": {}, "h": []}, + {"b": 2, "c": 3, "e": {}, "f": {}, "h": []} + ), + ( + {"b": 2, "e": {}, "h": [{}]}, + {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{}]} + ), + ( + {"b": 2, "e": {}, "h": [{"a": 1}]}, + {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{"a": 1}]} + ), + ( + {"b": 2, "e": {}, "h": [{"a": 1}, {}]}, + {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{"a": 1}, {}]} + ), + ( + {"b": 2, "e": {}, "i": 4}, + {"b": 2, "c": 3, "e": {}, "f": {}, "i": 4} + ), + ( + {"b": 2, "e": {}, "j": 4}, + {"b": 2, "c": 3, "e": {}, "f": {}, "j": 4} + ), + ) + check_serialization(NotRequiredTestModel, test_cases) + + +def test_update_metaclass(): + class NestedModel(BaseModel): + a: int + + class UpdateModel(BaseModel, metaclass=ForUpdateMetaclass): + b: int + c: NestedModel + + test_cases = ( + ( + {}, {} + ), + ( + {"b": 2}, {"b": 2} + ), + ( + {"c": {"a": 1}}, {"c": {"a": 1}} + ), + ) + check_serialization(UpdateModel, test_cases) diff --git a/src/middlewared/middlewared/pytest/unit/api/handler/result/test_alias.py b/src/middlewared/middlewared/pytest/unit/api/handler/result/test_alias.py new file mode 100644 index 0000000000000..8a97cba6faa56 --- /dev/null +++ b/src/middlewared/middlewared/pytest/unit/api/handler/result/test_alias.py @@ -0,0 +1,17 @@ +from pydantic import Field + +from middlewared.api.base import BaseModel +from middlewared.api.base.handler.result import serialize_result + + +def test_dump_by_alias(): + class AliasModel(BaseModel): + field1_: int = Field(alias='field1') + field2: str + field3_: bool = Field(alias='field3', default=False) + + class AliasModelResult(BaseModel): + result: AliasModel + + result = serialize_result(AliasModelResult, {'field1': 1, 'field2': 'two'}, True) + assert result == {'field1': 1, 'field2': 'two', 'field3': False} diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py deleted file mode 100644 index 17b5a91744df6..0000000000000 --- a/tests/unit/test_api.py +++ /dev/null @@ -1,128 +0,0 @@ -from pydantic import Field, Secret - -from middlewared.api.base import BaseModel, NotRequired, ForUpdateMetaclass, NotRequiredModel - - -def check_serialization(test_model, test_cases): - for args, dump in test_cases: - result = test_model(**args).model_dump(context={"expose_secrets": True}, warnings=False, by_alias=True) - assert result == dump, (args, dump, result) - - -def test_dump_by_alias(): - class AliasModel(BaseModel): - field1_: int = Field(alias='field1') - field2: str - field3_: bool = Field(alias='field3', default=False) - - class AliasModelResult(BaseModel): - result: AliasModel - - test_cases = ( - ( - {"result": {'field1': 1, 'field2': 'two'}}, - # args passed to AliasModelResult - {"result": {'field1': 1, 'field2': 'two', 'field3': False}} - # expected result for model_dump - ), - ) - check_serialization(AliasModelResult, test_cases) - - -def test_not_required(): - class NestedModel(NotRequiredModel): - a: int = NotRequired - - class NotRequiredTestModel(NotRequiredModel): - b: int - c: int = 3 - d: int = NotRequired - e: NestedModel - f: NestedModel = Field(default_factory=NestedModel) - # default_factory must be used here - g: NestedModel = NotRequired - h: list[NestedModel] = NotRequired - i_: int = Field(alias="i", default=NotRequired) - j: Secret[int] = NotRequired - - test_cases = ( - ( - {"b": 2, "e": {}}, - {"b": 2, "c": 3, "e": {}, "f": {}} - ), - ( - {"b": 2, "e": {"a": 1}}, - {"b": 2, "c": 3, "e": {"a": 1}, "f": {}} - ), - ( - {"b": 2, "c": -3, "e": {}}, - {"b": 2, "c": -3, "e": {}, "f": {}} - ), - ( - {"b": 2, "d": 4, "e": {}}, - {"b": 2, "c": 3, "d": 4, "e": {}, "f": {}} - ), - ( - {"b": 2, "e": {}, "f": {}}, - {"b": 2, "c": 3, "e": {}, "f": {}} - ), - ( - {"b": 2, "e": {}, "f": {"a": 1}}, - {"b": 2, "c": 3, "e": {}, "f": {"a": 1}} - ), - ( - {"b": 2, "e": {}, "g": {}}, - {"b": 2, "c": 3, "e": {}, "f": {}, "g": {}} - ), - ( - {"b": 2, "e": {}, "g": {"a": 1}}, - {"b": 2, "c": 3, "e": {}, "f": {}, "g": {"a": 1}} - ), - ( - {"b": 2, "e": {}, "h": []}, - {"b": 2, "c": 3, "e": {}, "f": {}, "h": []} - ), - ( - {"b": 2, "e": {}, "h": [{}]}, - {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{}]} - ), - ( - {"b": 2, "e": {}, "h": [{"a": 1}]}, - {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{"a": 1}]} - ), - ( - {"b": 2, "e": {}, "h": [{"a": 1}, {}]}, - {"b": 2, "c": 3, "e": {}, "f": {}, "h": [{"a": 1}, {}]} - ), - ( - {"b": 2, "e": {}, "i": 4}, - {"b": 2, "c": 3, "e": {}, "f": {}, "i": 4} - ), - ( - {"b": 2, "e": {}, "j": 4}, - {"b": 2, "c": 3, "e": {}, "f": {}, "j": 4} - ), - ) - check_serialization(NotRequiredTestModel, test_cases) - - -def test_update_metaclass(): - class NestedModel(BaseModel): - a: int - - class UpdateModel(BaseModel, metaclass=ForUpdateMetaclass): - b: int - c: NestedModel - - test_cases = ( - ( - {}, {} - ), - ( - {"b": 2}, {"b": 2} - ), - ( - {"c": {"a": 1}}, {"c": {"a": 1}} - ), - ) - check_serialization(UpdateModel, test_cases) From 059d80caeee6ea717c3c876ddb92302c421e2c64 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Mon, 27 Jan 2025 12:25:10 -0500 Subject: [PATCH 08/20] validator has to be recursive --- src/middlewared/middlewared/api/base/model.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/middlewared/middlewared/api/base/model.py b/src/middlewared/middlewared/api/base/model.py index 5d0d7b9780abd..7ee5d75f2f115 100644 --- a/src/middlewared/middlewared/api/base/model.py +++ b/src/middlewared/middlewared/api/base/model.py @@ -146,13 +146,17 @@ def serialize_model(self, serializer): class NotRequiredModel(BaseModel): @model_serializer(mode="wrap") - def serialize_model(self, serializer): - return { - k: v - for k, v in serializer(self).items() - if v is not undefined - } - + def serialize_basemodel(self, serializer): + def remove_undefined(d): + if isinstance(d, dict): + return { + k: remove_undefined(v) + for k, v in d.items() + if v is not undefined + } + return d + + return remove_undefined(serializer(self)) NotRequired = undefined """Use as the default value for fields that may be excluded from the model.""" From 2f8e3393490a221f7a0b9b0e1a65d2727b711d16 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Mon, 27 Jan 2025 12:41:12 -0500 Subject: [PATCH 09/20] fix test_excluded_field --- .../middlewared/pytest/unit/api/base/test_excluded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middlewared/middlewared/pytest/unit/api/base/test_excluded.py b/src/middlewared/middlewared/pytest/unit/api/base/test_excluded.py index aeb84caf93121..7b276bf1370b9 100644 --- a/src/middlewared/middlewared/pytest/unit/api/base/test_excluded.py +++ b/src/middlewared/middlewared/pytest/unit/api/base/test_excluded.py @@ -29,7 +29,7 @@ def test_excluded_field(): with pytest.raises(ValidationErrors) as ve: accept_params(CreateArgs, [{"id": 1, "name": "Ivan"}]) - assert ve.value.errors[0].attribute == "id" + assert ve.value.errors[0].attribute == "data.id" assert ve.value.errors[0].errmsg == "Extra inputs are not permitted" From 65939f87a1c1d07791197df9305eb4088b32c585 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Tue, 28 Jan 2025 14:49:56 -0500 Subject: [PATCH 10/20] recursion actually not necessary --- src/middlewared/middlewared/api/base/model.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/middlewared/middlewared/api/base/model.py b/src/middlewared/middlewared/api/base/model.py index 7ee5d75f2f115..21fec50b0ce84 100644 --- a/src/middlewared/middlewared/api/base/model.py +++ b/src/middlewared/middlewared/api/base/model.py @@ -147,16 +147,15 @@ def serialize_model(self, serializer): class NotRequiredModel(BaseModel): @model_serializer(mode="wrap") def serialize_basemodel(self, serializer): - def remove_undefined(d): - if isinstance(d, dict): - return { - k: remove_undefined(v) - for k, v in d.items() - if v is not undefined - } - return d - - return remove_undefined(serializer(self)) + obj = serializer(self) + if isinstance(obj, dict): + return { + k: v + for k, v in obj.items() + if v is not undefined + } + return obj + NotRequired = undefined """Use as the default value for fields that may be excluded from the model.""" From 03dd44c89775dd958835aa15d4497b9301824bea Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Wed, 29 Jan 2025 12:20:38 -0500 Subject: [PATCH 11/20] add roles, convert boot.format --- .../middlewared/api/v25_04_0/__init__.py | 1 + .../middlewared/api/v25_04_0/boot.py | 94 +++++++++++++++++++ src/middlewared/middlewared/plugins/boot.py | 44 +++------ .../middlewared/plugins/boot_/format.py | 15 +-- 4 files changed, 114 insertions(+), 40 deletions(-) create mode 100644 src/middlewared/middlewared/api/v25_04_0/boot.py diff --git a/src/middlewared/middlewared/api/v25_04_0/__init__.py b/src/middlewared/middlewared/api/v25_04_0/__init__.py index eb8e5a30c3585..abe542cc715b5 100644 --- a/src/middlewared/middlewared/api/v25_04_0/__init__.py +++ b/src/middlewared/middlewared/api/v25_04_0/__init__.py @@ -9,6 +9,7 @@ from .app_ix_volume import * # noqa from .app_registry import * # noqa from .auth import * # noqa +from .boot import * # noqa from .boot_environments import * # noqa from .catalog import * # noqa from .cloud_backup import * # noqa diff --git a/src/middlewared/middlewared/api/v25_04_0/boot.py b/src/middlewared/middlewared/api/v25_04_0/boot.py new file mode 100644 index 0000000000000..044106381fac8 --- /dev/null +++ b/src/middlewared/middlewared/api/v25_04_0/boot.py @@ -0,0 +1,94 @@ +from typing import Literal + +from pydantic import Field, PositiveInt + +from middlewared.api.base import BaseModel, NotRequired, NotRequiredModel + + +__all__ = [ + "BootGetDisksArgs", "BootGetDisksResult", "BootAttachArgs", "BootAttachResult", "BootDetachArgs", + "BootDetachResult", "BootReplaceArgs", "BootReplaceResult", "BootScrubArgs", "BootScrubResult", + "BootSetScrubIntervalArgs", "BootSetScrubIntervalResult", "BootUpdateInitramfsArgs", "BootUpdateInitramfsResult" + "BootFormatArgs", "BootFormatResult" +] + + +class BootAttachOptions(BaseModel): + expand: bool = False + + +class BootFormatOptions(NotRequiredModel): + size: int = NotRequired + legacy_schema: Literal["BIOS_ONLY", "EFI_ONLY", None] = None + + +class BootUpdateInitramfsOptions(BaseModel): + database: str | None = None + force: bool = False + + +class BootGetDisksArgs(BaseModel): + pass + + +class BootGetDisksResult(BaseModel): + result: list[str] + + +class BootAttachArgs(BaseModel): + dev: str + options: BootAttachOptions = Field(default_factory=BootAttachOptions) + + +class BootAttachResult(BaseModel): + result: None + + +class BootDetachArgs(BaseModel): + dev: str + + +class BootDetachResult(BaseModel): + result: None + + +class BootFormatArgs(BaseModel): + dev: str + options: BootFormatOptions = Field(default_factory=BootFormatOptions) + + +class BootFormatResult(BaseModel): + result: None + + +class BootReplaceArgs(BaseModel): + label: str + dev: str + + +class BootReplaceResult(BaseModel): + result: None + + +class BootScrubArgs(BaseModel): + pass + + +class BootScrubResult(BaseModel): + result: None + + +class BootSetScrubIntervalArgs(BaseModel): + interval: PositiveInt + + +class BootSetScrubIntervalResult(BaseModel): + result: PositiveInt + + +class BootUpdateInitramfsArgs(BaseModel): + options: BootUpdateInitramfsOptions = Field(default_factory=BootUpdateInitramfsOptions) + + +class BootUpdateInitramfsResult(BaseModel): + result: bool diff --git a/src/middlewared/middlewared/plugins/boot.py b/src/middlewared/middlewared/plugins/boot.py index 8f72dfd3314e5..edcac41c37f19 100644 --- a/src/middlewared/middlewared/plugins/boot.py +++ b/src/middlewared/middlewared/plugins/boot.py @@ -1,12 +1,17 @@ import asyncio import os -from contextlib import asynccontextmanager -from middlewared.schema import accepts, Bool, Dict, Int, List, Str, returns, Patch +from middlewared.api import api_method +from middlewared.api.current import ( + BootGetDisksArgs, BootGetDisksResult, BootAttachArgs, BootAttachResult, BootDetachArgs, + BootDetachResult, BootReplaceArgs, BootReplaceResult, BootScrubArgs, BootScrubResult, + BootSetScrubIntervalArgs, BootSetScrubIntervalResult, BootUpdateInitramfsArgs, BootUpdateInitramfsResult +) +from middlewared.schema import accepts, returns, Patch from middlewared.service import CallError, Service, job, private from middlewared.utils import run from middlewared.utils.disks import valid_zfs_partition_uuids -from middlewared.validators import Range + BOOT_ATTACH_REPLACE_LOCK = 'boot_attach_replace' BOOT_POOL_NAME = BOOT_POOL_DISKS = None @@ -62,8 +67,7 @@ async def clear_disks_cache(self): global BOOT_POOL_DISKS BOOT_POOL_DISKS = None - @accepts(roles=['READONLY_ADMIN']) - @returns(List('disks', items=[Str('disk')])) + @api_method(BootGetDisksArgs, BootGetDisksResult, roles=['READONLY_ADMIN']) async def get_disks(self): """ Returns disks of the boot pool. @@ -81,14 +85,7 @@ async def get_boot_type(self): # https://wiki.debian.org/UEFI return 'EFI' if os.path.exists('/sys/firmware/efi') else 'BIOS' - @accepts( - Str('dev'), - Dict( - 'options', - Bool('expand', default=False), - ), - ) - @returns() + @api_method(BootAttachArgs, BootAttachResult, roles=['FULL_ADMIN']) @job(lock=BOOT_ATTACH_REPLACE_LOCK) async def attach(self, job, dev, options): """ @@ -138,8 +135,7 @@ async def attach(self, job, dev, options): await self.middleware.call('zfs.pool.online', BOOT_POOL_NAME, zfs_dev_part['name'], True) await self.update_initramfs() - @accepts(Str('dev')) - @returns() + @api_method(BootDetachArgs, BootDetachResult, roles=['FULL_ADMIN']) async def detach(self, dev): """ Detach given `dev` from boot pool. @@ -148,8 +144,7 @@ async def detach(self, dev): await self.middleware.call('zfs.pool.detach', BOOT_POOL_NAME, dev, {'clear_label': True}) await self.update_initramfs() - @accepts(Str('label'), Str('dev')) - @returns() + @api_method(BootReplaceArgs, BootReplaceResult, roles=['FULL_ADMIN']) @job(lock=BOOT_ATTACH_REPLACE_LOCK) async def replace(self, job, label, dev): """ @@ -187,8 +182,7 @@ async def replace(self, job, label, dev): await self.middleware.call('boot.install_loader', dev) await self.update_initramfs() - @accepts() - @returns() + @api_method(BootScrubArgs, BootScrubResult, roles=['BOOT_ENV_WRITE']) @job(lock='boot_scrub') async def scrub(self, job): """ @@ -197,10 +191,7 @@ async def scrub(self, job): subjob = await self.middleware.call('pool.scrub.scrub', BOOT_POOL_NAME) return await job.wrap(subjob) - @accepts( - Int('interval', validators=[Range(min_=1)]) - ) - @returns(Int('interval')) + @api_method(BootSetScrubIntervalArgs, BootSetScrubIntervalResult, roles=['FULL_ADMIN']) async def set_scrub_interval(self, interval): """ Set Automatic Scrub Interval value in days. @@ -213,12 +204,7 @@ async def set_scrub_interval(self, interval): ) return interval - @accepts(Dict( - 'options', - Str('database', default=None, null=True), - Bool('force', default=False), - )) - @private + @api_method(BootUpdateInitramfsArgs, BootUpdateInitramfsResult, roles=['FULL_ADMIN'], private=True) async def update_initramfs(self, options): """ Returns true if initramfs was updated and false otherwise. diff --git a/src/middlewared/middlewared/plugins/boot_/format.py b/src/middlewared/middlewared/plugins/boot_/format.py index 54fc8721eb767..47d1c37850ab1 100644 --- a/src/middlewared/middlewared/plugins/boot_/format.py +++ b/src/middlewared/middlewared/plugins/boot_/format.py @@ -1,19 +1,12 @@ -from middlewared.schema import accepts, Dict, Int, Str -from middlewared.service import CallError, private, Service +from middlewared.api import api_method +from middlewared.api.current import BootFormatArgs, BootFormatResult +from middlewared.service import CallError, Service from middlewared.utils import run class BootService(Service): - @accepts( - Str('dev'), - Dict( - 'options', - Int('size'), - Str('legacy_schema', enum=[None, 'BIOS_ONLY', 'EFI_ONLY'], null=True, default=None), - ) - ) - @private + @api_method(BootFormatArgs, BootFormatResult, roles=['FULL_ADMIN'], private=True) async def format(self, dev, options): """ Format a given disk `dev` using the appropriate partition layout From 51e3ba439204cfb046214ed124763980b2779de5 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Wed, 29 Jan 2025 16:36:12 -0500 Subject: [PATCH 12/20] missing comma --- src/middlewared/middlewared/api/v25_04_0/boot.py | 2 +- src/middlewared/middlewared/plugins/boot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/middlewared/middlewared/api/v25_04_0/boot.py b/src/middlewared/middlewared/api/v25_04_0/boot.py index 044106381fac8..c38b262dbe052 100644 --- a/src/middlewared/middlewared/api/v25_04_0/boot.py +++ b/src/middlewared/middlewared/api/v25_04_0/boot.py @@ -8,7 +8,7 @@ __all__ = [ "BootGetDisksArgs", "BootGetDisksResult", "BootAttachArgs", "BootAttachResult", "BootDetachArgs", "BootDetachResult", "BootReplaceArgs", "BootReplaceResult", "BootScrubArgs", "BootScrubResult", - "BootSetScrubIntervalArgs", "BootSetScrubIntervalResult", "BootUpdateInitramfsArgs", "BootUpdateInitramfsResult" + "BootSetScrubIntervalArgs", "BootSetScrubIntervalResult", "BootUpdateInitramfsArgs", "BootUpdateInitramfsResult", "BootFormatArgs", "BootFormatResult" ] diff --git a/src/middlewared/middlewared/plugins/boot.py b/src/middlewared/middlewared/plugins/boot.py index edcac41c37f19..06cfa6e143795 100644 --- a/src/middlewared/middlewared/plugins/boot.py +++ b/src/middlewared/middlewared/plugins/boot.py @@ -182,7 +182,7 @@ async def replace(self, job, label, dev): await self.middleware.call('boot.install_loader', dev) await self.update_initramfs() - @api_method(BootScrubArgs, BootScrubResult, roles=['BOOT_ENV_WRITE']) + @api_method(BootScrubArgs, BootScrubResult, roles=['FULL_ADMIN']) @job(lock='boot_scrub') async def scrub(self, job): """ From 07f18011f658939a83cee8790c30a1f53d04e508 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Thu, 30 Jan 2025 10:06:55 -0500 Subject: [PATCH 13/20] use BOOT_ENV_WRITE --- src/middlewared/middlewared/plugins/boot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/middlewared/middlewared/plugins/boot.py b/src/middlewared/middlewared/plugins/boot.py index 06cfa6e143795..e2cea8b17156e 100644 --- a/src/middlewared/middlewared/plugins/boot.py +++ b/src/middlewared/middlewared/plugins/boot.py @@ -85,7 +85,7 @@ async def get_boot_type(self): # https://wiki.debian.org/UEFI return 'EFI' if os.path.exists('/sys/firmware/efi') else 'BIOS' - @api_method(BootAttachArgs, BootAttachResult, roles=['FULL_ADMIN']) + @api_method(BootAttachArgs, BootAttachResult, roles=['BOOT_ENV_WRITE']) @job(lock=BOOT_ATTACH_REPLACE_LOCK) async def attach(self, job, dev, options): """ @@ -135,7 +135,7 @@ async def attach(self, job, dev, options): await self.middleware.call('zfs.pool.online', BOOT_POOL_NAME, zfs_dev_part['name'], True) await self.update_initramfs() - @api_method(BootDetachArgs, BootDetachResult, roles=['FULL_ADMIN']) + @api_method(BootDetachArgs, BootDetachResult, roles=['BOOT_ENV_WRITE']) async def detach(self, dev): """ Detach given `dev` from boot pool. @@ -144,7 +144,7 @@ async def detach(self, dev): await self.middleware.call('zfs.pool.detach', BOOT_POOL_NAME, dev, {'clear_label': True}) await self.update_initramfs() - @api_method(BootReplaceArgs, BootReplaceResult, roles=['FULL_ADMIN']) + @api_method(BootReplaceArgs, BootReplaceResult, roles=['BOOT_ENV_WRITE']) @job(lock=BOOT_ATTACH_REPLACE_LOCK) async def replace(self, job, label, dev): """ @@ -182,7 +182,7 @@ async def replace(self, job, label, dev): await self.middleware.call('boot.install_loader', dev) await self.update_initramfs() - @api_method(BootScrubArgs, BootScrubResult, roles=['FULL_ADMIN']) + @api_method(BootScrubArgs, BootScrubResult, roles=['BOOT_ENV_WRITE']) @job(lock='boot_scrub') async def scrub(self, job): """ @@ -191,7 +191,7 @@ async def scrub(self, job): subjob = await self.middleware.call('pool.scrub.scrub', BOOT_POOL_NAME) return await job.wrap(subjob) - @api_method(BootSetScrubIntervalArgs, BootSetScrubIntervalResult, roles=['FULL_ADMIN']) + @api_method(BootSetScrubIntervalArgs, BootSetScrubIntervalResult, roles=['BOOT_ENV_WRITE']) async def set_scrub_interval(self, interval): """ Set Automatic Scrub Interval value in days. @@ -204,7 +204,7 @@ async def set_scrub_interval(self, interval): ) return interval - @api_method(BootUpdateInitramfsArgs, BootUpdateInitramfsResult, roles=['FULL_ADMIN'], private=True) + @api_method(BootUpdateInitramfsArgs, BootUpdateInitramfsResult, roles=['BOOT_ENV_WRITE'], private=True) async def update_initramfs(self, options): """ Returns true if initramfs was updated and false otherwise. From c58588f93fb692f65f2b1a9bd4e5e37711657321 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Thu, 30 Jan 2025 13:46:17 -0500 Subject: [PATCH 14/20] update roles --- src/middlewared/middlewared/plugins/boot.py | 8 ++++---- src/middlewared/middlewared/plugins/boot_/format.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/middlewared/middlewared/plugins/boot.py b/src/middlewared/middlewared/plugins/boot.py index e2cea8b17156e..dc3ae5e97c78c 100644 --- a/src/middlewared/middlewared/plugins/boot.py +++ b/src/middlewared/middlewared/plugins/boot.py @@ -67,7 +67,7 @@ async def clear_disks_cache(self): global BOOT_POOL_DISKS BOOT_POOL_DISKS = None - @api_method(BootGetDisksArgs, BootGetDisksResult, roles=['READONLY_ADMIN']) + @api_method(BootGetDisksArgs, BootGetDisksResult, roles=['DISK_READ']) async def get_disks(self): """ Returns disks of the boot pool. @@ -85,7 +85,7 @@ async def get_boot_type(self): # https://wiki.debian.org/UEFI return 'EFI' if os.path.exists('/sys/firmware/efi') else 'BIOS' - @api_method(BootAttachArgs, BootAttachResult, roles=['BOOT_ENV_WRITE']) + @api_method(BootAttachArgs, BootAttachResult, roles=['DISK_WRITE']) @job(lock=BOOT_ATTACH_REPLACE_LOCK) async def attach(self, job, dev, options): """ @@ -135,7 +135,7 @@ async def attach(self, job, dev, options): await self.middleware.call('zfs.pool.online', BOOT_POOL_NAME, zfs_dev_part['name'], True) await self.update_initramfs() - @api_method(BootDetachArgs, BootDetachResult, roles=['BOOT_ENV_WRITE']) + @api_method(BootDetachArgs, BootDetachResult, roles=['DISK_WRITE']) async def detach(self, dev): """ Detach given `dev` from boot pool. @@ -144,7 +144,7 @@ async def detach(self, dev): await self.middleware.call('zfs.pool.detach', BOOT_POOL_NAME, dev, {'clear_label': True}) await self.update_initramfs() - @api_method(BootReplaceArgs, BootReplaceResult, roles=['BOOT_ENV_WRITE']) + @api_method(BootReplaceArgs, BootReplaceResult, roles=['DISK_WRITE']) @job(lock=BOOT_ATTACH_REPLACE_LOCK) async def replace(self, job, label, dev): """ diff --git a/src/middlewared/middlewared/plugins/boot_/format.py b/src/middlewared/middlewared/plugins/boot_/format.py index 47d1c37850ab1..c150f5a1fbcfa 100644 --- a/src/middlewared/middlewared/plugins/boot_/format.py +++ b/src/middlewared/middlewared/plugins/boot_/format.py @@ -6,7 +6,7 @@ class BootService(Service): - @api_method(BootFormatArgs, BootFormatResult, roles=['FULL_ADMIN'], private=True) + @api_method(BootFormatArgs, BootFormatResult, roles=['DISK_WRITE'], private=True) async def format(self, dev, options): """ Format a given disk `dev` using the appropriate partition layout From 711849d5e97de08d443944ff4388b1628750da6c Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Thu, 30 Jan 2025 13:49:15 -0500 Subject: [PATCH 15/20] fix usage of NotRequired --- src/middlewared/middlewared/api/v25_04_0/boot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/middlewared/middlewared/api/v25_04_0/boot.py b/src/middlewared/middlewared/api/v25_04_0/boot.py index c38b262dbe052..c28d17755f587 100644 --- a/src/middlewared/middlewared/api/v25_04_0/boot.py +++ b/src/middlewared/middlewared/api/v25_04_0/boot.py @@ -2,7 +2,7 @@ from pydantic import Field, PositiveInt -from middlewared.api.base import BaseModel, NotRequired, NotRequiredModel +from middlewared.api.base import BaseModel, NotRequired __all__ = [ @@ -17,7 +17,7 @@ class BootAttachOptions(BaseModel): expand: bool = False -class BootFormatOptions(NotRequiredModel): +class BootFormatOptions(BaseModel): size: int = NotRequired legacy_schema: Literal["BIOS_ONLY", "EFI_ONLY", None] = None From aeafe7b67a9f5e0353778bf153f4563a1abb2c90 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Thu, 30 Jan 2025 18:10:54 -0500 Subject: [PATCH 16/20] do NOT set new roles on private endpoints --- src/middlewared/middlewared/plugins/boot.py | 2 +- src/middlewared/middlewared/plugins/boot_/format.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/middlewared/middlewared/plugins/boot.py b/src/middlewared/middlewared/plugins/boot.py index dc3ae5e97c78c..6dcb61d96f47a 100644 --- a/src/middlewared/middlewared/plugins/boot.py +++ b/src/middlewared/middlewared/plugins/boot.py @@ -204,7 +204,7 @@ async def set_scrub_interval(self, interval): ) return interval - @api_method(BootUpdateInitramfsArgs, BootUpdateInitramfsResult, roles=['BOOT_ENV_WRITE'], private=True) + @api_method(BootUpdateInitramfsArgs, BootUpdateInitramfsResult, private=True) async def update_initramfs(self, options): """ Returns true if initramfs was updated and false otherwise. diff --git a/src/middlewared/middlewared/plugins/boot_/format.py b/src/middlewared/middlewared/plugins/boot_/format.py index c150f5a1fbcfa..c40bfee72d9f4 100644 --- a/src/middlewared/middlewared/plugins/boot_/format.py +++ b/src/middlewared/middlewared/plugins/boot_/format.py @@ -6,7 +6,7 @@ class BootService(Service): - @api_method(BootFormatArgs, BootFormatResult, roles=['DISK_WRITE'], private=True) + @api_method(BootFormatArgs, BootFormatResult, private=True) async def format(self, dev, options): """ Format a given disk `dev` using the appropriate partition layout From 81785e1a2511936263758e457130d09c64a25e5d Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Wed, 5 Feb 2025 23:49:08 -0500 Subject: [PATCH 17/20] move to 25.10 --- src/middlewared/middlewared/api/v25_04_0/__init__.py | 1 - src/middlewared/middlewared/api/v25_10_0/__init__.py | 1 + src/middlewared/middlewared/api/{v25_04_0 => v25_10_0}/boot.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename src/middlewared/middlewared/api/{v25_04_0 => v25_10_0}/boot.py (100%) diff --git a/src/middlewared/middlewared/api/v25_04_0/__init__.py b/src/middlewared/middlewared/api/v25_04_0/__init__.py index f49bb23908a58..091a4fde8d346 100644 --- a/src/middlewared/middlewared/api/v25_04_0/__init__.py +++ b/src/middlewared/middlewared/api/v25_04_0/__init__.py @@ -9,7 +9,6 @@ from .app_ix_volume import * # noqa from .app_registry import * # noqa from .auth import * # noqa -from .boot import * # noqa from .boot_environments import * # noqa from .catalog import * # noqa from .cloud_backup import * # noqa diff --git a/src/middlewared/middlewared/api/v25_10_0/__init__.py b/src/middlewared/middlewared/api/v25_10_0/__init__.py index 091a4fde8d346..f49bb23908a58 100644 --- a/src/middlewared/middlewared/api/v25_10_0/__init__.py +++ b/src/middlewared/middlewared/api/v25_10_0/__init__.py @@ -9,6 +9,7 @@ from .app_ix_volume import * # noqa from .app_registry import * # noqa from .auth import * # noqa +from .boot import * # noqa from .boot_environments import * # noqa from .catalog import * # noqa from .cloud_backup import * # noqa diff --git a/src/middlewared/middlewared/api/v25_04_0/boot.py b/src/middlewared/middlewared/api/v25_10_0/boot.py similarity index 100% rename from src/middlewared/middlewared/api/v25_04_0/boot.py rename to src/middlewared/middlewared/api/v25_10_0/boot.py From 55a39cda1e1623b5b4d5fcf0852c8f0f7eca58c1 Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Thu, 6 Feb 2025 00:01:02 -0500 Subject: [PATCH 18/20] don't version private endpoints --- .../middlewared/api/v25_10_0/boot.py | 34 ++----------------- src/middlewared/middlewared/plugins/boot.py | 18 +++++++++- .../middlewared/plugins/boot_/format.py | 20 ++++++++++- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/middlewared/middlewared/api/v25_10_0/boot.py b/src/middlewared/middlewared/api/v25_10_0/boot.py index c28d17755f587..dc3e04bcadcd3 100644 --- a/src/middlewared/middlewared/api/v25_10_0/boot.py +++ b/src/middlewared/middlewared/api/v25_10_0/boot.py @@ -1,15 +1,12 @@ -from typing import Literal - from pydantic import Field, PositiveInt -from middlewared.api.base import BaseModel, NotRequired +from middlewared.api.base import BaseModel __all__ = [ "BootGetDisksArgs", "BootGetDisksResult", "BootAttachArgs", "BootAttachResult", "BootDetachArgs", "BootDetachResult", "BootReplaceArgs", "BootReplaceResult", "BootScrubArgs", "BootScrubResult", - "BootSetScrubIntervalArgs", "BootSetScrubIntervalResult", "BootUpdateInitramfsArgs", "BootUpdateInitramfsResult", - "BootFormatArgs", "BootFormatResult" + "BootSetScrubIntervalArgs", "BootSetScrubIntervalResult", ] @@ -17,16 +14,6 @@ class BootAttachOptions(BaseModel): expand: bool = False -class BootFormatOptions(BaseModel): - size: int = NotRequired - legacy_schema: Literal["BIOS_ONLY", "EFI_ONLY", None] = None - - -class BootUpdateInitramfsOptions(BaseModel): - database: str | None = None - force: bool = False - - class BootGetDisksArgs(BaseModel): pass @@ -52,15 +39,6 @@ class BootDetachResult(BaseModel): result: None -class BootFormatArgs(BaseModel): - dev: str - options: BootFormatOptions = Field(default_factory=BootFormatOptions) - - -class BootFormatResult(BaseModel): - result: None - - class BootReplaceArgs(BaseModel): label: str dev: str @@ -84,11 +62,3 @@ class BootSetScrubIntervalArgs(BaseModel): class BootSetScrubIntervalResult(BaseModel): result: PositiveInt - - -class BootUpdateInitramfsArgs(BaseModel): - options: BootUpdateInitramfsOptions = Field(default_factory=BootUpdateInitramfsOptions) - - -class BootUpdateInitramfsResult(BaseModel): - result: bool diff --git a/src/middlewared/middlewared/plugins/boot.py b/src/middlewared/middlewared/plugins/boot.py index 6dcb61d96f47a..cc56a9e4f0568 100644 --- a/src/middlewared/middlewared/plugins/boot.py +++ b/src/middlewared/middlewared/plugins/boot.py @@ -1,11 +1,14 @@ import asyncio import os +from pydantic import Field + from middlewared.api import api_method +from middlewared.api.base import BaseModel from middlewared.api.current import ( BootGetDisksArgs, BootGetDisksResult, BootAttachArgs, BootAttachResult, BootDetachArgs, BootDetachResult, BootReplaceArgs, BootReplaceResult, BootScrubArgs, BootScrubResult, - BootSetScrubIntervalArgs, BootSetScrubIntervalResult, BootUpdateInitramfsArgs, BootUpdateInitramfsResult + BootSetScrubIntervalArgs, BootSetScrubIntervalResult ) from middlewared.schema import accepts, returns, Patch from middlewared.service import CallError, Service, job, private @@ -18,6 +21,19 @@ BOOT_POOL_NAME_VALID = ['freenas-boot', 'boot-pool'] +class BootUpdateInitramfsOptions(BaseModel): + database: str | None = None + force: bool = False + + +class BootUpdateInitramfsArgs(BaseModel): + options: BootUpdateInitramfsOptions = Field(default_factory=BootUpdateInitramfsOptions) + + +class BootUpdateInitramfsResult(BaseModel): + result: bool + + class BootService(Service): class Config: diff --git a/src/middlewared/middlewared/plugins/boot_/format.py b/src/middlewared/middlewared/plugins/boot_/format.py index c40bfee72d9f4..9c6bdddb89cb4 100644 --- a/src/middlewared/middlewared/plugins/boot_/format.py +++ b/src/middlewared/middlewared/plugins/boot_/format.py @@ -1,9 +1,27 @@ +from typing import Literal + +from pydantic import Field + from middlewared.api import api_method -from middlewared.api.current import BootFormatArgs, BootFormatResult +from middlewared.api.base import BaseModel, NotRequired from middlewared.service import CallError, Service from middlewared.utils import run +class BootFormatOptions(BaseModel): + size: int = NotRequired + legacy_schema: Literal["BIOS_ONLY", "EFI_ONLY", None] = None + + +class BootFormatArgs(BaseModel): + dev: str + options: BootFormatOptions = Field(default_factory=BootFormatOptions) + + +class BootFormatResult(BaseModel): + result: None + + class BootService(Service): @api_method(BootFormatArgs, BootFormatResult, private=True) From 17807bf1a87edf3a70172bfc52700edaf631484e Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Tue, 11 Feb 2025 14:45:03 -0500 Subject: [PATCH 19/20] move BOOT_POOL_NAME_VALID --- src/middlewared/middlewared/plugins/boot.py | 3 +-- src/middlewared/middlewared/plugins/pool_/dataset.py | 5 ++--- src/middlewared/middlewared/plugins/pool_/pool.py | 2 +- src/middlewared/middlewared/plugins/sysdataset.py | 3 +-- src/middlewared/middlewared/plugins/virt/global.py | 3 +-- src/middlewared/middlewared/plugins/zfs_/pool_actions.py | 5 ++--- src/middlewared/middlewared/plugins/zfs_/utils.py | 6 ++---- src/middlewared/middlewared/utils/__init__.py | 1 + 8 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/middlewared/middlewared/plugins/boot.py b/src/middlewared/middlewared/plugins/boot.py index cc56a9e4f0568..90b8efea04b2f 100644 --- a/src/middlewared/middlewared/plugins/boot.py +++ b/src/middlewared/middlewared/plugins/boot.py @@ -12,13 +12,12 @@ ) from middlewared.schema import accepts, returns, Patch from middlewared.service import CallError, Service, job, private -from middlewared.utils import run +from middlewared.utils import run, BOOT_POOL_NAME_VALID from middlewared.utils.disks import valid_zfs_partition_uuids BOOT_ATTACH_REPLACE_LOCK = 'boot_attach_replace' BOOT_POOL_NAME = BOOT_POOL_DISKS = None -BOOT_POOL_NAME_VALID = ['freenas-boot', 'boot-pool'] class BootUpdateInitramfsOptions(BaseModel): diff --git a/src/middlewared/middlewared/plugins/pool_/dataset.py b/src/middlewared/middlewared/plugins/pool_/dataset.py index 895f3502e6ea6..9d0b4cbd43c17 100644 --- a/src/middlewared/middlewared/plugins/pool_/dataset.py +++ b/src/middlewared/middlewared/plugins/pool_/dataset.py @@ -3,16 +3,15 @@ import os import middlewared.sqlalchemy as sa -from middlewared.plugins.boot import BOOT_POOL_NAME_VALID from middlewared.plugins.zfs_.exceptions import ZFSSetPropertyError from middlewared.plugins.zfs_.validation_utils import validate_dataset_name from middlewared.schema import ( accepts, Any, Attribute, EnumMixin, Bool, Dict, Int, List, NOT_PROVIDED, Patch, Ref, returns, Str ) from middlewared.service import ( - CallError, CRUDService, filterable, InstanceNotFound, item_method, job, private, ValidationErrors + CallError, CRUDService, filterable, InstanceNotFound, item_method, private, ValidationErrors ) -from middlewared.utils import filter_list +from middlewared.utils import filter_list, BOOT_POOL_NAME_VALID from middlewared.validators import Exact, Match, Or, Range from .utils import ( diff --git a/src/middlewared/middlewared/plugins/pool_/pool.py b/src/middlewared/middlewared/plugins/pool_/pool.py index 509a1e7441d78..feb1b5e0aa206 100644 --- a/src/middlewared/middlewared/plugins/pool_/pool.py +++ b/src/middlewared/middlewared/plugins/pool_/pool.py @@ -5,10 +5,10 @@ from fenced.fence import ExitCode as FencedExitCodes -from middlewared.plugins.boot import BOOT_POOL_NAME_VALID from middlewared.plugins.zfs_.validation_utils import validate_pool_name from middlewared.schema import Bool, Dict, Int, List, Patch, Str from middlewared.service import accepts, CallError, CRUDService, job, private, returns, ValidationErrors +from middlewared.utils import BOOT_POOL_NAME_VALID from middlewared.utils.size import format_size from middlewared.validators import Range diff --git a/src/middlewared/middlewared/plugins/sysdataset.py b/src/middlewared/middlewared/plugins/sysdataset.py index 6ccac62db0818..8bde3077c462b 100644 --- a/src/middlewared/middlewared/plugins/sysdataset.py +++ b/src/middlewared/middlewared/plugins/sysdataset.py @@ -11,13 +11,12 @@ import middlewared.sqlalchemy as sa -from middlewared.plugins.boot import BOOT_POOL_NAME_VALID from middlewared.plugins.system_dataset.hierarchy import get_system_dataset_spec from middlewared.plugins.system_dataset.utils import SYSDATASET_PATH from middlewared.schema import accepts, Bool, Dict, Int, returns, Str from middlewared.service import CallError, ConfigService, ValidationErrors, job, private from middlewared.service_exception import InstanceNotFound -from middlewared.utils import filter_list, MIDDLEWARE_RUN_DIR +from middlewared.utils import filter_list, MIDDLEWARE_RUN_DIR, BOOT_POOL_NAME_VALID from middlewared.utils.directoryservices.constants import DSStatus, DSType from middlewared.utils.size import format_size from middlewared.utils.tdb import close_sysdataset_tdb_handles diff --git a/src/middlewared/middlewared/plugins/virt/global.py b/src/middlewared/middlewared/plugins/virt/global.py index 9fb39ac6d8c47..3bfedb0a01d9b 100644 --- a/src/middlewared/middlewared/plugins/virt/global.py +++ b/src/middlewared/middlewared/plugins/virt/global.py @@ -14,8 +14,7 @@ from middlewared.service import job, private from middlewared.service import ConfigService, ValidationErrors from middlewared.service_exception import CallError -from middlewared.utils import run -from middlewared.plugins.boot import BOOT_POOL_NAME_VALID +from middlewared.utils import run, BOOT_POOL_NAME_VALID from .utils import Status, incus_call if TYPE_CHECKING: diff --git a/src/middlewared/middlewared/plugins/zfs_/pool_actions.py b/src/middlewared/middlewared/plugins/zfs_/pool_actions.py index 24680fcc50530..a9c3e5f2bd48d 100644 --- a/src/middlewared/middlewared/plugins/zfs_/pool_actions.py +++ b/src/middlewared/middlewared/plugins/zfs_/pool_actions.py @@ -3,10 +3,9 @@ import subprocess import functools -from middlewared.plugins.boot import BOOT_POOL_NAME_VALID -from middlewared.schema import accepts, Bool, Dict, Int, Str +from middlewared.schema import accepts, Bool, Dict, Str from middlewared.service import CallError, Service -from middlewared.validators import Range +from middlewared.utils import BOOT_POOL_NAME_VALID from .pool_utils import find_vdev, SEARCH_PATHS diff --git a/src/middlewared/middlewared/plugins/zfs_/utils.py b/src/middlewared/middlewared/plugins/zfs_/utils.py index ae563cb3bf229..411fd2369a818 100644 --- a/src/middlewared/middlewared/plugins/zfs_/utils.py +++ b/src/middlewared/middlewared/plugins/zfs_/utils.py @@ -4,14 +4,12 @@ import os import re +from middlewared.plugins.audit.utils import AUDIT_DEFAULT_FILL_CRITICAL, AUDIT_DEFAULT_FILL_WARNING from middlewared.service_exception import CallError, MatchNotFound +from middlewared.utils import BOOT_POOL_NAME_VALID from middlewared.utils.filesystem.constants import ZFSCTL from middlewared.utils.mount import getmntinfo from middlewared.utils.path import is_child -from middlewared.plugins.audit.utils import ( - AUDIT_DEFAULT_FILL_CRITICAL, AUDIT_DEFAULT_FILL_WARNING -) -from middlewared.plugins.boot import BOOT_POOL_NAME_VALID from middlewared.utils.tdb import ( get_tdb_handle, TDBBatchAction, diff --git a/src/middlewared/middlewared/utils/__init__.py b/src/middlewared/middlewared/utils/__init__.py index d064d262342f6..cc83fbd7ba877 100644 --- a/src/middlewared/middlewared/utils/__init__.py +++ b/src/middlewared/middlewared/utils/__init__.py @@ -36,6 +36,7 @@ class ProductNames: MIDDLEWARE_RUN_DIR = '/var/run/middleware' MIDDLEWARE_STARTED_SENTINEL_PATH = f'{MIDDLEWARE_RUN_DIR}/middlewared-started' BOOTREADY = f'{MIDDLEWARE_RUN_DIR}/.bootready' +BOOT_POOL_NAME_VALID = ['freenas-boot', 'boot-pool'] MANIFEST_FILE = '/data/manifest.json' BRAND = ProductName.PRODUCT_NAME NULLS_FIRST = 'nulls_first:' From bf2a11c7ddf37a7a66dd3728d6e845c9ada0fede Mon Sep 17 00:00:00 2001 From: Logan Cary Date: Wed, 12 Feb 2025 10:59:56 -0500 Subject: [PATCH 20/20] don't import from boot --- src/middlewared/middlewared/plugins/zfs_/zfs_events.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/middlewared/middlewared/plugins/zfs_/zfs_events.py b/src/middlewared/middlewared/plugins/zfs_/zfs_events.py index 6ac31f9d869eb..2f09673a882a3 100644 --- a/src/middlewared/middlewared/plugins/zfs_/zfs_events.py +++ b/src/middlewared/middlewared/plugins/zfs_/zfs_events.py @@ -5,7 +5,6 @@ from middlewared.alert.base import ( Alert, AlertCategory, AlertClass, AlertLevel, OneShotAlertClass, SimpleOneShotAlertClass ) -from middlewared.plugins.boot import BOOT_POOL_NAME from middlewared.utils.threading import start_daemon_thread CACHE_POOLS_STATUSES = 'system.system_health_pools' @@ -158,7 +157,7 @@ async def zfs_events(middleware, data): if pool_name: await middleware.call('cache.pop', 'VolumeStatusAlerts') - if pool_name == BOOT_POOL_NAME: + if pool_name == await middleware.call('boot.pool_name'): # a change was made to the boot drive, so let's clear # the disk mapping for this pool await middleware.call('boot.clear_disks_cache')