From c818bea9f0497bf4c38ddcbab0c6ad7eeea22844 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Tue, 28 Jan 2025 10:16:52 +0100 Subject: [PATCH 01/15] optimize loops in recalculate_scan_profiles --- octopoes/octopoes/core/service.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index ce019172c25..aacfcbab956 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -264,21 +264,17 @@ def recalculate_scan_profiles(self, valid_time: datetime) -> None: # fetch all scan profiles all_scan_profiles = self.scan_profile_repository.list_scan_profiles(None, valid_time=valid_time) - # cache all declared - all_declared_scan_profiles = { - scan_profile for scan_profile in all_scan_profiles if isinstance(scan_profile, DeclaredScanProfile) - } - # cache all inherited - inherited_scan_profiles = { - scan_profile.reference: scan_profile - for scan_profile in all_scan_profiles - if isinstance(scan_profile, InheritedScanProfile) - } - - # track all scan level assignments - assigned_scan_levels: dict[Reference, ScanLevel] = { - scan_profile.reference: scan_profile.level for scan_profile in all_declared_scan_profiles - } + all_declared_scan_profiles = set() + inherited_scan_profiles = {} + assigned_scan_levels: dict[Reference, ScanLevel] = {} + + # fill profile caches + for scan_profile in all_scan_profiles: + if isinstance(scan_profile, DeclaredScanProfile): + all_declared_scan_profiles.update(scan_profile) + assigned_scan_levels[scan_profile.reference] = scan_profile.level + elif isinstance(scan_profile, InheritedScanProfile) + inherited_scan_profiles[scan_profile.reference] = scan_profile for current_level in range(4, 0, -1): # start point: all scan profiles with current level + all higher scan levels From 28829569e8ca6ad2e90995b486b8f43a590e7223 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Tue, 28 Jan 2025 10:23:10 +0100 Subject: [PATCH 02/15] Update service.py --- octopoes/octopoes/core/service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index aacfcbab956..959f894649f 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -267,12 +267,14 @@ def recalculate_scan_profiles(self, valid_time: datetime) -> None: all_declared_scan_profiles = set() inherited_scan_profiles = {} assigned_scan_levels: dict[Reference, ScanLevel] = {} - + source_scan_profile_references = set() + # fill profile caches for scan_profile in all_scan_profiles: if isinstance(scan_profile, DeclaredScanProfile): all_declared_scan_profiles.update(scan_profile) assigned_scan_levels[scan_profile.reference] = scan_profile.level + source_scan_profile_references.update(scan_profile.reference) elif isinstance(scan_profile, InheritedScanProfile) inherited_scan_profiles[scan_profile.reference] = scan_profile @@ -327,7 +329,6 @@ def recalculate_scan_profiles(self, valid_time: datetime) -> None: # Save all assigned scan levels update_count = 0 - source_scan_profile_references = {sp.reference for sp in all_declared_scan_profiles} for reference, scan_level in assigned_scan_levels.items(): # Skip source scan profiles if reference in source_scan_profile_references: From 8cc1c57d33fd7bb6cc2b4d91c12809fc8d67593b Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Tue, 28 Jan 2025 14:53:02 +0100 Subject: [PATCH 03/15] Update service.py --- octopoes/octopoes/core/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index 959f894649f..c2a0e958244 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -268,14 +268,14 @@ def recalculate_scan_profiles(self, valid_time: datetime) -> None: inherited_scan_profiles = {} assigned_scan_levels: dict[Reference, ScanLevel] = {} source_scan_profile_references = set() - + # fill profile caches for scan_profile in all_scan_profiles: if isinstance(scan_profile, DeclaredScanProfile): all_declared_scan_profiles.update(scan_profile) assigned_scan_levels[scan_profile.reference] = scan_profile.level source_scan_profile_references.update(scan_profile.reference) - elif isinstance(scan_profile, InheritedScanProfile) + elif isinstance(scan_profile, InheritedScanProfile): inherited_scan_profiles[scan_profile.reference] = scan_profile for current_level in range(4, 0, -1): From cc52e42ce0e70d88cf7547d93ca73d731951e76d Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Tue, 28 Jan 2025 15:32:15 +0100 Subject: [PATCH 04/15] add type hinting to satisfy MyPy --- octopoes/octopoes/core/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index c2a0e958244..1c008ef6746 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -264,10 +264,10 @@ def recalculate_scan_profiles(self, valid_time: datetime) -> None: # fetch all scan profiles all_scan_profiles = self.scan_profile_repository.list_scan_profiles(None, valid_time=valid_time) - all_declared_scan_profiles = set() - inherited_scan_profiles = {} + all_declared_scan_profiles: set[DeclaredScanProfile] = set() + inherited_scan_profiles: dict[Reference, InheritedScanProfile] = {} assigned_scan_levels: dict[Reference, ScanLevel] = {} - source_scan_profile_references = set() + source_scan_profile_references: set[Reference] = set() # fill profile caches for scan_profile in all_scan_profiles: From 9acafe9d4ca241825a0aaeaece2732d5e1f82526 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Tue, 28 Jan 2025 15:46:20 +0100 Subject: [PATCH 05/15] Only split once, when extracting the object_type form a reference. --- octopoes/octopoes/models/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octopoes/octopoes/models/__init__.py b/octopoes/octopoes/models/__init__.py index fc6915cf9f3..b1f0be9d264 100644 --- a/octopoes/octopoes/models/__init__.py +++ b/octopoes/octopoes/models/__init__.py @@ -11,8 +11,8 @@ class Reference(str): @classmethod def parse(cls, ref_str: str) -> tuple[str, str]: - object_type, *natural_key_parts = ref_str.split("|") - return object_type, "|".join(natural_key_parts) + object_type, *natural_key_parts = ref_str.split("|", 1) + return object_type, natural_key_parts[0] if natural_key_parts else '' @property def class_(self) -> str: From a61647d0fc4ffc625cb5e6970c182477407f7015 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Tue, 28 Jan 2025 16:22:22 +0100 Subject: [PATCH 06/15] Update service.py --- octopoes/octopoes/core/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index 1c008ef6746..1793bfd0269 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -272,9 +272,9 @@ def recalculate_scan_profiles(self, valid_time: datetime) -> None: # fill profile caches for scan_profile in all_scan_profiles: if isinstance(scan_profile, DeclaredScanProfile): - all_declared_scan_profiles.update(scan_profile) + all_declared_scan_profiles.add(scan_profile) assigned_scan_levels[scan_profile.reference] = scan_profile.level - source_scan_profile_references.update(scan_profile.reference) + source_scan_profile_references.add(scan_profile.reference) elif isinstance(scan_profile, InheritedScanProfile): inherited_scan_profiles[scan_profile.reference] = scan_profile From d7e4194e6650a84a58d1a66fbfa9800219d8ac05 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Tue, 28 Jan 2025 16:27:50 +0100 Subject: [PATCH 07/15] Update __init__.py --- octopoes/octopoes/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octopoes/octopoes/models/__init__.py b/octopoes/octopoes/models/__init__.py index b1f0be9d264..7eca964fc20 100644 --- a/octopoes/octopoes/models/__init__.py +++ b/octopoes/octopoes/models/__init__.py @@ -12,7 +12,7 @@ class Reference(str): @classmethod def parse(cls, ref_str: str) -> tuple[str, str]: object_type, *natural_key_parts = ref_str.split("|", 1) - return object_type, natural_key_parts[0] if natural_key_parts else '' + return object_type, natural_key_parts[0] if natural_key_parts else "" @property def class_(self) -> str: From 655f016e638346e66131ca620f818a775c251ef1 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Thu, 30 Jan 2025 09:50:05 +0100 Subject: [PATCH 08/15] Update scan_profile_repository.py --- octopoes/octopoes/repositories/scan_profile_repository.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octopoes/octopoes/repositories/scan_profile_repository.py b/octopoes/octopoes/repositories/scan_profile_repository.py index c27954e0009..db066918f8f 100644 --- a/octopoes/octopoes/repositories/scan_profile_repository.py +++ b/octopoes/octopoes/repositories/scan_profile_repository.py @@ -41,6 +41,7 @@ def get_bulk(self, references: set[Reference], valid_time: datetime) -> list[Sca class XTDBScanProfileRepository(ScanProfileRepository): object_type = "ScanProfile" pk_prefix = "xt/id" + typeadapter = TypeAdapter(ScanProfile) # def __init__(self, event_manager: EventManager, session: XTDBSession): super().__init__(event_manager) @@ -62,7 +63,7 @@ def serialize(cls, scan_profile: ScanProfile) -> dict[str, Any]: @classmethod def deserialize(cls, data: dict[str, Any]) -> ScanProfileBase: - return TypeAdapter(ScanProfile).validate_python(data) + return cls.typeadapter.validate_python(data) def list_scan_profiles(self, scan_profile_type: str | None, valid_time: datetime) -> list[ScanProfileBase]: where = {"type": self.object_type} From 6da83c3a31e4f2f24a32c438e9a0e87014866024 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Thu, 30 Jan 2025 09:50:16 +0100 Subject: [PATCH 09/15] Update scan_profile_repository.py --- octopoes/octopoes/repositories/scan_profile_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octopoes/octopoes/repositories/scan_profile_repository.py b/octopoes/octopoes/repositories/scan_profile_repository.py index db066918f8f..cb639023487 100644 --- a/octopoes/octopoes/repositories/scan_profile_repository.py +++ b/octopoes/octopoes/repositories/scan_profile_repository.py @@ -41,7 +41,7 @@ def get_bulk(self, references: set[Reference], valid_time: datetime) -> list[Sca class XTDBScanProfileRepository(ScanProfileRepository): object_type = "ScanProfile" pk_prefix = "xt/id" - typeadapter = TypeAdapter(ScanProfile) # + typeadapter = TypeAdapter(ScanProfile) def __init__(self, event_manager: EventManager, session: XTDBSession): super().__init__(event_manager) From 456f1fd66b43d00d99160c4a57f3f17203f61f90 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Thu, 30 Jan 2025 13:32:02 +0100 Subject: [PATCH 10/15] Update scan_profile_repository.py --- octopoes/octopoes/repositories/scan_profile_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octopoes/octopoes/repositories/scan_profile_repository.py b/octopoes/octopoes/repositories/scan_profile_repository.py index cb639023487..0467f72d08c 100644 --- a/octopoes/octopoes/repositories/scan_profile_repository.py +++ b/octopoes/octopoes/repositories/scan_profile_repository.py @@ -15,6 +15,7 @@ from octopoes.xtdb.client import XTDBSession from octopoes.xtdb.query_builder import generate_pull_query +scan_profile_adapter = TypeAdapter(ScanProfile) class ScanProfileRepository(Repository): def __init__(self, event_manager: EventManager): @@ -41,7 +42,6 @@ def get_bulk(self, references: set[Reference], valid_time: datetime) -> list[Sca class XTDBScanProfileRepository(ScanProfileRepository): object_type = "ScanProfile" pk_prefix = "xt/id" - typeadapter = TypeAdapter(ScanProfile) def __init__(self, event_manager: EventManager, session: XTDBSession): super().__init__(event_manager) @@ -63,7 +63,7 @@ def serialize(cls, scan_profile: ScanProfile) -> dict[str, Any]: @classmethod def deserialize(cls, data: dict[str, Any]) -> ScanProfileBase: - return cls.typeadapter.validate_python(data) + return scan_profile_adapter.validate_python(data) def list_scan_profiles(self, scan_profile_type: str | None, valid_time: datetime) -> list[ScanProfileBase]: where = {"type": self.object_type} From 9c45a6fa4b2ca0b49bb833cd8d0f31d2317880fe Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Thu, 30 Jan 2025 13:35:15 +0100 Subject: [PATCH 11/15] Update scan_profile_repository.py --- octopoes/octopoes/repositories/scan_profile_repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octopoes/octopoes/repositories/scan_profile_repository.py b/octopoes/octopoes/repositories/scan_profile_repository.py index 0467f72d08c..b8af824bc15 100644 --- a/octopoes/octopoes/repositories/scan_profile_repository.py +++ b/octopoes/octopoes/repositories/scan_profile_repository.py @@ -17,6 +17,7 @@ scan_profile_adapter = TypeAdapter(ScanProfile) + class ScanProfileRepository(Repository): def __init__(self, event_manager: EventManager): self.event_manager = event_manager From 8f68bb3e70cf8615b20f0f7ff119b69f42c15145 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Thu, 30 Jan 2025 14:40:44 +0100 Subject: [PATCH 12/15] Update octopoes/octopoes/models/__init__.py Co-authored-by: Jeroen Dekkers --- octopoes/octopoes/models/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/octopoes/octopoes/models/__init__.py b/octopoes/octopoes/models/__init__.py index 7eca964fc20..a39a1d4e895 100644 --- a/octopoes/octopoes/models/__init__.py +++ b/octopoes/octopoes/models/__init__.py @@ -11,8 +11,7 @@ class Reference(str): @classmethod def parse(cls, ref_str: str) -> tuple[str, str]: - object_type, *natural_key_parts = ref_str.split("|", 1) - return object_type, natural_key_parts[0] if natural_key_parts else "" + return ref_str.split("|", 1) @property def class_(self) -> str: From f027df5178db832afd72dc17c1bad3912d42e081 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Wed, 5 Feb 2025 09:20:25 +0100 Subject: [PATCH 13/15] Update __init__.py --- octopoes/octopoes/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octopoes/octopoes/models/__init__.py b/octopoes/octopoes/models/__init__.py index a39a1d4e895..62507ba4335 100644 --- a/octopoes/octopoes/models/__init__.py +++ b/octopoes/octopoes/models/__init__.py @@ -10,7 +10,7 @@ class Reference(str): @classmethod - def parse(cls, ref_str: str) -> tuple[str, str]: + def parse(cls, ref_str: str) -> list[str]: return ref_str.split("|", 1) @property From d4f7834cd23b02452030ba028b50fe1576349afa Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Wed, 5 Feb 2025 09:26:06 +0100 Subject: [PATCH 14/15] Update __init__.py --- octopoes/octopoes/models/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octopoes/octopoes/models/__init__.py b/octopoes/octopoes/models/__init__.py index 62507ba4335..66efec27792 100644 --- a/octopoes/octopoes/models/__init__.py +++ b/octopoes/octopoes/models/__init__.py @@ -10,8 +10,9 @@ class Reference(str): @classmethod - def parse(cls, ref_str: str) -> list[str]: - return ref_str.split("|", 1) + def parse(cls, ref_str: str) -> tuple[str, str]: + object_type, natural_key_parts = ref_str.split("|", 1) + return object_type, natural_key_parts @property def class_(self) -> str: From d4d69666268d34f4c7242f8db9564e2387a82fbd Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Wed, 5 Feb 2025 10:20:42 +0100 Subject: [PATCH 15/15] Update __init__.py --- octopoes/octopoes/models/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octopoes/octopoes/models/__init__.py b/octopoes/octopoes/models/__init__.py index 66efec27792..fc6915cf9f3 100644 --- a/octopoes/octopoes/models/__init__.py +++ b/octopoes/octopoes/models/__init__.py @@ -11,8 +11,8 @@ class Reference(str): @classmethod def parse(cls, ref_str: str) -> tuple[str, str]: - object_type, natural_key_parts = ref_str.split("|", 1) - return object_type, natural_key_parts + object_type, *natural_key_parts = ref_str.split("|") + return object_type, "|".join(natural_key_parts) @property def class_(self) -> str: