From 050ed61bf13264b2b446e054e6071a5932280290 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Mon, 8 Apr 2024 16:08:50 +0200 Subject: [PATCH 1/4] fix: make doctor fix for workflow ids also update oid and derived_from (#3723) * fix(cli): fix the doctor fix for plan ids to reassign oid's * allow overriding renku lock path * also fix derived from --- renku/command/checks/workflow.py | 18 ++++++++++++++++-- renku/core/config.py | 14 +++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/renku/command/checks/workflow.py b/renku/command/checks/workflow.py index 1f3bed770f..10dcbe9d41 100644 --- a/renku/command/checks/workflow.py +++ b/renku/command/checks/workflow.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Checks needed to determine integrity of workflows.""" + from datetime import timedelta from typing import List, Optional, Tuple, cast @@ -145,14 +146,17 @@ def check_plan_id(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, bool, Opt plans: List[AbstractPlan] = plan_gateway.get_all_plans() to_be_processed = [] + to_be_processed_derived = [] for plan in plans: if isinstance(plan.id, str) and plan.id.startswith("/plans//plans"): to_be_processed.append(plan) + if isinstance(plan.derived_from, str) and plan.derived_from.startswith("/plans//plans"): + to_be_processed_derived.append(plan) - if not to_be_processed: + if not to_be_processed and not to_be_processed_derived: return True, False, None if not fix: - ids = [plan.id for plan in to_be_processed] + ids = [plan.id for plan in to_be_processed + to_be_processed_derived] message = ( WARNING + "The following workflows have incorrect IDs (use 'renku doctor --fix' to fix them):\n\t" @@ -163,7 +167,17 @@ def check_plan_id(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, bool, Opt for plan in to_be_processed: plan.unfreeze() plan.id = plan.id.replace("//plans/", "/") + plan.reassign_oid() + plan._p_changed = True plan.freeze() + + for plan in to_be_processed_derived: + if plan.derived_from is not None: + plan.unfreeze() + plan.derived_from = plan.derived_from.replace("//plans/", "/") + plan._p_changed = True + plan.freeze() + project_context.database.commit() communication.info("Workflow IDs were fixed") diff --git a/renku/core/config.py b/renku/core/config.py index 899676113b..1e9d742240 100644 --- a/renku/core/config.py +++ b/renku/core/config.py @@ -18,6 +18,7 @@ import configparser import os from io import StringIO +from pathlib import Path from renku.core.constant import DATA_DIR_CONFIG_KEY from renku.domain_model.enums import ConfigFilter @@ -28,6 +29,10 @@ def global_config_read_lock(): """Create a user-level config read lock.""" from renku.core.util.contexts import Lock + lock_path = os.environ.get("RENKU_LOCK_PATH") + if lock_path is not None: + return Lock(Path(lock_path)) + return Lock(project_context.global_config_path) @@ -35,6 +40,10 @@ def global_config_write_lock(): """Create a user-level config write lock.""" from renku.core.util.contexts import Lock + lock_path = os.environ.get("RENKU_LOCK_PATH") + if lock_path is not None: + return Lock(Path(lock_path), mode="exclusive") + return Lock(project_context.global_config_path, mode="exclusive") @@ -112,7 +121,10 @@ def load_config(config_filter=ConfigFilter.ALL): elif config_filter == ConfigFilter.GLOBAL_ONLY: config_files += [project_context.global_config_path] elif config_filter == ConfigFilter.ALL: - config_files += [project_context.global_config_path, project_context.local_config_path] + config_files += [ + project_context.global_config_path, + project_context.local_config_path, + ] if config_filter != ConfigFilter.LOCAL_ONLY: with global_config_read_lock(): From b2418a15c89e16e4442a1ecf11f703f4e15922e8 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Thu, 11 Jul 2024 16:47:52 +0200 Subject: [PATCH 2/4] fix: properly remove activities in workflow remove if catalog is corrupt (#3729) --- renku/infrastructure/gateway/activity_gateway.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/renku/infrastructure/gateway/activity_gateway.py b/renku/infrastructure/gateway/activity_gateway.py index 7e8d970890..d9f24ddf75 100644 --- a/renku/infrastructure/gateway/activity_gateway.py +++ b/renku/infrastructure/gateway/activity_gateway.py @@ -272,6 +272,10 @@ def _index_activity(activity: Activity, database: Database): def _unindex_activity(activity: Activity, database: Database): """Add an activity to database indexes and create its up/downstream relations.""" + # there are cases where the activity catalog can be corrupt and removing activities doesn't actually remove them + # here we reindex them so removal works as expected + reindex_catalog(database) + upstreams = set() downstreams = set() From d7d84bc5c9b035027c5f0ea76e5e62d628b01bad Mon Sep 17 00:00:00 2001 From: Renku Bot Date: Fri, 12 Jul 2024 06:06:22 +0000 Subject: [PATCH 3/4] chore: release v2.9.4 --- CHANGES.rst | 13 +++++++++++++ helm-chart/renku-core/Chart.yaml | 2 +- helm-chart/renku-core/values.yaml | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 83c8988837..b41c7f0182 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,6 +18,19 @@ Changes ======= +`2.9.4 `__ (2024-07-12) +------------------------------------------------------------------------------------------------------- + +Bug Fixes +~~~~~~~~~ + +- make doctor fix for workflow ids also update oid and derived_from + (`#3723 `__) + (`050ed61 `__) +- properly remove activities in workflow remove if catalog is corrupt + (`#3729 `__) + (`b2418a1 `__) + `2.9.3 `__ (2024-04-08) ------------------------------------------------------------------------------------------------------- diff --git a/helm-chart/renku-core/Chart.yaml b/helm-chart/renku-core/Chart.yaml index f6154349de..d14b66054a 100644 --- a/helm-chart/renku-core/Chart.yaml +++ b/helm-chart/renku-core/Chart.yaml @@ -3,4 +3,4 @@ appVersion: "1.0" description: A Helm chart for Kubernetes name: renku-core icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4 -version: 2.9.3 +version: 2.9.4 diff --git a/helm-chart/renku-core/values.yaml b/helm-chart/renku-core/values.yaml index 17357c2c59..aefe4ab5e4 100644 --- a/helm-chart/renku-core/values.yaml +++ b/helm-chart/renku-core/values.yaml @@ -8,4 +8,4 @@ global: versions: latest: image: - tag: v2.9.3 + tag: v2.9.4 From 7021fa895a331216ab092b17a2489bc381c2a548 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Fri, 12 Jul 2024 15:48:13 +0200 Subject: [PATCH 4/4] Update CHANGES.rst --- CHANGES.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b41c7f0182..958edd338d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,9 +24,6 @@ Changes Bug Fixes ~~~~~~~~~ -- make doctor fix for workflow ids also update oid and derived_from - (`#3723 `__) - (`050ed61 `__) - properly remove activities in workflow remove if catalog is corrupt (`#3729 `__) (`b2418a1 `__)