diff --git a/CHANGELOG.md b/CHANGELOG.md index 457342c98..6fdebc540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Update prettier to 3.5.0 ([#3448](https://github.com/nf-core/tools/pull/3448)) - chore(deps): update python:3.12-slim docker digest to 34656cd ([#3450](https://github.com/nf-core/tools/pull/3450)) - Remove Twitter from README ([#3454](https://github.com/nf-core/tools/pull/3454)) +- Continuation of #3083 ([#3456](https://github.com/nf-core/tools/pull/3456)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 112f1480f..7cfba6453 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -54,7 +54,7 @@ subworkflows_test, subworkflows_update, ) -from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_REMOTE from nf_core.pipelines.download import DownloadError from nf_core.utils import check_if_outdated, nfcore_logo, rich_force_colors, setup_nfcore_dir diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 3882b7aa1..054b3f3c9 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,24 +1,18 @@ import logging import re from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import questionary import requests import rich.prompt - -if TYPE_CHECKING: - from nf_core.modules.modules_repo import ModulesRepo +import yaml import nf_core.utils +from nf_core.modules.modules_repo import ModulesRepo log = logging.getLogger(__name__) -# Constants for the nf-core/modules repo used throughout the module files -NF_CORE_MODULES_NAME = "nf-core" -NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" -NF_CORE_MODULES_DEFAULT_BRANCH = "master" - def get_repo_info(directory: Path, use_prompt: Optional[bool] = True) -> Tuple[Path, Optional[str], str]: """ @@ -143,12 +137,15 @@ def prompt_component_version_sha( return git_sha -def get_components_to_install(subworkflow_dir: Union[str, Path]) -> Tuple[List[str], List[str]]: +def get_components_to_install( + subworkflow_dir: Union[str, Path], +) -> Tuple[List[Dict[str, Optional[str]]], List[Dict[str, Optional[str]]]]: """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ - modules = [] - subworkflows = [] + modules: Dict[str, Dict[str, Optional[str]]] = {} + subworkflows: Dict[str, Dict[str, Optional[str]]] = {} + with open(Path(subworkflow_dir, "main.nf")) as fh: for line in fh: regex = re.compile( @@ -159,10 +156,38 @@ def get_components_to_install(subworkflow_dir: Union[str, Path]) -> Tuple[List[s name, link = match.groups() if link.startswith("../../../"): name_split = name.lower().split("_") - modules.append("/".join(name_split)) + component_name = "/".join(name_split) + component_dict: Dict[str, Optional[str]] = { + "name": component_name, + } + modules[component_name] = component_dict elif link.startswith("../"): - subworkflows.append(name.lower()) - return modules, subworkflows + component_name = name.lower() + component_dict = {"name": component_name} + subworkflows[component_name] = component_dict + + if (sw_meta := Path(subworkflow_dir, "meta.yml")).exists(): + with open(sw_meta) as fh: + meta = yaml.safe_load(fh) + if "components" in meta: + components = meta["components"] + for component in components: + if isinstance(component, dict): + component_name = list(component.keys())[0].lower() + branch = component[component_name].get("branch") + git_remote = component[component_name]["git_remote"] + modules_repo = ModulesRepo(git_remote, branch=branch) + current_comp_dict = subworkflows if component_name in subworkflows else modules + + component_dict = { + "org_path": modules_repo.repo_path, + "git_remote": git_remote, + "branch": branch, + } + + current_comp_dict[component_name].update(component_dict) + + return list(modules.values()), list(subworkflows.values()) def get_biotools_response(tool_name: str) -> Optional[Dict]: diff --git a/nf_core/components/constants.py b/nf_core/components/constants.py new file mode 100644 index 000000000..cc155f3d5 --- /dev/null +++ b/nf_core/components/constants.py @@ -0,0 +1,4 @@ +# Constants for the nf-core/modules repo used throughout the module files +NF_CORE_MODULES_NAME = "nf-core" +NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git" +NF_CORE_MODULES_DEFAULT_BRANCH = "master" diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 4cf7dc946..dc3816292 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -15,7 +15,7 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand -from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_REMOTE from nf_core.modules.modules_json import ModulesJson log = logging.getLogger(__name__) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 5bdcd1ebd..d45b4d2c3 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -1,7 +1,7 @@ import logging import os from pathlib import Path -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union import questionary from rich import print @@ -15,11 +15,14 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand from nf_core.components.components_utils import ( - NF_CORE_MODULES_NAME, get_components_to_install, prompt_component_version_sha, ) +from nf_core.components.constants import ( + NF_CORE_MODULES_NAME, +) from nf_core.modules.modules_json import ModulesJson +from nf_core.modules.modules_repo import ModulesRepo log = logging.getLogger(__name__) @@ -38,15 +41,33 @@ def __init__( installed_by: Optional[List[str]] = None, ): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + self.current_remote = ModulesRepo(remote_url, branch) + self.branch = branch self.force = force self.prompt = prompt self.sha = sha + self.current_sha = sha if installed_by is not None: self.installed_by = installed_by else: self.installed_by = [self.component_type] - def install(self, component: str, silent: bool = False) -> bool: + def install(self, component: Union[str, Dict[str, str]], silent: bool = False) -> bool: + if isinstance(component, dict): + # Override modules_repo when the component to install is a dependency from a subworkflow. + remote_url = component.get("git_remote", self.current_remote.remote_url) + branch = component.get("branch", self.branch) + self.modules_repo = ModulesRepo(remote_url, branch) + component = component["name"] + + if self.current_remote is None: + self.current_remote = self.modules_repo + + if self.current_remote.remote_url == self.modules_repo.remote_url and self.sha is not None: + self.current_sha = self.sha + else: + self.current_sha = None + if self.repo_type == "modules": log.error(f"You cannot install a {component} in a clone of nf-core/modules") return False @@ -70,8 +91,8 @@ def install(self, component: str, silent: bool = False) -> bool: return False # Verify SHA - if not self.modules_repo.verify_sha(self.prompt, self.sha): - err_msg = f"SHA '{self.sha}' is not a valid commit SHA for the repository '{self.modules_repo.remote_url}'" + if not self.modules_repo.verify_sha(self.prompt, self.current_sha): + err_msg = f"SHA '{self.current_sha}' is not a valid commit SHA for the repository '{self.modules_repo.remote_url}'" log.error(err_msg) return False @@ -114,7 +135,7 @@ def install(self, component: str, silent: bool = False) -> bool: modules_json.update(self.component_type, self.modules_repo, component, current_version, self.installed_by) return False try: - version = self.get_version(component, self.sha, self.prompt, current_version, self.modules_repo) + version = self.get_version(component, self.current_sha, self.prompt, current_version, self.modules_repo) except UserWarning as e: log.error(e) return False @@ -199,7 +220,7 @@ def collect_and_verify_name( if component is None: component = questionary.autocomplete( f"{'Tool' if self.component_type == 'modules' else 'Subworkflow'} name:", - choices=sorted(modules_repo.get_avail_components(self.component_type, commit=self.sha)), + choices=sorted(modules_repo.get_avail_components(self.component_type, commit=self.current_sha)), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -207,7 +228,9 @@ def collect_and_verify_name( return "" # Check that the supplied name is an available module/subworkflow - if component and component not in modules_repo.get_avail_components(self.component_type, commit=self.sha): + if component and component not in modules_repo.get_avail_components( + self.component_type, commit=self.current_sha + ): log.error(f"{self.component_type[:-1].title()} '{component}' not found in available {self.component_type}") print( Panel( @@ -223,9 +246,10 @@ def collect_and_verify_name( raise ValueError - if not modules_repo.component_exists(component, self.component_type, commit=self.sha): - warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" - log.warning(warn_msg) + if self.current_remote.remote_url == modules_repo.remote_url: + if not modules_repo.component_exists(component, self.component_type, commit=self.current_sha): + warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" + log.warning(warn_msg) return component diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 316b8e7cb..afe12c88a 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -19,13 +19,15 @@ class ComponentRemove(ComponentCommand): def __init__(self, component_type, pipeline_dir, remote_url=None, branch=None, no_pull=False): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) - def remove(self, component, removed_by=None, removed_components=None, force=False): + def remove(self, component, repo_url=None, repo_path=None, removed_by=None, removed_components=None, force=False): """ Remove an already installed module/subworkflow This command only works for modules/subworkflows that are installed from 'nf-core/modules' Args: component (str): Name of the component to remove + repo_url (str): URL of the repository where the component is located + repo_path (str): Directory where the component is installed removed_by (str): Name of the component that is removing the current component (a subworkflow name if the component is a dependency or "modules" or "subworkflows" if it is not a dependency) removed_components (list[str]): list of components that have been removed during a recursive remove of subworkflows @@ -46,7 +48,10 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals self.has_valid_directory() self.has_modules_file() - repo_path = self.modules_repo.repo_path + if repo_path is None: + repo_path = self.modules_repo.repo_path + if repo_url is None: + repo_url = self.modules_repo.remote_url if component is None: component = questionary.autocomplete( f"{self.component_type[:-1]} name:", @@ -68,9 +73,9 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals if not component_dir.exists(): log.error(f"Installation directory '{component_dir}' does not exist.") - if modules_json.component_present(component, self.modules_repo.remote_url, repo_path, self.component_type): + if modules_json.component_present(component, repo_url, repo_path, self.component_type): log.error(f"Found entry for '{component}' in 'modules.json'. Removing...") - modules_json.remove_entry(self.component_type, component, self.modules_repo.remote_url, repo_path) + modules_json.remove_entry(self.component_type, component, repo_url, repo_path) return False # remove all dependent components based on installed_by entry @@ -81,7 +86,7 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals removed_component = modules_json.remove_entry( self.component_type, component, - self.modules_repo.remote_url, + repo_url, repo_path, removed_by=removed_by, ) @@ -149,16 +154,19 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals if removed: if self.component_type == "subworkflows": removed_by = component - dependent_components = modules_json.get_dependent_components( - self.component_type, component, self.modules_repo.remote_url, repo_path, {} - ) - for component_name, component_type in dependent_components.items(): + dependent_components = modules_json.get_dependent_components(self.component_type, component, {}) + for component_name, component_data in dependent_components.items(): + component_repo, component_install_dir, component_type = component_data if component_name in removed_components: continue original_component_type = self.component_type self.component_type = component_type dependency_removed = self.remove( - component_name, removed_by=removed_by, removed_components=removed_components + component_name, + component_repo, + component_install_dir, + removed_by=removed_by, + removed_components=removed_components, ) self.component_type = original_component_type # remember removed dependencies diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 901a7f02f..7c61b6b00 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -41,6 +41,8 @@ def __init__( limit_output=False, ): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + self.current_remote = ModulesRepo(remote_url, branch) + self.branch = branch self.force = force self.prompt = prompt self.sha = sha @@ -92,6 +94,13 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr Returns: (bool): True if the update was successful, False otherwise. """ + if isinstance(component, dict): + # Override modules_repo when the component to install is a dependency from a subworkflow. + remote_url = component.get("git_remote", self.current_remote.remote_url) + branch = component.get("branch", self.branch) + self.modules_repo = ModulesRepo(remote_url, branch) + component = component["name"] + self.component = component if updated is None: updated = [] @@ -882,7 +891,17 @@ def get_components_to_update(self, component): if self.component_type == "modules": # All subworkflow names in the installed_by section of a module are subworkflows using this module # We need to update them too - subworkflows_to_update = [subworkflow for subworkflow in installed_by if subworkflow != self.component_type] + git_remote = self.current_remote.remote_url + for subworkflow in installed_by: + if subworkflow != component: + for remote_url, content in mods_json["repos"].items(): + if all_subworkflows := content.get("subworkflows"): + for _, details in all_subworkflows.items(): + if subworkflow in details: + git_remote = remote_url + if subworkflow != self.component_type: + subworkflows_to_update.append({"name": subworkflow, "git_remote": git_remote}) + elif self.component_type == "subworkflows": for repo, repo_content in mods_json["repos"].items(): for component_type, dir_content in repo_content.items(): @@ -893,9 +912,9 @@ def get_components_to_update(self, component): # We need to update it too if component in comp_content["installed_by"]: if component_type == "modules": - modules_to_update.append(comp) + modules_to_update.append({"name": comp, "git_remote": repo, "org_path": dir}) elif component_type == "subworkflows": - subworkflows_to_update.append(comp) + subworkflows_to_update.append({"name": comp, "git_remote": repo, "org_path": dir}) return modules_to_update, subworkflows_to_update @@ -910,7 +929,7 @@ def update_linked_components( Update modules and subworkflows linked to the component being updated. """ for s_update in subworkflows_to_update: - if s_update in updated: + if s_update["name"] in updated: continue original_component_type, original_update_all = self._change_component_type("subworkflows") self.update( @@ -922,7 +941,7 @@ def update_linked_components( self._reset_component_type(original_component_type, original_update_all) for m_update in modules_to_update: - if m_update in updated: + if m_update["name"] in updated: continue original_component_type, original_update_all = self._change_component_type("modules") try: @@ -945,28 +964,42 @@ def update_linked_components( def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update): """Check for linked components added or removed in the new subworkflow version""" if self.component_type == "subworkflows": - subworkflow_directory = Path(self.directory, self.component_type, self.modules_repo.repo_path, component) + org_path = self.current_remote.repo_path + + subworkflow_directory = Path(self.directory, self.component_type, org_path, component) included_modules, included_subworkflows = get_components_to_install(subworkflow_directory) # If a module/subworkflow has been removed from the subworkflow for module in modules_to_update: - if module not in included_modules: - log.info(f"Removing module '{module}' which is not included in '{component}' anymore.") + module_name = module["name"] + included_modules_names = [m["name"] for m in included_modules] + if module_name not in included_modules_names: + log.info(f"Removing module '{module_name}' which is not included in '{component}' anymore.") remove_module_object = ComponentRemove("modules", self.directory) - remove_module_object.remove(module, removed_by=component) + remove_module_object.remove(module_name, removed_by=component) for subworkflow in subworkflows_to_update: - if subworkflow not in included_subworkflows: - log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.") + subworkflow_name = subworkflow["name"] + included_subworkflow_names = [m["name"] for m in included_subworkflows] + if subworkflow_name not in included_subworkflow_names: + log.info( + f"Removing subworkflow '{subworkflow_name}' which is not included in '{component}' anymore." + ) remove_subworkflow_object = ComponentRemove("subworkflows", self.directory) - remove_subworkflow_object.remove(subworkflow, removed_by=component) + remove_subworkflow_object.remove(subworkflow_name, removed_by=component) # If a new module/subworkflow is included in the subworklfow and wasn't included before for module in included_modules: - if module not in modules_to_update: - log.info(f"Installing newly included module '{module}' for '{component}'") + module_name = module["name"] + module["git_remote"] = module.get("git_remote", self.current_remote.remote_url) + module["branch"] = module.get("branch", self.branch) + if module_name not in modules_to_update: + log.info(f"Installing newly included module '{module_name}' for '{component}'") install_module_object = ComponentInstall(self.directory, "modules", installed_by=component) install_module_object.install(module, silent=True) for subworkflow in included_subworkflows: - if subworkflow not in subworkflows_to_update: - log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'") + subworkflow_name = subworkflow["name"] + subworkflow["git_remote"] = subworkflow.get("git_remote", self.current_remote.remote_url) + subworkflow["branch"] = subworkflow.get("branch", self.branch) + if subworkflow_name not in subworkflows_to_update: + log.info(f"Installing newly included subworkflow '{subworkflow_name}' for '{component}'") install_subworkflow_object = ComponentInstall( self.directory, "subworkflows", installed_by=component ) @@ -985,3 +1018,5 @@ def _reset_component_type(self, original_component_type, original_update_all): self.component_type = original_component_type self.modules_json.pipeline_components = None self.update_all = original_update_all + if self.current_remote is None: + self.current_remote = self.modules_repo diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 15e98ffff..21820b3cf 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -15,7 +15,8 @@ from typing_extensions import NotRequired, TypedDict # for py<3.11 import nf_core.utils -from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, get_components_to_install +from nf_core.components.components_utils import get_components_to_install +from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.modules.modules_repo import ModulesRepo from nf_core.pipelines.lint_utils import dump_json_with_prettier @@ -676,7 +677,7 @@ def check_up_to_date(self): dump_modules_json = True for repo, subworkflows in subworkflows_dict.items(): for org, subworkflow in subworkflows: - self.recreate_dependencies(repo, org, subworkflow) + self.recreate_dependencies(repo, org, {"name": subworkflow}) self.pipeline_components = original_pipeline_components if dump_modules_json: @@ -1041,10 +1042,8 @@ def get_dependent_components( self, component_type, name, - repo_url, - install_dir, dependent_components, - ): + ) -> Tuple[(str, str, str)]: """ Retrieves all pipeline modules/subworkflows that are reported in the modules.json as being installed by the given component @@ -1052,8 +1051,6 @@ def get_dependent_components( Args: component_type (str): Type of component [modules, subworkflows] name (str): Name of the component to find dependencies for - repo_url (str): URL of the repository containing the components - install_dir (str): Name of the directory where components are installed Returns: (dict[str: str,]): Dictionary indexed with the component names, with component_type as value @@ -1065,15 +1062,17 @@ def get_dependent_components( component_types = ["modules"] if component_type == "modules" else ["modules", "subworkflows"] # Find all components that have an entry of install by of a given component, recursively call this function for subworkflows for type in component_types: - try: - components = self.modules_json["repos"][repo_url][type][install_dir].items() - except KeyError as e: - # This exception will raise when there are only modules installed - log.debug(f"Trying to retrieve all {type}. There aren't {type} installed. Failed with error {e}") - continue - for component_name, component_entry in components: - if name in component_entry["installed_by"]: - dependent_components[component_name] = type + for repo_url in self.modules_json["repos"].keys(): + modules_repo = ModulesRepo(repo_url) + install_dir = modules_repo.repo_path + try: + for comp in self.modules_json["repos"][repo_url][type][install_dir]: + if name in self.modules_json["repos"][repo_url][type][install_dir][comp]["installed_by"]: + dependent_components[comp] = (repo_url, install_dir, type) + except KeyError as e: + # This exception will raise when there are only modules installed + log.debug(f"Trying to retrieve all {type}. There aren't {type} installed. Failed with error {e}") + continue return dependent_components @@ -1261,20 +1260,27 @@ def recreate_dependencies(self, repo, org, subworkflow): i.e., no module or subworkflow has been installed by the user in the meantime """ - sw_path = Path(self.subworkflows_dir, org, subworkflow) + sw_name = subworkflow["name"] + sw_path = Path(self.subworkflows_dir, org, sw_name) dep_mods, dep_subwfs = get_components_to_install(sw_path) assert self.modules_json is not None # mypy for dep_mod in dep_mods: - installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] + name = dep_mod["name"] + current_repo = dep_mod.get("git_remote", repo) + current_org = dep_mod.get("org_path", org) + installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name]["installed_by"] if installed_by == ["modules"]: self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = [] - if subworkflow not in installed_by: - self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"].append(subworkflow) + if sw_name not in installed_by: + self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"].append(sw_name) for dep_subwf in dep_subwfs: - installed_by = self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] + name = dep_subwf["name"] + current_repo = dep_subwf.get("git_remote", repo) + current_org = dep_subwf.get("org_path", org) + installed_by = self.modules_json["repos"][current_repo]["subworkflows"][current_org][name]["installed_by"] if installed_by == ["subworkflows"]: self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] = [] - if subworkflow not in installed_by: - self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"].append(subworkflow) + if sw_name not in installed_by: + self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"].append(sw_name) self.recreate_dependencies(repo, org, dep_subwf) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 357fc49cc..30a724d73 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -10,9 +10,8 @@ import rich.prompt from git.exc import GitCommandError, InvalidGitRepositoryError -import nf_core.modules.modules_json import nf_core.modules.modules_utils -from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.synced_repo import RemoteProgressbar, SyncedRepo from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 8a2120ed0..0e62cccfc 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -113,9 +113,10 @@ def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): included_components_ = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) included_components = included_components_[0] + included_components_[1] # join included modules and included subworkflows in a single list + included_components_names = [component["name"] for component in included_components] if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] - for component in set(included_components): + for component in set(included_components_names): if component in meta_components: subworkflow.passed.append( ( diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 3ac5eaa49..43f9b8046 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -9,11 +9,7 @@ import git from git.exc import GitCommandError -from nf_core.components.components_utils import ( - NF_CORE_MODULES_DEFAULT_BRANCH, - NF_CORE_MODULES_NAME, - NF_CORE_MODULES_REMOTE, -) +from nf_core.components.constants import NF_CORE_MODULES_DEFAULT_BRANCH, NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.utils import load_tools_config log = logging.getLogger(__name__) diff --git a/tests/modules/test_modules_json.py b/tests/modules/test_modules_json.py index 325a8073b..029eb32cc 100644 --- a/tests/modules/test_modules_json.py +++ b/tests/modules/test_modules_json.py @@ -3,7 +3,7 @@ import shutil from pathlib import Path -from nf_core.components.components_utils import ( +from nf_core.components.constants import ( NF_CORE_MODULES_DEFAULT_BRANCH, NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, diff --git a/tests/modules/test_update.py b/tests/modules/test_update.py index 6c8eacc66..807f67cb8 100644 --- a/tests/modules/test_update.py +++ b/tests/modules/test_update.py @@ -8,7 +8,7 @@ import yaml import nf_core.utils -from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.modules.install import ModuleInstall from nf_core.modules.modules_json import ModulesJson from nf_core.modules.patch import ModulePatch diff --git a/tests/subworkflows/test_install.py b/tests/subworkflows/test_install.py index 00ba88841..b5010d49f 100644 --- a/tests/subworkflows/test_install.py +++ b/tests/subworkflows/test_install.py @@ -7,6 +7,7 @@ from ..test_subworkflows import TestSubworkflows from ..utils import ( + CROSS_ORGANIZATION_URL, GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, @@ -83,6 +84,33 @@ def test_subworkflows_install_different_branch_fail(self): install_obj.install("bam_stats_samtools") assert "Subworkflow 'bam_stats_samtools' not found in available subworkflows" in str(excinfo.value) + def test_subworkflows_install_across_organizations(self): + """Test installing a subworkflow with modules from different organizations""" + # The fastq_trim_fastp_fastqc subworkflow contains modules from different organizations + self.subworkflow_install_cross_org.install("fastq_trim_fastp_fastqc") + # Verify that the installed_by entry was added correctly + modules_json = ModulesJson(self.pipeline_dir) + mod_json = modules_json.get_modules_json() + assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["nf-core-test"]["fastqc"]["installed_by"] == [ + "fastq_trim_fastp_fastqc" + ] + + def test_subworkflow_install_with_same_module(self): + """Test installing a subworkflow with a module from a different organization that is already installed from another org""" + # The fastq_trim_fastp_fastqc subworkflow contains the cross-org fastqc module, not the nf-core one + self.subworkflow_install_cross_org.install("fastq_trim_fastp_fastqc") + # Verify that the installed_by entry was added correctly + modules_json = ModulesJson(self.pipeline_dir) + mod_json = modules_json.get_modules_json() + + assert mod_json["repos"]["https://github.com/nf-core/modules.git"]["modules"]["nf-core"]["fastqc"][ + "installed_by" + ] == ["modules"] + + assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["nf-core-test"]["fastqc"]["installed_by"] == [ + "fastq_trim_fastp_fastqc" + ] + def test_subworkflows_install_tracking(self): """Test installing a subworkflow and finding the correct entries in installed_by section of modules.json""" assert self.subworkflow_install.install("bam_sort_stats_samtools") diff --git a/tests/subworkflows/test_remove.py b/tests/subworkflows/test_remove.py index bad5a2ddb..94cd64d0c 100644 --- a/tests/subworkflows/test_remove.py +++ b/tests/subworkflows/test_remove.py @@ -1,6 +1,7 @@ from pathlib import Path from nf_core.modules.modules_json import ModulesJson +from tests.utils import CROSS_ORGANIZATION_URL from ..test_subworkflows import TestSubworkflows @@ -99,3 +100,31 @@ def test_subworkflows_remove_included_subworkflow(self): assert Path.exists(samtools_index_path) is True assert Path.exists(samtools_stats_path) is True self.subworkflow_remove.remove("bam_sort_stats_samtools") + + def test_subworkflows_remove_subworkflow_keep_installed_cross_org_module(self): + """Test removing subworkflow and all it's dependencies after installing it, except for a separately installed module from another organisation""" + self.subworkflow_install_cross_org.install("fastq_trim_fastp_fastqc") + self.mods_install.install("fastqc") + + subworkflow_path = Path(self.subworkflow_install.directory, "subworkflows", "nf-core-test") + fastq_trim_fastp_fastqc_path = Path(subworkflow_path, "fastq_trim_fastp_fastqc") + fastqc_path = Path(self.subworkflow_install.directory, "modules", "nf-core-test", "fastqc") + nfcore_fastqc_path = Path(self.subworkflow_install.directory, "modules", "nf-core", "fastqc") + + mod_json_before = ModulesJson(self.pipeline_dir).get_modules_json() + assert self.subworkflow_remove_cross_org.remove("fastq_trim_fastp_fastqc") + mod_json_after = ModulesJson(self.pipeline_dir).get_modules_json() + + assert Path.exists(fastq_trim_fastp_fastqc_path) is False + assert Path.exists(fastqc_path) is False + assert Path.exists(nfcore_fastqc_path) is True + assert mod_json_before != mod_json_after + # assert subworkflows key is removed from modules.json + assert CROSS_ORGANIZATION_URL not in mod_json_after["repos"].keys() + assert ( + "fastqc" in mod_json_after["repos"]["https://github.com/nf-core/modules.git"]["modules"]["nf-core"].keys() + ) + assert ( + "fastp" + not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"]["modules"]["nf-core"].keys() + ) diff --git a/tests/subworkflows/test_update.py b/tests/subworkflows/test_update.py index 9f5d1939f..b540d3555 100644 --- a/tests/subworkflows/test_update.py +++ b/tests/subworkflows/test_update.py @@ -8,13 +8,14 @@ import yaml import nf_core.utils -from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE from nf_core.modules.modules_json import ModulesJson from nf_core.modules.update import ModuleUpdate +from nf_core.subworkflows.install import SubworkflowInstall from nf_core.subworkflows.update import SubworkflowUpdate from ..test_subworkflows import TestSubworkflows -from ..utils import OLD_SUBWORKFLOWS_SHA, cmp_component +from ..utils import CROSS_ORGANIZATION_URL, OLD_SUBWORKFLOWS_SHA, cmp_component class TestSubworkflowsUpdate(TestSubworkflows): @@ -372,3 +373,31 @@ def test_update_change_of_included_modules(self): assert "ensemblvep" not in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] assert "ensemblvep/vep" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "ensemblvep/vep").is_dir() + + def test_update_subworkflow_across_orgs(self): + """Install and update a subworkflow with modules from different organizations""" + install_obj = SubworkflowInstall( + self.pipeline_dir, + remote_url=CROSS_ORGANIZATION_URL, + # Hash for an old version of fastq_trim_fastp_fastqc + # A dummy code change was made in order to have a commit to compare with + sha="9627f4367b11527194ef14473019d0e1a181b741", + ) + # The fastq_trim_fastp_fastqc subworkflow contains the cross-org fastqc module, not the nf-core one + install_obj.install("fastq_trim_fastp_fastqc") + + patch_path = Path(self.pipeline_dir, "fastq_trim_fastp_fastqc.patch") + update_obj = SubworkflowUpdate( + self.pipeline_dir, + remote_url=CROSS_ORGANIZATION_URL, + save_diff_fn=patch_path, + update_all=False, + update_deps=True, + show_diff=False, + ) + assert update_obj.update("fastq_trim_fastp_fastqc") is True + + with open(patch_path) as fh: + content = fh.read() + assert "- fastqc_raw_html = FASTQC_RAW.out.html" in content + assert "+ ch_fastqc_raw_html = FASTQC_RAW.out.html" in content diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 7c18ab0a2..446d4aedf 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -12,6 +12,8 @@ import nf_core.subworkflows from .utils import ( + CROSS_ORGANIZATION_BRANCH, + CROSS_ORGANIZATION_URL, GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, GITLAB_URL, @@ -103,10 +105,17 @@ def setUp(self): force=False, sha="8c343b3c8a0925949783dc547666007c245c235b", ) + self.subworkflow_install_cross_org = nf_core.subworkflows.SubworkflowInstall( + self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, branch=CROSS_ORGANIZATION_BRANCH + ) + self.mods_install = nf_core.modules.install.ModuleInstall(self.pipeline_dir, prompt=False, force=True) # Set up remove objects self.subworkflow_remove = nf_core.subworkflows.SubworkflowRemove(self.pipeline_dir) + self.subworkflow_remove_cross_org = nf_core.subworkflows.SubworkflowRemove( + self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, branch=CROSS_ORGANIZATION_BRANCH + ) @pytest.fixture(autouse=True) def _use_caplog(self, caplog): diff --git a/tests/utils.py b/tests/utils.py index b09e99131..5ecd0b3f5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -20,6 +20,8 @@ OLD_TRIMGALORE_SHA = "9b7a3bdefeaad5d42324aa7dd50f87bea1b04386" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" +CROSS_ORGANIZATION_URL = "https://github.com/nf-core-test/modules.git" +CROSS_ORGANIZATION_BRANCH = "main" GITLAB_REPO = "nf-core-test" GITLAB_DEFAULT_BRANCH = "main" GITLAB_SUBWORKFLOWS_BRANCH = "subworkflows"