diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0efea13ec9..053a146bdf 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -54,7 +54,12 @@ ) from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE from nf_core.pipelines.download import DownloadError -from nf_core.utils import check_if_outdated, nfcore_logo, rich_force_colors, setup_nfcore_dir +from nf_core.utils import ( + check_if_outdated, + nfcore_logo, + rich_force_colors, + setup_nfcore_dir, +) # Set up logging as the root logger # Submodules should all traverse back to this @@ -85,7 +90,14 @@ }, { "name": "For developers", - "commands": ["create", "lint", "bump-version", "sync", "schema", "create-logo"], + "commands": [ + "create", + "lint", + "bump-version", + "sync", + "schema", + "create-logo", + ], }, ], "nf-core modules": [ @@ -257,7 +269,13 @@ def pipelines(ctx): @click.option("-d", "--description", type=str, help="A short description of your pipeline") @click.option("-a", "--author", type=str, help="Name of the main author(s)") @click.option("--version", type=str, default="1.0.0dev", help="The initial version number to use") -@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") +@click.option( + "-f", + "--force", + is_flag=True, + default=False, + help="Overwrite output directory if it already exists", +) @click.option("-o", "--outdir", help="Output directory for new pipeline (default: pipeline name)") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") @click.option( @@ -270,7 +288,17 @@ def command_pipelines_create(ctx, name, description, author, version, force, out """ Create a new pipeline using the nf-core template. """ - pipelines_create(ctx, name, description, author, version, force, outdir, template_yaml, organisation) + pipelines_create( + ctx, + name, + description, + author, + version, + force, + outdir, + template_yaml, + organisation, + ) # nf-core pipelines lint @@ -346,7 +374,19 @@ def command_pipelines_lint( """ Check pipeline code against nf-core guidelines. """ - pipelines_lint(ctx, directory, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, sort_by) + pipelines_lint( + ctx, + directory, + release, + fix, + key, + show_passed, + fail_ignored, + fail_warned, + markdown, + json, + sort_by, + ) # nf-core pipelines download @@ -556,7 +596,18 @@ def command_pipelines_launch( """ Launch a pipeline using a web GUI or command line prompts. """ - pipelines_launch(ctx, pipeline, id, revision, command_only, params_in, params_out, save_all, show_hidden, url) + pipelines_launch( + ctx, + pipeline, + id, + revision, + command_only, + params_in, + params_out, + save_all, + show_hidden, + url, + ) # nf-core pipelines list @@ -613,12 +664,28 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived): @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") def command_pipelines_sync( - ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr + ctx, + directory, + from_branch, + pull_request, + github_repository, + username, + template_yaml, + force_pr, ): """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. """ - pipelines_sync(ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr) + pipelines_sync( + ctx, + directory, + from_branch, + pull_request, + github_repository, + username, + template_yaml, + force_pr, + ) # nf-core pipelines bump-version @@ -650,7 +717,14 @@ def command_pipelines_bump_version(ctx, new_version, directory, nextflow): # nf-core pipelines create-logo @pipelines.command("create-logo") @click.argument("logo-text", metavar="") -@click.option("-d", "--dir", "directory", type=click.Path(), default=".", help="Directory to save the logo in.") +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(), + default=".", + help="Directory to save the logo in.", +) @click.option( "-n", "--name", @@ -885,7 +959,13 @@ def command_modules_list_local(ctx, keywords, json, directory): # pylint: disab # nf-core modules install @modules.command("install") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -919,7 +999,13 @@ def command_modules_install(ctx, tool, directory, prompt, force, sha): # nf-core modules update @modules.command("update") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -990,13 +1076,31 @@ def command_modules_update( """ Update DSL2 modules within a pipeline. """ - modules_update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output) + modules_update( + ctx, + tool, + directory, + force, + prompt, + sha, + install_all, + preview, + save_diff, + update_deps, + limit_output, + ) # nf-core modules patch @modules.command("patch") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1016,7 +1120,13 @@ def command_modules_patch(ctx, tool, directory, remove): # nf-core modules remove @modules.command("remove") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1036,7 +1146,14 @@ def command_modules_remove(ctx, directory, tool): @modules.command("create") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") -@click.option("-d", "--dir", "directory", type=click.Path(exists=True), default=".", metavar="") +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(exists=True), + default=".", + metavar="", +) @click.option( "-a", "--author", @@ -1099,6 +1216,12 @@ def command_modules_remove(ctx, directory, tool): default=False, help="Migrate a module with pytest tests to nf-test", ) +@click.option( + "--migrate-pytest-hard", + is_flag=True, + default=False, + help="Try hard when migrating pytest tests", +) def command_modules_create( ctx, tool, @@ -1112,6 +1235,7 @@ def command_modules_create( conda_package_version, empty_template, migrate_pytest, + migrate_pytest_hard, ): """ Create a new DSL2 module from the nf-core template. @@ -1129,13 +1253,20 @@ def command_modules_create( conda_package_version, empty_template, migrate_pytest, + migrate_pytest_hard, ) # nf-core modules test @modules.command("test") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-v", "--verbose", @@ -1178,19 +1309,52 @@ def command_modules_create( default=False, help="Migrate a module with pytest tests to nf-test", ) -def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest, verbose): +@click.option( + "--migrate-pytest-hard", + is_flag=True, + default=False, + help="Try hard when migrating pytest tests", +) +def command_modules_test( + ctx, + tool, + directory, + no_prompts, + update, + once, + profile, + migrate_pytest, + migrate_pytest_hard, + verbose, +): """ Run nf-test for a module. """ if verbose: ctx.obj["verbose"] = verbose - modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest) + modules_test( + ctx, + tool, + directory, + no_prompts, + update, + once, + profile, + migrate_pytest, + migrate_pytest_hard, + ) # nf-core modules lint @modules.command("lint") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1231,17 +1395,47 @@ def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile is_flag=True, help="Fix the module version if a newer version is available", ) -def command_modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version): +def command_modules_lint( + ctx, + tool, + directory, + registry, + key, + all, + fail_warned, + local, + passed, + sort_by, + fix_version, +): """ Lint one or more modules in a directory. """ - modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version) + modules_lint( + ctx, + tool, + directory, + registry, + key, + all, + fail_warned, + local, + passed, + sort_by, + fix_version, + ) # nf-core modules info @modules.command("info") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1260,7 +1454,13 @@ def command_modules_info(ctx, tool, directory): # nf-core modules bump-versions @modules.command("bump-versions") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") +@click.argument( + "tool", + type=str, + callback=normalize_case, + required=False, + metavar=" or ", +) @click.option( "-d", "--dir", @@ -1321,7 +1521,14 @@ def subworkflows(ctx, git_remote, branch, no_pull): @subworkflows.command("create") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") -@click.option("-d", "--dir", "directory", type=click.Path(exists=True), default=".", metavar="") +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(exists=True), + default=".", + metavar="", +) @click.option( "-a", "--author", @@ -1352,7 +1559,13 @@ def command_subworkflows_create(ctx, subworkflow, directory, author, force, migr # nf-core subworkflows test @subworkflows.command("test") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1440,7 +1653,13 @@ def command_subworkflows_list_local(ctx, keywords, json, directory): # pylint: # nf-core subworkflows lint @subworkflows.command("lint") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1480,13 +1699,30 @@ def command_subworkflows_lint(ctx, subworkflow, directory, registry, key, all, f """ Lint one or more subworkflows in a directory. """ - subworkflows_lint(ctx, subworkflow, directory, registry, key, all, fail_warned, local, passed, sort_by) + subworkflows_lint( + ctx, + subworkflow, + directory, + registry, + key, + all, + fail_warned, + local, + passed, + sort_by, + ) # nf-core subworkflows info @subworkflows.command("info") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1505,7 +1741,13 @@ def command_subworkflows_info(ctx, subworkflow, directory): # nf-core subworkflows install @subworkflows.command("install") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1545,7 +1787,13 @@ def command_subworkflows_install(ctx, subworkflow, directory, prompt, force, sha # nf-core subworkflows remove @subworkflows.command("remove") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1564,7 +1812,13 @@ def command_subworkflows_remove(ctx, directory, subworkflow): # nf-core subworkflows update @subworkflows.command("update") @click.pass_context -@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument( + "subworkflow", + type=str, + callback=normalize_case, + required=False, + metavar="subworkflow name", +) @click.option( "-d", "--dir", @@ -1643,7 +1897,17 @@ def command_subworkflows_update( Update DSL2 subworkflow within a pipeline. """ subworkflows_update( - ctx, subworkflow, directory, force, prompt, sha, install_all, preview, save_diff, update_deps, limit_output + ctx, + subworkflow, + directory, + force, + prompt, + sha, + install_all, + preview, + save_diff, + update_deps, + limit_output, ) @@ -1772,7 +2036,14 @@ def command_schema_docs(schema_path, output, format, force, columns): # nf-core create-logo (deprecated) @nf_core_cli.command("create-logo", deprecated=True, hidden=True) @click.argument("logo-text", metavar="") -@click.option("-d", "--dir", "directory", type=click.Path(), default=".", help="Directory to save the logo in.") +@click.option( + "-d", + "--dir", + "directory", + type=click.Path(), + default=".", + help="Directory to save the logo in.", +) @click.option( "-n", "--name", @@ -1849,14 +2120,30 @@ def command_create_logo(logo_text, directory, name, theme, width, format, force) @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") -def command_sync(directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr): +def command_sync( + directory, + from_branch, + pull_request, + github_repository, + username, + template_yaml, + force_pr, +): """ Use `nf-core pipelines sync` instead. """ log.warning( "The `[magenta]nf-core sync[/]` command is deprecated. Use `[magenta]nf-core pipelines sync[/]` instead." ) - pipelines_sync(directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr) + pipelines_sync( + directory, + from_branch, + pull_request, + github_repository, + username, + template_yaml, + force_pr, + ) # nf-core bump-version (deprecated) @@ -1976,7 +2263,18 @@ def command_launch( log.warning( "The `[magenta]nf-core launch[/]` command is deprecated. Use `[magenta]nf-core pipelines launch[/]` instead." ) - pipelines_launch(ctx, pipeline, id, revision, command_only, params_in, params_out, save_all, show_hidden, url) + pipelines_launch( + ctx, + pipeline, + id, + revision, + command_only, + params_in, + params_out, + save_all, + show_hidden, + url, + ) # nf-core create-params-file (deprecated) @@ -2202,7 +2500,19 @@ def command_lint( log.warning( "The `[magenta]nf-core lint[/]` command is deprecated. Use `[magenta]nf-core pipelines lint[/]` instead." ) - pipelines_lint(ctx, directory, release, fix, key, show_passed, fail_ignored, fail_warned, markdown, json, sort_by) + pipelines_lint( + ctx, + directory, + release, + fix, + key, + show_passed, + fail_ignored, + fail_warned, + markdown, + json, + sort_by, + ) # nf-core create (deprecated) @@ -2216,7 +2526,13 @@ def command_lint( @click.option("-d", "--description", type=str, help="A short description of your pipeline") @click.option("-a", "--author", type=str, help="Name of the main author(s)") @click.option("--version", type=str, default="1.0.0dev", help="The initial version number to use") -@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") +@click.option( + "-f", + "--force", + is_flag=True, + default=False, + help="Overwrite output directory if it already exists", +) @click.option("-o", "--outdir", help="Output directory for new pipeline (default: pipeline name)") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") @click.option("--plain", is_flag=True, help="Use the standard nf-core template") @@ -2227,14 +2543,35 @@ def command_lint( help="The name of the GitHub organisation where the pipeline will be hosted (default: nf-core)", ) @click.pass_context -def command_create(ctx, name, description, author, version, force, outdir, template_yaml, plain, organisation): +def command_create( + ctx, + name, + description, + author, + version, + force, + outdir, + template_yaml, + plain, + organisation, +): """ Use `nf-core pipelines create` instead. """ log.warning( "The `[magenta]nf-core create[/]` command is deprecated. Use `[magenta]nf-core pipelines create[/]` instead." ) - pipelines_create(ctx, name, description, author, version, force, outdir, template_yaml, organisation) + pipelines_create( + ctx, + name, + description, + author, + version, + force, + outdir, + template_yaml, + organisation, + ) # Main script is being run - launch the CLI diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py index 57c8e9777c..30e8137cfe 100644 --- a/nf_core/commands_modules.py +++ b/nf_core/commands_modules.py @@ -175,6 +175,7 @@ def modules_create( conda_package_version, empty_template, migrate_pytest, + migrate_pytest_hard, ): """ Create a new DSL2 module from the nf-core template. @@ -194,6 +195,10 @@ def modules_create( elif no_meta: has_meta = False + if migrate_pytest_hard and not migrate_pytest: + log.error("--migrate_pytest_hard can only allowed in combination with --migrate_pytest.") + sys.exit(1) + from nf_core.modules.create import ModuleCreate # Run function @@ -209,6 +214,7 @@ def modules_create( conda_package_version, empty_template, migrate_pytest, + migrate_pytest_hard, ) module_create.create() except UserWarning as e: @@ -219,7 +225,17 @@ def modules_create( sys.exit(1) -def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrate_pytest): +def modules_test( + ctx, + tool, + directory, + no_prompts, + update, + once, + profile, + migrate_pytest, + migrate_pytest_hard, +): """ Run nf-test for a module. @@ -227,6 +243,10 @@ def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrat """ from nf_core.components.components_test import ComponentsTest + if migrate_pytest_hard and not migrate_pytest: + log.error("--migrate_pytest_hard can only allowed in combination with --migrate_pytest.") + sys.exit(1) + if migrate_pytest: modules_create( ctx, @@ -241,6 +261,7 @@ def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrat conda_package_version=None, empty_template=False, migrate_pytest=migrate_pytest, + migrate_pytest_hard=migrate_pytest_hard, ) try: module_tester = ComponentsTest( @@ -261,7 +282,19 @@ def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrat sys.exit(1) -def modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version): +def modules_lint( + ctx, + tool, + directory, + registry, + key, + all, + fail_warned, + local, + passed, + sort_by, + fix_version, +): """ Lint one or more modules in a directory. diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c71b128415..7e33c93cc0 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -40,6 +40,7 @@ def __init__( conda_version: Optional[str] = None, empty_template: bool = False, migrate_pytest: bool = False, + migrate_pytest_hard=False, ): super().__init__(component_type, directory) self.directory = directory @@ -61,6 +62,9 @@ def __init__( self.file_paths: Dict[str, Path] = {} self.not_empty_template = not empty_template self.migrate_pytest = migrate_pytest + self.migrate_pytest_hard = migrate_pytest_hard + self.pytest_units_str = None + self.pytest_has_nextflow_config = False def create(self) -> bool: """ @@ -137,6 +141,23 @@ def create(self) -> bool: # Check existence of directories early for fast-fail self.file_paths = self._get_component_dirs() + component_outputs = {} # output: meta_data + if self.migrate_pytest and self.migrate_pytest_hard: + # Extract outputs data + main_nf_data = Path( + self.directory, self.component_type, self.org, self.component_dir, "main.nf" + ).read_text() + if "output:" not in main_nf_data: + log.debug(f"Could not find any outputs in {self.component_name}") + else: + component_outputs_data = main_nf_data.split("output:")[1].split("when:")[0] + matches = re.findall(r"(.*)emit:\s*([^)\s,]+)", component_outputs_data, re.MULTILINE) + for match in matches: + component_outputs[match[1]] = match[0] + log.debug( + f"Found {len(component_outputs)} outputs in {self.component_name}: {component_outputs.keys()}" + ) + if self.migrate_pytest: # Rename the component directory to old component_old_dir = Path(str(self.component_dir) + "_old") @@ -160,12 +181,22 @@ def create(self) -> bool: not_alphabet = re.compile(r"[^a-zA-Z]") self.org_alphabet = not_alphabet.sub("", self.org) + # Extract pytest nextflow config + pytest_nextflow_config_contents = None + if self.migrate_pytest: + pytest_nextflow_config_contents = self._extract_nextflow_config() + + # Extract pytest units + if self.migrate_pytest and self.migrate_pytest_hard: + self._extract_pytest_units(component_outputs) + # Create component template with jinja2 assert self._render_template() log.info(f"Created component template: '{self.component_name}'") if self.migrate_pytest: self._copy_old_files(component_old_path) + self._copy_nextflow_config(pytest_nextflow_config_contents) log.info("Migrate pytest tests: Copied original module files to new module") shutil.rmtree(component_old_path) self._print_and_delete_pytest_files() @@ -458,6 +489,8 @@ def _copy_old_files(self, component_old_path): component_old_path / "templates", self.file_paths["environment.yml"].parent / "templates", ) + + def _extract_nextflow_config(self): # Create a nextflow.config file if it contains information other than publishDir pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) nextflow_config = pytest_dir / "nextflow.config" @@ -468,20 +501,29 @@ def _copy_old_files(self, component_old_path): if "publishDir" not in line and line.strip() != "": config_lines += line # if the nextflow.config file only contained publishDir, non_publish_dir_lines will be 11 characters long (`process {\n}`) - if len(config_lines) > 11: - log.debug("Copying nextflow.config file from pytest tests") - with open( - Path( - self.directory, - self.component_type, - self.org, - self.component_dir, - "tests", - "nextflow.config", - ), - "w+", - ) as ofh: - ofh.write(config_lines) + if not re.match(r"^\s*process\s*{\s*}\s*$", config_lines, re.DOTALL): + self.pytest_has_nextflow_config = True + return config_lines + + return None + + def _copy_nextflow_config(self, config_lines): + if config_lines is None: + return + + log.debug("Copying nextflow.config file from pytest tests") + with open( + Path( + self.directory, + self.component_type, + self.org, + self.component_dir, + "tests", + "nextflow.config", + ), + "w+", + ) as ofh: + ofh.write(config_lines) def _print_and_delete_pytest_files(self): """Prompt if pytest files should be deleted and printed to stdout""" @@ -519,3 +561,261 @@ def _print_and_delete_pytest_files(self): with open(modules_yml, "w") as fh: yaml.dump(yml_file, fh) run_prettier_on_file(modules_yml) + + def _extract_pytest_units(self, component_outputs): + pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) + main_nf_contents = Path(pytest_dir, "main.nf").read_text(encoding="UTF-8") + + include_statements = re.findall(r"include\s*{\s*(\w+)([\sas]+)?(\w+)?\s*}", main_nf_contents) + + log.debug(f"Found {len(include_statements)} include statements {include_statements}") + + if len(include_statements) > 1: + raise ValueError("Multiple include statements are not yet supported") + + main_nf_workflows = re.findall(r"workflow\s*(\w+)\s*{([^}]+)}", main_nf_contents, re.DOTALL) + + log.debug(f"Found {len(main_nf_workflows)} workflows {[x[0] for x in main_nf_workflows]}") + + nf_test_workflow = [] + for workflow in main_nf_workflows: + workflow_name = workflow[0] + workflow_content = str(workflow[1]) + + log.debug(f"Looking for NXF symbols in workflow {workflow_name}") + + nxf_symbols = self._extract_pytest_nxf_symbols(workflow_content) + + invoked_components = re.findall(r"(\w+)\s*\(([^\)]*)\)", workflow_content, re.DOTALL) + + invoked_components = [c for c in invoked_components if c[0] != "file"] + + log.debug( + f"Found {len(invoked_components)} component(s) invoked by '{workflow_name}': {invoked_components}" + ) + + if len(invoked_components) > 1: + raise ValueError( + f"Test workflow {workflow_name} invokes multiple components. This is not supported currently." + ) + + # TODO: Generalize to multiple components + invoked_component = invoked_components[0] + + # invoked_component_name = invoked_component[0].strip() + invoked_component_args = invoked_component[1].strip() + + extracted_component_args = self._extract_pytest_component_args(invoked_component_args, nxf_symbols) + + nf_test_workflow.append({"name": workflow_name.replace("_", "-"), "input": extracted_component_args}) + + test_units_str = "" + for test in nf_test_workflow: + test_name = test["name"] + log.debug(f"Scaffolding nf-test '{test_name}'") + + input_data_lines = self._make_nf_test_input(test["input"]) + add_stub_option = "options '-stub'" if "stub" in test_name else "" + + power_assertions = self._get_power_assertions(component_outputs, is_stub=("stub" in test_name)) + + test_unit_str = f""" + test("{test_name}") {{ + {add_stub_option} + when {{ + process {{ + \"\"\" + {input_data_lines} + \"\"\" + }} + }} + + then {{ + assertAll( + {{ assert process.success }}, + {power_assertions} + ) + }} + }} + """ + test_units_str += test_unit_str + + self.pytest_units_str = test_units_str + + def _extract_pytest_nxf_symbols(self, workflow_content: str) -> dict[str, str]: + symbols = {} + found_all = False + + remaining_content = workflow_content + while not found_all: + match = self._extract_pytest_nxf_symbol_match(remaining_content) + if match: + match = match[0] + + log.debug(f"Found symbol '{match[0]}' with data {match[1]}") + remaining_content = remaining_content.replace(f"{match[0]}={match[1]}", "", 1) + symbols[match[0].strip()] = match[1].strip() + continue + + found_all = True + + return symbols + + def _extract_pytest_nxf_symbol_match(self, workflow_content): + # multiline list such as input = [etc] + list_match = re.findall(r"(\w+\s*)=(\s*\[.*?\n\s*\])", workflow_content, re.DOTALL) + + if list_match != []: + return list_match + + # second rule for multiline list such as input = [etc] + # The first rule might not be needed! + list_match = re.findall(r"(\w+\s*)=(\s*\[[^=]+\]\s+)", workflow_content, re.DOTALL) + + if list_match != []: + return list_match + + # simple list such as input = [ ] + list_match = re.findall(r"(\w+\s*)=(\s*\[\s*\])", workflow_content, re.DOTALL) + + if list_match != []: + return list_match + + # String match such as 'etc', "etc" + string_match = re.findall(r"(\w+\s*)=(\s*['\"][^'\"]*['\"])", workflow_content) + + if string_match != []: + return string_match + + # Number match such as 123.1 + num_match = re.findall(r"(\w+\s*)=(\s*[\d\.]+)", workflow_content, re.DOTALL) + + if num_match != []: + return num_match + + # File match such as file(params.test_data['sarscov2']['genome']['transcriptome_fasta'], checkIfExists: true) + file_match = re.findall(r"(\w+\s*)=(\s*file\s*\(.*?\s*\))", workflow_content, re.DOTALL) + + if file_match != []: + return file_match + + return [] + + def _make_nf_test_input(self, input_data): + input_data_lines = "" + for index in range(len(input_data)): + arg_data = input_data[index] + if "\n" in arg_data: + input_data_str = self._indent_nf_test_arg(arg_data) + else: + input_data_str = arg_data.strip() + + indent = "" + if index > 0: + indent = "\t\t\t\t" + input_data_lines += indent + f"input[{index}] = " + input_data_str + "\n" + + return input_data_lines + + def _indent_nf_test_arg(self, arg_data): + arg_data_lines = arg_data.split("\n") + + return arg_data_lines[0].strip() + "\n" + "\n".join(["\t\t\t\t" + line.strip() for line in arg_data_lines[1:]]) + + def _extract_pytest_component_args(self, args_str: str, nxf_symbols: dict[str, str]) -> list[str]: + # No arg + if args_str == "": + return [] + + # Single argument + if args_str in nxf_symbols.keys(): + return [nxf_symbols[args_str]] + + # All arguments are named + match = re.match(r"^[\sa-zA-Z_,]+$", args_str, re.DOTALL) + + if match: + # Double check that any arg name is not on the prohibited list + args_list = args_str.split(",") + prohibited_names = ["false", "true"] + has_prohibited = any([arg.strip() in prohibited_names for arg in args_list]) + if not has_prohibited: + return [nxf_symbols[arg.strip()] for arg in args_str.split(",")] + + # Split args while keeping brackets grouped + args = re.findall(r"\[.+\]|\w+|\[\]|[\w'\"]+", args_str) + + if not args: + raise ValueError(f"Can not split args: {args_str}") + + log.debug(f"Split args: {args_str} into: {args}") + + # Replace the symbols embedded in list args + normalized_args = [] + for arg in args: + if "[" in arg: + normalized_arg = self._replace_pytest_symbol_in_list_arg(arg, nxf_symbols) + normalized_args.append(normalized_arg) + continue + + # For remaining args, try to replace symbol if possible + normalized_args.append(nxf_symbols[arg] if arg in nxf_symbols.keys() else arg) + + return normalized_args + + def _replace_pytest_symbol_in_list_arg(self, arg: str, nxf_symbols: dict[str, str]) -> str: + symbols = re.findall(r"[a-zA-Z_]+", arg) + + for symbol in symbols: + arg = arg.replace(symbol, nxf_symbols[symbol] if symbol in nxf_symbols.keys() else symbol, 1) + + return arg + + def _get_power_assertions(self, component_outputs, is_stub): + power_assertions = "{ assert snapshot(process.out).match() }" + + if is_stub: + return power_assertions + + non_stable_outputs = ["bam", "txt", "log", "gz", "rds", "png"] + + outputs_str = " ".join([f"{key} {value}" for (key, value) in component_outputs.items()]).lower() + has_non_stable = any([ns_output in outputs_str for ns_output in non_stable_outputs]) + + if not has_non_stable: + return power_assertions + + power_assertions = "{ assert snapshot(" + for output_name, output_meta in component_outputs.items(): + if output_name == "versions": + continue + + if "bam" in output_name or "bam" in output_meta: + power_assertions += f"\n\t\t\t\t\tbam(process.out.{output_name}[0][1]).getReadsMD5()," + continue + + if "log" in output_name or "log" in output_meta: + power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).name," + continue + + if "gz" in output_name or "gz" in output_meta: + power_assertions += f"\n\t\t\t\t\tpath(process.out.{output_name}[0][1]).linesGzip[3..7]," + continue + + if "txt" in output_name or "txt" in output_meta: + power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).readLines()[3..7]," + continue + + if "rds" in output_name or "rds" in output_meta: + power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).name," + continue + + if "png" in output_name or "png" in output_meta: + power_assertions += f"\n\t\t\t\t\tfile(process.out.{output_name}[0][1]).name," + continue + + power_assertions += f"\n\t\t\t\t\tprocess.out.{output_name}," + + power_assertions += "\n\t\t\t\t\tprocess.out.versions\n\t\t\t\t\t).match()\n\t\t\t\t}" + + return power_assertions diff --git a/nf_core/module-template/tests/main.nf.test.j2 b/nf_core/module-template/tests/main.nf.test.j2 index a50ecc6a07..c29585b52b 100644 --- a/nf_core/module-template/tests/main.nf.test.j2 +++ b/nf_core/module-template/tests/main.nf.test.j2 @@ -1,10 +1,15 @@ +{%- if not migrate_pytest_hard %} // TODO nf-core: Once you have added the required tests, please run the following command to build this file: // nf-core modules test {{ component_name }} +{%- endif %} nextflow_process { name "Test Process {{ component_name_underscore|upper }}" script "../main.nf" process "{{ component_name_underscore|upper }}" + {%- if pytest_has_nextflow_config %} + config "./nextflow.config" + {%- endif %} tag "modules" tag "modules_{{ org_alphabet }}" @@ -13,6 +18,7 @@ nextflow_process { {%- endif %} tag "{{ component_name }}" + {%- if not migrate_pytest_hard %} // TODO nf-core: Change the test name preferably indicating the test-data and file-format used test("sarscov2 - bam") { @@ -78,5 +84,9 @@ nextflow_process { } } + {%- endif %} +{%- if migrate_pytest_hard %} +{{ pytest_units_str }} +{%- endif %} } diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index a5e0795a9f..208b8bc667 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -18,6 +18,7 @@ def __init__( conda_version=None, empty_template=False, migrate_pytest=False, + migrate_pytest_hard=False, ): super().__init__( "modules", @@ -31,4 +32,5 @@ def __init__( conda_version, empty_template, migrate_pytest, + migrate_pytest_hard, )