From b065fa6ee953986908fa7f951f3166580bd31dde Mon Sep 17 00:00:00 2001 From: Tim Band Date: Mon, 22 Jan 2024 13:13:36 +0000 Subject: [PATCH 01/10] `qw configure` split into --rules and --workflow Requirements can now have multiple User Needs --- src/qw/cli.py | 68 ++++++++++++++++++++++++++--------- src/qw/design_stages/_base.py | 12 +++++++ src/qw/design_stages/main.py | 22 +++++------- src/qw/local_store/main.py | 2 +- tests/test_cli.py | 16 ++++----- tests/test_design_stages.py | 57 +++++++++++++++++++++++++++++ 6 files changed, 137 insertions(+), 40 deletions(-) create mode 100644 tests/test_design_stages.py diff --git a/src/qw/cli.py b/src/qw/cli.py index 4a05960..088a26e 100644 --- a/src/qw/cli.py +++ b/src/qw/cli.py @@ -145,31 +145,46 @@ def check( issue: Annotated[ Optional[int], typer.Option( - help="Issue number to check", + help="Issue number to check (default is all of them)", + show_default=False, ), ] = None, review_request: Annotated[ Optional[int], typer.Option( - help="Review request number to check", + help="Review request number to check (default is all of them)", + show_default=False, ), ] = None, token: Annotated[ Optional[str], typer.Option( - help="CI access token to use for checking.", + help=( + "CI access token to use for checking, if not run from a" + " git repository." + ), + show_default=False, ), ] = None, repository: Annotated[ Optional[str], typer.Option( - help="Repository URL (like https://github.com/me/repo or github.com:me/repo)", + help=( + "Repository URL (like https://github.com/me/repo or" + " github.com:me/repo) if not run from a git repository." + ), + show_default=False, ), ] = None, remote: Annotated[ bool, typer.Option( - help="Use remote repository rather than local store (implied by --repository)", + "--remote/--local", + help=( + "Use remote repository or local store (default is --remote" + " if --repository is supplied or --local otherwise)" + ), + show_default=False, ), ] = False, ) -> None: @@ -281,18 +296,30 @@ def configure( force: Annotated[ Optional[bool], typer.Option( + "--force", help="Replace existing configuration.", + show_default=False, ), ] = False, - ci: Annotated[ + rules: Annotated[ Optional[bool], typer.Option( + "--rules", help=( - "Configure service's continuous integration" - " (default unless --release-templates is provided)" + "Configure service's branch rules" + " (Note: this will prevent pushes to the default branch!)" ), + show_default=False, ), - ] = None, + ] = False, + workflow: Annotated[ + Optional[bool], + typer.Option( + "--workflow", + help="Creates workflows, issue templates and PR templates.", + show_default=False, + ), + ] = False, release_templates: Annotated[ list[LocalStore.ReleaseTemplateSet], typer.Option( @@ -300,34 +327,41 @@ def configure( "Release file template sets to install in qw_release_templates" " (and so be used by qw release)" ), + show_default=False, ), # Stop mypy, ruff and black from fighting each other. ] = [], # noqa: B006 ): """Configure remote repository for qw (after initialisation and login credentials added).""" + if not (rules or workflow or release_templates): + typer.echo( + "Nothing to do. Please provide at least one option from" + " --service-templates, --release-templates, or --ci", + ) + return 0 service = _build_and_check_service() - if ci is None: - ci = not bool(release_templates) - done = False - if ci: + repo_updated = False + if workflow: store.write_templates_and_ci(service, force=force) - done = True + repo_updated = True for template_set in release_templates: store.write_release_document_templates( service, template_set, force=force, ) - done = True - if done: + repo_updated = True + if repo_updated: typer.echo( "Local repository updated, please commit the changes made to your local repository.", ) - if ci: + if rules: service.update_remote(force=force) typer.echo( "Updated remote repository with rules", ) + return None + return None @app.command() diff --git a/src/qw/design_stages/_base.py b/src/qw/design_stages/_base.py index bdc152f..efc1d6f 100644 --- a/src/qw/design_stages/_base.py +++ b/src/qw/design_stages/_base.py @@ -1,4 +1,5 @@ """Base class for design stages.""" +import re from abc import ABC from collections.abc import Callable from copy import copy @@ -18,6 +19,8 @@ class DesignBase(ABC): ) design_stage: DesignStage | None = None + LINK_RE = re.compile(r"#(\d+)") + def __init__(self) -> None: """Shared fields for all design stage classes.""" self.title: str | None = None @@ -40,6 +43,15 @@ def _validate_required_fields(self): msg = f"No {field} in {self.__class__.__name__}" raise QwError(msg) + def get_links_from_text(self, text) -> list[int]: + """Get #IDs contained in text.""" + if text is None: + return [] + return [ + int(match) + for match in self.LINK_RE.findall(text) + ] + def to_dict(self) -> dict[str, Any]: """ Serialise data to dictionary. diff --git a/src/qw/design_stages/main.py b/src/qw/design_stages/main.py index ee296db..1a10049 100644 --- a/src/qw/design_stages/main.py +++ b/src/qw/design_stages/main.py @@ -123,6 +123,10 @@ def is_dict_reference(cls, self_dict, from_stage_name): return lambda d: internal_id in d.get("closing_issues", []) return None + def user_need_links(self) -> list[int]: + """Get IDs of user needs linked to.""" + return self.get_links_from_text(self.user_need) + @check( "User need links have qw-user-need label", "Requirement {0.internal_id} ({0.title}) has bad user need:", @@ -135,25 +139,15 @@ def user_need_is_labelled_as_such(self, user_needs, **_kwargs) -> list[str]: Design Outputs (PRs) have closing issues, and all these must refer to issues with the qw-requirement label (or qw-ignore). """ - if isinstance(self.user_need, str) and self.user_need.startswith("#"): - un = self.user_need[1:] - if un.isnumeric() and int(un) not in user_needs: - return [self.user_need] - return [] + return [str(un) for un in self.user_need_links() if un not in user_needs] @check( "User Need links must exist", - "Requirement {0.internal_id} ({0.title}) has no user need:", + "Requirement {0.internal_id} ({0.title}) has no user need.", ) def user_need_must_exit(self, **_kwargs) -> bool: - """Test if the User Needs are actually links to Github issues.""" - if ( - isinstance(self.user_need, str) - and self.user_need.startswith("#") - and self.user_need[1:].isnumeric() - ): - return False - return True + """Test that there is at least one User Need link.""" + return len(self.user_need_links()) == 0 class DesignOutput(DesignBase): diff --git a/src/qw/local_store/main.py b/src/qw/local_store/main.py index ed17d63..b08d81c 100644 --- a/src/qw/local_store/main.py +++ b/src/qw/local_store/main.py @@ -159,7 +159,7 @@ def write_templates_and_ci(self, service: GitService, *, force: bool): should_not_exist.append(target_path) if should_not_exist: - msg = f"Templates already exists, rerun with '--force' to override existing templates:'n {should_not_exist}" + msg = f"Templates already exist, rerun with '--force' to override existing templates:'n {should_not_exist}" raise QwError(msg) for source_path, target_path in zip( diff --git a/tests/test_cli.py b/tests/test_cli.py index 7134941..c843182 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -105,10 +105,10 @@ def test_configure_adds_templates(mocked_store): """ Given no templates exist in git root (tmpdir). - When `qw configure` is run + When `qw configure --workflow` is run Then templates should be copied and the cli should exit without error """ - result = runner.invoke(app, ["configure"]) + result = runner.invoke(app, ["configure", "--workflow"]) requirements_template = ( mocked_store.base_dir / ".github" / "ISSUE_TEMPLATE" / "requirement.yml" @@ -123,7 +123,7 @@ def test_configure_adds_requirement_components(mocked_store): """ Given no templates exist in git root (tmpdir) and custom components with leading and trailing whitespace. - When `qw configure` is run + When `qw configure --workflow` is run Then requirement template should have the component names in the dropdown, without the whitespace. """ components_file = mocked_store.qw_dir / "components.csv" @@ -135,7 +135,7 @@ def test_configure_adds_requirement_components(mocked_store): mocked_store.qw_dir, ) - result = runner.invoke(app, ["configure"]) + result = runner.invoke(app, ["configure", "--workflow"]) bullet_point = " - " component_options = ( @@ -161,9 +161,9 @@ def test_configure_throws_if_templates_exist(mocked_store): existing_file.parent.mkdir(parents=True) existing_file.write_text("Now I exist.") - result = runner.invoke(app, ["configure"]) + result = runner.invoke(app, ["configure", "--workflow"]) - assert "Templates already exists" in " ".join(result.exception.args) + assert "Templates already exist" in " ".join(result.exception.args) assert str(existing_file) in " ".join(result.exception.args) assert result.exit_code != 0 assert not (mocked_store.base_dir / ".github" / "PULL_REQUEST_TEMPLATE.md").exists() @@ -173,7 +173,7 @@ def test_configure_force_templates_exist(mocked_store): """ Given the pull request template exists already. - When `qw configure --force` is run + When `qw configure --force --workflow` is run Then an exception should be thrown and the other templates should not exist """ existing_file = ( @@ -182,7 +182,7 @@ def test_configure_force_templates_exist(mocked_store): existing_file.parent.mkdir(parents=True) existing_file.write_text("Now I exist.") - result = runner.invoke(app, ["configure", "--force"]) + result = runner.invoke(app, ["configure", "--force", "--workflow"]) assert (mocked_store.base_dir / ".github" / "PULL_REQUEST_TEMPLATE.md").exists() assert result.exit_code == 0 diff --git a/tests/test_design_stages.py b/tests/test_design_stages.py new file mode 100644 index 0000000..eefc575 --- /dev/null +++ b/tests/test_design_stages.py @@ -0,0 +1,57 @@ +"""Testing of common service functions.""" +import pytest + +from qw.design_stages.main import Requirement + + +@pytest.mark.parametrize( + ("store", "results"), + [ + ( + { + "component": "System", + "description": "Desc", + "internal_id": 4, + "remote_item_type": "issue", + "req_type": "Functional", + "stage": "requirement", + "title": "Control", + "user_need": "#45", + "version": 1, + }, + [45], + ), + ( + { + "component": "System", + "description": "Desc", + "internal_id": 5, + "remote_item_type": "issue", + "req_type": "Functional", + "stage": "requirement", + "title": "Control", + "user_need": "Mostly #34 but also #322. #76 and #119", + "version": 1, + }, + [34, 322, 76, 119], + ), + ( + { + "component": "System", + "description": "Desc", + "internal_id": 6, + "remote_item_type": "issue", + "req_type": "Functional", + "stage": "requirement", + "title": "Control", + "user_need": "#1\n#2", + "version": 2, + }, + [1, 2], + ), + ], +) +def test_requirement_user_need_links(store, results): + """Test requirement's user need links are extracted.""" + req = Requirement.from_dict(store) + assert results == req.user_need_links() From a52b063fd73147e3f44a26502789300292a0fb41 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Tue, 23 Jan 2024 11:02:44 +0000 Subject: [PATCH 02/10] Freeze output includes additions and deletions --- src/qw/changes.py | 42 ++++++++++++++++++++++++++++------- src/qw/cli.py | 2 +- src/qw/design_stages/_base.py | 15 ++++++++----- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/qw/changes.py b/src/qw/changes.py index 4cd0020..8f0a621 100644 --- a/src/qw/changes.py +++ b/src/qw/changes.py @@ -39,7 +39,28 @@ def __init__( def show(self): """Show this difference on the screen.""" if not bool(self._diff): - return + if self._local_item is None: + Console().print( + "[green]New item ({}) #{}: {}[/]".format( + self._remote_item.stage.value, + self._remote_item.internal_id, + self._remote_item.title.replace("[", "\\["), + ), + ) + return self._remote_item + if self._remote_item is not None: + # Both exist, but no difference + return self._local_item + if not self._local_item.is_marked_deleted(): + # Only a local item, and it has not yet been marked as deleted + Console().print( + "[red]Removed item ({}) #{}: {}[/]".format( + self._local_item.stage.value, + self._local_item.internal_id, + self._local_item.title.replace("[", "\\["), + ), + ) + return None table = Table( title=f"Changes detected for {self._local_item}:", show_lines=True, @@ -56,6 +77,7 @@ def show(self): table.add_row(field, differences["self"], differences["other"]) Console().print(table) + return None def prompt_for_version_change(self): """ @@ -72,13 +94,17 @@ def prompt_for_version_change(self): if self._remote_item is not None: # Both exist, but no difference return self._local_item - # Only local exists, remote has been deleted - if Confirm.ask( - f"{self._local_item} no longer exists in remote, would you like to remove it from the local store?", - ): - # Remove the local item - return None - # Keep the local item + if not self._local_item.is_marked_deleted(): + # Only local exists, remote has been deleted, + # user has not requested to keep it yet + if Confirm.ask( + f"{self._local_item} no longer exists in remote," + " would you like to remove it from the local store?", + ): + # Remove the local item + return None + # Keep the local item + self._local_item.mark_as_deleted() return self._local_item prompt = "\n".join( [ diff --git a/src/qw/cli.py b/src/qw/cli.py index 088a26e..2afeb94 100644 --- a/src/qw/cli.py +++ b/src/qw/cli.py @@ -283,10 +283,10 @@ def freeze( service = get_service(conf) change_handler = ChangeHandler(service, store) diff_elements = change_handler.diff_remote_and_local_items() + to_save = change_handler.get_local_items_from_diffs(diff_elements) if dry_run: logger.info("Finished freeze (dry run)") else: - to_save = change_handler.get_local_items_from_diffs(diff_elements) store.write_local_data([x.to_dict() for x in to_save]) logger.info("Finished freeze") diff --git a/src/qw/design_stages/_base.py b/src/qw/design_stages/_base.py index efc1d6f..26dddd0 100644 --- a/src/qw/design_stages/_base.py +++ b/src/qw/design_stages/_base.py @@ -47,10 +47,7 @@ def get_links_from_text(self, text) -> list[int]: """Get #IDs contained in text.""" if text is None: return [] - return [ - int(match) - for match in self.LINK_RE.findall(text) - ] + return [int(match) for match in self.LINK_RE.findall(text)] def to_dict(self) -> dict[str, Any]: """ @@ -107,7 +104,7 @@ def diff(self, other: Self) -> dict[str, dict[str, str]]: """ Compare the data of each field with another instance, returning the fields with differences only. - Ignores the version as this is only stored locally. + Ignores the version number and deleted flag as these are only stored locally. :param other: Another instance of the same class :raises ValueError: if other is not the same class as self. @@ -119,7 +116,7 @@ def diff(self, other: Self) -> dict[str, dict[str, str]]: output_fields = {} for field_name in self.__dict__: - if field_name == "version": + if field_name in ["version", "deleted"]: continue self_data = getattr(self, field_name) other_data = getattr(other, field_name) @@ -129,3 +126,9 @@ def diff(self, other: Self) -> dict[str, dict[str, str]]: output_fields[field_name]["other"] = str(other_data) return output_fields + + def mark_as_deleted(self): + self.deleted = True + + def is_marked_deleted(self): + return hasattr(self, "deleted") and self.deleted From 978065306e4520d31e834fc9db1370fcdb07af6b Mon Sep 17 00:00:00 2001 From: Tim Band Date: Tue, 23 Jan 2024 12:10:43 +0000 Subject: [PATCH 03/10] --version option --- src/qw/cli.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/qw/cli.py b/src/qw/cli.py index 2afeb94..284f156 100644 --- a/src/qw/cli.py +++ b/src/qw/cli.py @@ -12,8 +12,10 @@ import git import typer from loguru import logger +from rich.console import Console from rich.prompt import Prompt +from qw._version import __version__ from qw.base import QwError from qw.changes import ChangeHandler from qw.design_stages.checks import run_checks @@ -45,6 +47,14 @@ class LogLevel(str, Enum): INFO = "info" DEBUG = "debug" + def __str__(self): + """ + Return the value. + + This makes Typer print the correct thing as a default value. + """ + return self.value + LOGLEVEL_TO_LOGURU = { LogLevel.DEBUG: 10, @@ -65,8 +75,24 @@ def _build_and_check_service(conf: dict | None = None): return service +def version_callback(value): + """Print the version and exit.""" + if value: + Console().print(f"qw version: [cyan]{__version__}[/]", highlight=False) + raise typer.Exit + + @app.callback() def main( + _version: Annotated[ + Optional[bool], + typer.Option( + "--version", + help="Print the version and exit", + callback=version_callback, + is_eager=True, + ), + ] = None, loglevel: Annotated[ LogLevel, typer.Option( From f2ef5fa0bce75a218c714b0c93e34359619b5f01 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Wed, 24 Jan 2024 12:58:24 +0000 Subject: [PATCH 04/10] Initial attempt to grab PR file paths --- src/qw/remote_repo/_github.py | 44 +++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/qw/remote_repo/_github.py b/src/qw/remote_repo/_github.py index d52edcc..3ed312c 100644 --- a/src/qw/remote_repo/_github.py +++ b/src/qw/remote_repo/_github.py @@ -84,6 +84,12 @@ def __init__( self._number = number self._title = title self._closing_issues = kwargs.get("closingIssuesReferences", []) + self._paths = [] + + def add_paths(self, paths: list[str]) -> None: + self._paths.extend(paths) + for p in paths: + logger.info(p) @property def number(self) -> int: @@ -190,13 +196,47 @@ def get_pull_request(self, number: int) -> PullRequest | None: ) if not status_is_ok(response.status_code): logger.info( - "Failed ({}) to get the closing numbers for issue {}", + "Failed ({}) to get the closing numbers for pull request #{}", response.status_code, number, ) return None result = json.loads(response.content) - return PullRequest(**result["data"]["repository"]["pullRequest"]) + pr = PullRequest(**result["data"]["repository"]["pullRequest"]) + has_next_page = True + cursor = "" + while has_next_page: + response = self._graph_ql( + f"""query {{ +repository(owner: "{self.username}", name: "{self.reponame}") {{ + pullRequest(number: {number}) {{ + files(first: 100{cursor}) {{ + nodes {{ + path + }} + endCursor + hasNextPage + }} + }} +}} +}}""", + ) + if not status_is_ok(response.status_code): + logger.error( + "Failed ({}) to get the files for pull request #{}", + response.status_code, + number, + ) + return None + result = json.loads(response.content) + file_result = result["data"]["repository"]["pullRequest"]["files"] + pr.add_paths([ + node["path"] + for node in file_result["nodes"] + ]) + has_next_page = file_result["hasNextPage"] + cursor = f', after: {file_result["endCursor"]}' + return pr @property def issues(self): From 985a149d1ef9f8b1fdf129e534b61f63b01070a0 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Thu, 25 Jan 2024 15:00:32 +0000 Subject: [PATCH 05/10] freeze dry-run no longer prompts --- src/qw/changes.py | 143 +++++++++++++++++++++++++++++++++++----------- src/qw/cli.py | 12 +++- 2 files changed, 119 insertions(+), 36 deletions(-) diff --git a/src/qw/changes.py b/src/qw/changes.py index 8f0a621..5c0cb36 100644 --- a/src/qw/changes.py +++ b/src/qw/changes.py @@ -1,4 +1,5 @@ """Compares changes between remote and local data, allowing the user to make decisions.""" +from abc import ABC, abstractmethod from collections import OrderedDict, defaultdict from rich.console import Console @@ -10,6 +11,73 @@ from qw.remote_repo.service import GitService +class LocalChangeDeterminer(ABC): + """ + Determines what should happen to local objects. + + When a remote change is detected, the user needs to say what + should happen to the equivalent local object. The user could + respond interactively or select a policy to apply to all + changes. + """ + + @abstractmethod + def should_remove_deleted_object(self, name): + """Ask the user if the object should be removed locally.""" + ... + + @abstractmethod + def version_increment(self): + """ + Ask the user if the version should be incremented. + + Possible return values are: 0 (update but don't increment), + 1 (update and increment), None (don't increment). + """ + ... + + +class LocalChangeInteractive(LocalChangeDeterminer, str): + """Asks the user every time.""" + + def should_remove_deleted_object(self, name): + """Ask the user if they want to delete the object.""" + return Confirm.ask( + f"{name} no longer exists in remote," + " would you like to remove it from the local store?", + ) + + def version_increment(self): + """Ask the user if they want the version incremented.""" + prompt = "\n".join( + [ + "Would you like to:", + "n (Don't save the update)", + "u (Update, but trivial change so don't increment the version)", + "i (Update and increment the version)", + "", + ], + ) + response = Prompt.ask(prompt, choices=["n", "u", "i"]) + if response == "n": + return None + if response == "i": + return 1 + return 0 + + +class LocalChangeNone(LocalChangeDeterminer, str): + """Always says not to change anything.""" + + def should_remove_deleted_object(self, _name): + """Say not to remove.""" + return False + + def version_increment(self): + """Say not to update.""" + return + + class ChangeHandler: """Allow user interaction to manage changes between local and remote design stage data.""" @@ -79,7 +147,28 @@ def show(self): Console().print(table) return None - def prompt_for_version_change(self): + def _version_change_where_remote_deleted( + self, + determiner: LocalChangeDeterminer, + ) -> DesignStages: + """Prompt the user for a deleted object, if appropriate.""" + if self._local_item is None: + return None + if self._local_item.is_marked_deleted(): + # User has already requested to keep it + return self._local_item + # User has not requested to keep it yet + if determiner.should_remove_deleted_object(self._local_item): + # Remove the local item + return None + # Keep the local item + self._local_item.mark_as_deleted() + return self._local_item + + def prompt_for_version_change( + self, + determiner: LocalChangeDeterminer, + ) -> DesignStages: """ Prompt the user for what they want to do with this diff. @@ -87,40 +176,19 @@ def prompt_for_version_change(self): the local item (for no change) or the remote item (possibly with the version number incremented). """ + if self._local_item is None: + # New remote item, no prompt required + return self._remote_item + if self._remote_item is None: + return self._version_change_where_remote_deleted(determiner) if not bool(self._diff): - if self._local_item is None: - # New remote item, no prompt required - return self._remote_item - if self._remote_item is not None: - # Both exist, but no difference - return self._local_item - if not self._local_item.is_marked_deleted(): - # Only local exists, remote has been deleted, - # user has not requested to keep it yet - if Confirm.ask( - f"{self._local_item} no longer exists in remote," - " would you like to remove it from the local store?", - ): - # Remove the local item - return None - # Keep the local item - self._local_item.mark_as_deleted() + # Both exist, but no difference return self._local_item - prompt = "\n".join( - [ - "Would you like to:", - "n (Don't save the update)", - "u (Update, but trivial change so don't increment the version)", - "i (Update and increment the version)", - "", - ], - ) - - response = Prompt.ask(prompt, choices=["n", "u", "i"]) - if response == "n": + # Both exist and there is a difference + update_decision = determiner.version_increment() + if update_decision is None: return self._local_item - if response == "i": - self._remote_item.version += 1 + self._remote_item.version = self._local_item.version + update_decision return self._remote_item def __init__(self, service: GitService, store: LocalStore): @@ -131,7 +199,10 @@ def __init__(self, service: GitService, store: LocalStore): def combine_local_and_remote_items(self) -> list[DesignStages]: """Compare local and remote design stages and prompt on any differences.""" diff_elements = self.diff_remote_and_local_items() - return self.get_local_items_from_diffs(diff_elements) + return self.get_local_items_from_diffs( + diff_elements, + LocalChangeInteractive(), + ) def diff_remote_and_local_items(self) -> list[DiffElement]: """ @@ -158,6 +229,7 @@ def diff_remote_and_local_items(self) -> list[DiffElement]: def get_local_items_from_diffs( cls, diff_elements: list[DiffElement], + determiner: LocalChangeDeterminer, ) -> list[DesignStages]: """ Transform DiffElements into local items to be stored. @@ -165,12 +237,15 @@ def get_local_items_from_diffs( :param diff_elements: An iterable of DiffElements, each representing a difference between the remote and local items. + :param determiner: Determines what to do with each change. :return: A list of local items to be set in the store. """ output_items = [] for diff_element in diff_elements: diff_element.show() - output_item = diff_element.prompt_for_version_change() + output_item = diff_element.prompt_for_version_change( + determiner, + ) if output_item is not None: output_items.append(output_item) diff --git a/src/qw/cli.py b/src/qw/cli.py index 284f156..b409100 100644 --- a/src/qw/cli.py +++ b/src/qw/cli.py @@ -17,7 +17,11 @@ from qw._version import __version__ from qw.base import QwError -from qw.changes import ChangeHandler +from qw.changes import ( + ChangeHandler, + LocalChangeInteractive, + LocalChangeNone, +) from qw.design_stages.checks import run_checks from qw.design_stages.main import ( DESIGN_STAGE_CLASSES, @@ -309,7 +313,11 @@ def freeze( service = get_service(conf) change_handler = ChangeHandler(service, store) diff_elements = change_handler.diff_remote_and_local_items() - to_save = change_handler.get_local_items_from_diffs(diff_elements) + determiner = LocalChangeNone() if dry_run else LocalChangeInteractive() + to_save = change_handler.get_local_items_from_diffs( + diff_elements, + determiner, + ) if dry_run: logger.info("Finished freeze (dry run)") else: From 6579f5fd3fa17e98803e3a75454513f8f7209c8d Mon Sep 17 00:00:00 2001 From: Tim Band Date: Thu, 25 Jan 2024 15:01:10 +0000 Subject: [PATCH 06/10] pr paths examined for non-.qw content --- src/qw/design_stages/main.py | 8 +++++++- src/qw/remote_repo/_github.py | 21 ++++++++++++++++++--- src/qw/remote_repo/service.py | 12 ++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/qw/design_stages/main.py b/src/qw/design_stages/main.py index 1a10049..2566f95 100644 --- a/src/qw/design_stages/main.py +++ b/src/qw/design_stages/main.py @@ -292,12 +292,18 @@ def get_remote_stages(service: Service) -> DesignStages: issue.number, ) for pr in service.pull_requests: - if "qw-ignore" in issue.labels: + if "qw-ignore" in pr.labels: logger.debug( "PR {number} tagged to be ignored, skipping", number=pr.number, ) continue + if pr.changes_only_qw(): + logger.debug( + "PR {number} only affects qw data, skipping", + number=pr.number, + ) + continue output_stages.append(DesignOutput.from_pr(pr)) logger.debug("PR #{} added", pr.number) return output_stages diff --git a/src/qw/remote_repo/_github.py b/src/qw/remote_repo/_github.py index d52edcc..6fc251b 100644 --- a/src/qw/remote_repo/_github.py +++ b/src/qw/remote_repo/_github.py @@ -84,6 +84,7 @@ def __init__( self._number = number self._title = title self._closing_issues = kwargs.get("closingIssuesReferences", []) + self._paths = kwargs.get("files", []) @property def number(self) -> int: @@ -117,6 +118,11 @@ def closing_issues(self) -> list[int]: """Get the list of ID numbers of closing issues for this PR.""" return [ci["number"] for ci in self._closing_issues["nodes"]] + @property + def paths(self) -> list[str]: + """Get the list of paths of files changed in this PR.""" + return [p["path"] for p in self._paths["nodes"]] + LOWEST_HTTP_OK = 200 LOWEST_HTTP_NOT_OK = 300 @@ -190,7 +196,7 @@ def get_pull_request(self, number: int) -> PullRequest | None: ) if not status_is_ok(response.status_code): logger.info( - "Failed ({}) to get the closing numbers for issue {}", + "Failed ({}) to get the issue closing numbers for pull request #{}", response.status_code, number, ) @@ -226,6 +232,15 @@ def pull_requests(self) -> list[PullRequest]: number }} }} + files(first: 100) {{ + nodes {{ + path + }} + pageInfo {{ + endCursor + hasNextPage + }} + }} isDraft labels(last: 100) {{ nodes {{ @@ -258,8 +273,8 @@ def check(self) -> bool: """Check that the credentials can connect to the service.""" try: logger.info( - "There are currently {issues} issues and PRs", - issues=len(self.issues), + "There are currently {count} issues and PRs", + count=len(self.issues) + len(self.pull_requests), ) except ConnectionError as exception: msg = "Could not connect to Github, please check internet connection" diff --git a/src/qw/remote_repo/service.py b/src/qw/remote_repo/service.py index a908003..aaa2d15 100644 --- a/src/qw/remote_repo/service.py +++ b/src/qw/remote_repo/service.py @@ -175,6 +175,8 @@ def item_type(self) -> RemoteItemType: class PullRequest(Issue): """Pull Request.""" + _QW_DATA_RE = re.compile(r"^(.qw|qw_release_templates)/") + @property @abstractmethod def closing_issues(self) -> list[int]: @@ -186,6 +188,16 @@ def closing_issues(self) -> list[int]: """ ... + @property + @abstractmethod + def paths(self) -> list[str]: + """Get the list of paths of files changed by this PR.""" + ... + + def changes_only_qw(self) -> bool: + """Return True if only qw data is affected by this PR.""" + return all(self._QW_DATA_RE.match(p) for p in self.paths) + class GitService(ABC): """A service hosting a git repository and project management tools.""" From 281707e2cffda64c2a1e900c1cea46056e564823 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Thu, 25 Jan 2024 16:49:26 +0000 Subject: [PATCH 07/10] Fixed qw check's return value --- src/qw/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qw/cli.py b/src/qw/cli.py index 284f156..3642980 100644 --- a/src/qw/cli.py +++ b/src/qw/cli.py @@ -256,7 +256,7 @@ def check( logger.error("Some checks failed:") for failure in result.failures: logger.error(failure) - typer.Exit(code=1) + raise typer.Exit(code=1) else: logger.success( "OK: Ran {} check(s) over {} object(s), all successful", From 7afb0dd44df840ca40ac4003bcd78b043a520151 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Tue, 13 Feb 2024 12:29:37 +0000 Subject: [PATCH 08/10] Feature list, conda instructions --- README.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/README.md b/README.md index 8167f74..b3af0df 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,58 @@ These different stages are linked together using github or gitlab as part of the QW also will ensure that versioning of items are updated if their contents have changed, and notify when manual verification should be rerun due to changes in requirements or outputs. +# Actual and Potential Features + +- [x] Manage project on hosted service + - [x] github.com + - [ ] gitlab anywhere +- [x] Issues as regulated objects + - [x] User Needs + - [x] Requirements + - [ ] Hazardous Situation + - [ ] Risk Mitigation + - [ ] Anomaly (bug) template + - [ ] Risk analysis in comment +- [x] Pull Requests as regulated objects + - [x] Design Objects + - [x] Automated workflow + - [x] Cannot merge without review + - [x] Cannot merge without traceability to User Needs + - [ ] User configurability of checks +- [ ] Automated test gathering? + - [ ] Ensure automated tests pass before PR merge? +- [ ] Extra information (in CSV files?) + - [ ] Manual test script description + - [ ] Manual test run results + - [ ] Risk class of each component + - [ ] Risk likelihood, impact, matrix + - [ ] Decision for each risk not entered as a Hazardous Situation issue +- [x] Produce documents from data in repo and service + - [x] MS Word document + - [ ] Markdown document production + - [ ] Excel document production + - [x] "database file" production to allow users to make their own document templates with MS word or LibreOffice + - [ ] Built-in standards documents + - [ ] ISO13485 + - [ ] DCB0129 + - [x] Inserting data into documents + - [x] Data produces repeated, nested paragraphs + - [ ] Data produces repeated rows in tables + - [ ] Data produces charts + - [ ] Management documentation + - [ ] Burndown charts + - [ ] Or burnup charts + - [ ] Requirements satisfied with Design Object (PR) + - [ ] User Needs covered with Requirements + - [ ] User Needs satisfied with Design Objects + - [ ] Anomalies remaining in different risk categories + - [ ] Remaining items report + - [ ] Anomalies + - [ ] Requirements + - [ ] User Needs + - [ ] Risk decisions made/yet to be made + - [ ] Unmet risks + # Setup ## Installation @@ -39,6 +91,17 @@ Install from the source code directory: pipx install . ``` +### Using conda + +After creating and activating your conda environment (with +`conda create` and `conda activate`), install `qw` into +that environment with: + +``` +conda install pip git +pip install git+https://github.com/UCL-ARC/qw +``` + ### Using venv The `qw` tool requires python 3.9 or greater, if your project is in python then we suggest adding it to the developer requirements. From 5fbacd643a89bd0c54968a41523ed105987e26d7 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Thu, 15 Feb 2024 12:23:05 +0000 Subject: [PATCH 09/10] Updated README.md to reflect actual implemented functionality --- README.md | 122 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index b3af0df..c9a6dff 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,8 @@ adding extra branches to the ruleset (e.g. `develop`) You can omit `--service github` if the repo is at `github.com` (as in this case) and you can omit `--repo ` if you have the desired -repository configured as the git remote named `upstream`. +repository configured as the git remote named `upstream` (or `origin` +if you have no `upstream` remote set). #### Setup Personal Access Token @@ -206,13 +207,6 @@ to [add the personal access token](#setup-personal-access-token) ### Customising configuration for identifiers -QW creates identifiers for User Needs and Requirements. - -| Design stage | format | example | -| ------------ | --------------------------------------------------- | ------------ | -| User Need | `URS-U${user need number}` | URS-U001 | -| Requirement | `REQ-${component short code}-${requirement number}` | REQ-SWR-0001 | - QW initialises with a `System` component for requirements (used for generating a unique identifier, short code value is `X`). You can add extra components to the configuration by editing the `/.qw/components.csv` @@ -222,21 +216,22 @@ System,X,Whole system requirements. Drug dosage,D,Drug dosage calculation module. ``` +You set up the workflow files and the release templates files like this: + ```shell -qw configure +qw configure --workflow +qw configure --release-templates basic ``` -> INFO: there are currently 27 issues and PRs -> -> Can connect to remote repository :tada: -> -> INFO: Writing templates to local repository, force=False -> -> Local repository updated, please commit the changes made to your local repository. +At the moment `basic` is the only option for release templates at the moment. +In the future we should have options such as `iso13485`, `dbc0129` and +`management` to allow you to produce regulatory documents and management +documents without having to build them yourself. ## Configuration using Gitlab -Intentionally left blank at the moment for brevity. Will aim for being able to implement this. +Gitlab is not supported yet. When it is, it will work both on gitlab.com and on +instances hosted elsewhere. # Using QW with github @@ -333,6 +328,8 @@ QW uses existing issues and pull requests to track the different design and deve ## Closing QW items when they are not resolved by a PR +**Note**: this functionality has not yet been completed. + - There may be times when a QW-tracked issue is required to be closed not by a PR. - You may close the issue as either `completed` or `not planned` ![](https://hackmd.io/_uploads/Sy22Bi9x6.png) - Then please add another section to QW information in the issue in the form: @@ -351,6 +348,28 @@ QW uses existing issues and pull requests to track the different design and deve Duplicate of #5 ``` +## Checking QW items for consistency + +You can check that all the closing issues of all PRs (not marked with +`qw-ignore`) are requirements, that all Requirements have User Needs +links and that all these User Needs links are marked with `qw-user-need`: + +```sh +qw check --remote +``` + +There are clearly many more checks that we could run in this stage. + +The `--remote` flag tells `qw` to examine the PRs and issues as they +currently exist on the server. The alternative is: + +```sh +qw check --local +``` + +This checks the PRs and issues as they were gathered by the last +invocation of the `qw freeze` command, described next. + ## Versioning of QW items The `qw freeze` function is used to save the current state of the qw items in github, and to ensure that the versions of items are updated if their @@ -410,29 +429,60 @@ Example response when the change is trivial and does not warrant a change in the ### Creating a documentation release When you're ready to update the documentation in your Quality Management System (QMS), you can use the QW tool's `release` command. -Running this will: - -- Ensure that all issues and pull requests have been marked with `qw-ignore` or one of the `qw-` item tags, raising an error (and stopping) to ensure - all items are tagged -- Ensure that all QW items have versions -- Ensure the entire chain `design validation -> design verification -> design output -> requirement -> user need` is consistent with QW rules, - starting from the furthest stage of QW items. So if there is no `design validation`, then the chain will start from `design verification`. If there - was only a `user need` and `requirement`s, then only these would be validated -- Create word documents based on the QW template for export -- [name=Stef] Optionally? Create an html page that shows a burndown graph for each of the QW item types, showing the number completed and outstanding - over time. Would this be useful? +Running this will create word documents based on the QW template for export ```shell -qw release qms_docs +qw release ``` -> Creating a release for QW -> -> Creating "qms_docs" directory if it doesn't already exist, and overwriting any previously exported files -> -> INFO: :heavy_check_mark: 47 QW items checked -> -> INFO: :heavy_check_mark: Documents have been written to the "qms_docs" directory +This turns the files in the `qw_release_templates` directory into the same +named files (within the same named subdirectories) into correctly filled-in +documents in the `qw_release_out` directory. Any paragraphs containing +mailmerge fields will be repeated however many times they need to be to +be filled in with all the data qw knows about. Also, dependent paragraphs +below those repeated paragraphs (at "higher" outline level) will be repeated. +The mailmerge fields in these dependent paragraphs will be filled with +dependent data from qw. + +If you want to write your own document template, or update them, you +need to be able to add these mailmerge fields to your document. For this +you need a "database file". Get that from the `qw generate-merge-fields` +command. This produces a file called `fields.csv`. + +What do we do with this `fields.csv` file? In MS Word you choose +"Select Recipients|Use Existing List..." from the "Mailings" ribbon, then +select the `fields.csv` file. Now the "Insert Merge Field" button lets you +add fields. In LibreOffice you select "Insert|Field|More Fields...". In the +dialog that pops up, select the "Database" tab, highlight the "Mail Merge +Fields" type. In the "Add database file" box click the "Browse..." button +and select the `fields.csv` file. Now the fields appear in the right hand +box. You can select the one you want and click "Insert". You can keep +the dialog open as you type if you like. + +Now, as long as you save this new file within the `qw_release_templates` +directory, `qw release` will fill in your document and output the result to the +`qw_release_out` directory. + +You probably want to add `qw_release_out` to your `.gitignore` file, +especially if you are adding the documents inside it to some other QMS tool. + +## Tab completion of commands + +You can use tab to complete the subcommands of `qw`, for example you can type +`qw gen` then press tab, and `qw generate-merge-fields` will appear, but only +if you have installed the tab completions. Do this with the following on Windows: + +```sh +qw --install-completion powershell +``` + +to get tab completion in PowerShell. Similarly on Linux, Mac or WSL use: + +```sh +qw --install-completion bash +``` + +or use the `zsh` or `fish` options if you use those shells. # FAQ and common issues From 21eff5e15b96adf975cf49a44c72d72d13766c54 Mon Sep 17 00:00:00 2001 From: Tim Band Date: Thu, 15 Feb 2024 12:27:36 +0000 Subject: [PATCH 10/10] ruff fix --- src/qw/cli.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/qw/cli.py b/src/qw/cli.py index 893d154..54dd7a6 100644 --- a/src/qw/cli.py +++ b/src/qw/cli.py @@ -261,12 +261,11 @@ def check( for failure in result.failures: logger.error(failure) raise typer.Exit(code=1) - else: - logger.success( - "OK: Ran {} check(s) over {} object(s), all successful", - result.check_count, - result.object_count, - ) + logger.success( + "OK: Ran {} check(s) over {} object(s), all successful", + result.check_count, + result.object_count, + ) @app.command()