From ed282712642eed670d047c6cc83f5c4a084b5e7d Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:19:55 -0300 Subject: [PATCH 1/7] Convert .gitignore entries to radon ignore/exlude configs. --- src/wily/commands/build.py | 32 ++++++++++++++++++++++++--- src/wily/operators/cyclomatic.py | 3 ++- src/wily/operators/halstead.py | 3 ++- src/wily/operators/maintainability.py | 3 ++- src/wily/operators/raw.py | 4 +++- 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/wily/commands/build.py b/src/wily/commands/build.py index 4d01b5a8..e4715b64 100644 --- a/src/wily/commands/build.py +++ b/src/wily/commands/build.py @@ -10,18 +10,40 @@ from sys import exit from typing import Any, Dict, List, Tuple +from git import Repo from progress.bar import Bar from wily import logger from wily.archivers import Archiver, FilesystemArchiver, Revision from wily.archivers.git import InvalidGitRepositoryError +from wily.config import load as load_config from wily.config.types import WilyConfig +from wily.defaults import DEFAULT_CONFIG_PATH from wily.operators import Operator, resolve_operator from wily.state import State +def gitignore_to_radon(): + """Convert entries in a .gitignore file to radon ignore/exclude configs.""" + config = load_config(DEFAULT_CONFIG_PATH) + repo = Repo(config.path) + gitignore_path = pathlib.Path(repo.git_dir) / ".gitignore" + ignore = [] + with gitignore_path.open() as gitignore: + for line in gitignore: + line = line.strip() + if not line or line.startswith(("#", "\\")): + continue + ignore.append(line) + return ",".join(ignore) + + def run_operator( - operator: Operator, revision: Revision, config: WilyConfig, targets: List[str] + operator: Operator, + revision: Revision, + config: WilyConfig, + targets: List[str], + ignore: str, ) -> Tuple[str, Dict[str, Any]]: """ Run an operator for the multiprocessing pool. @@ -30,8 +52,9 @@ def run_operator( :param revision: The revision index :param config: The runtime configuration :param targets: Files/paths to scan + :param ignore: Files/paths to ignore/exclude """ - instance = operator.operator_cls(config, targets) + instance = operator.operator_cls(config, targets, ignore=ignore, exclude=ignore) logger.debug(f"Running {operator.name} operator on {revision}") data = instance.run(revision, config) @@ -87,6 +110,9 @@ def build(config: WilyConfig, archiver: Archiver, operators: List[Operator]) -> bar = Bar("Processing", max=len(revisions) * len(operators)) state.operators = operators + # Get patterns to ignore from .gitignore + ignore = gitignore_to_radon() + # Index all files the first time, only scan changes afterward seed = True try: @@ -108,7 +134,7 @@ def build(config: WilyConfig, archiver: Archiver, operators: List[Operator]) -> # Run each operator as a separate process data = pool.starmap( run_operator, - [(operator, revision, config, targets) for operator in operators], + [(operator, revision, config, targets, ignore) for operator in operators], ) # data is a list of tuples, where for each operator, it is a tuple of length 2, operator_data_len = 2 diff --git a/src/wily/operators/cyclomatic.py b/src/wily/operators/cyclomatic.py index 19a62b97..be92f406 100644 --- a/src/wily/operators/cyclomatic.py +++ b/src/wily/operators/cyclomatic.py @@ -43,7 +43,7 @@ class CyclomaticComplexityOperator(BaseOperator): default_metric_index = 0 # MI - def __init__(self, config, targets): + def __init__(self, config, targets, **kwargs): """ Instantiate a new Cyclomatic Complexity operator. @@ -53,6 +53,7 @@ def __init__(self, config, targets): # TODO: Import config for harvester from .wily.cfg logger.debug(f"Using {targets} with {self.defaults} for CC metrics") + self.defaults.update(kwargs) self.harvester = harvesters.CCHarvester(targets, config=Config(**self.defaults)) def run(self, module, options): diff --git a/src/wily/operators/halstead.py b/src/wily/operators/halstead.py index 9bcc03bc..1554ca2e 100644 --- a/src/wily/operators/halstead.py +++ b/src/wily/operators/halstead.py @@ -44,7 +44,7 @@ class HalsteadOperator(BaseOperator): default_metric_index = 0 # MI - def __init__(self, config, targets): + def __init__(self, config, targets, **kwargs): """ Instantiate a new HC operator. @@ -54,6 +54,7 @@ def __init__(self, config, targets): # TODO : Import config from wily.cfg logger.debug(f"Using {targets} with {self.defaults} for HC metrics") + self.defaults.update(kwargs) self.harvester = harvesters.HCHarvester(targets, config=Config(**self.defaults)) def run(self, module, options): diff --git a/src/wily/operators/maintainability.py b/src/wily/operators/maintainability.py index 856664a1..6e18308f 100644 --- a/src/wily/operators/maintainability.py +++ b/src/wily/operators/maintainability.py @@ -52,7 +52,7 @@ class MaintainabilityIndexOperator(BaseOperator): default_metric_index = 1 # MI - def __init__(self, config, targets): + def __init__(self, config, targets, **kwargs): """ Instantiate a new MI operator. @@ -62,6 +62,7 @@ def __init__(self, config, targets): # TODO : Import config from wily.cfg logger.debug(f"Using {targets} with {self.defaults} for MI metrics") + self.defaults.update(kwargs) self.harvester = harvesters.MIHarvester(targets, config=Config(**self.defaults)) def run(self, module, options): diff --git a/src/wily/operators/raw.py b/src/wily/operators/raw.py index 9a3c5ae6..c03aefc9 100644 --- a/src/wily/operators/raw.py +++ b/src/wily/operators/raw.py @@ -39,7 +39,7 @@ class RawMetricsOperator(BaseOperator): ) default_metric_index = 0 # LOC - def __init__(self, config, targets): + def __init__(self, config, targets, **kwargs): """ Instantiate a new raw operator. @@ -48,6 +48,8 @@ def __init__(self, config, targets): """ # TODO: Use config from wily.cfg for harvester logger.debug(f"Using {targets} with {self.defaults} for Raw metrics") + + self.defaults.update(kwargs) self.harvester = harvesters.RawHarvester( targets, config=Config(**self.defaults) ) From ae2219e36d8b081813052acf63b851a1f8def211 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:30:17 -0300 Subject: [PATCH 2/7] Fix path and check that .gitignore exists in gitignore_to_radon(). --- src/wily/commands/build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/wily/commands/build.py b/src/wily/commands/build.py index e4715b64..d31fb1cc 100644 --- a/src/wily/commands/build.py +++ b/src/wily/commands/build.py @@ -10,7 +10,6 @@ from sys import exit from typing import Any, Dict, List, Tuple -from git import Repo from progress.bar import Bar from wily import logger @@ -26,8 +25,10 @@ def gitignore_to_radon(): """Convert entries in a .gitignore file to radon ignore/exclude configs.""" config = load_config(DEFAULT_CONFIG_PATH) - repo = Repo(config.path) - gitignore_path = pathlib.Path(repo.git_dir) / ".gitignore" + gitignore_path = pathlib.Path(config.path) / ".gitignore" + if not gitignore_path.exists(): + logger.info(f".gitignore file not found at {pathlib.Path(config.path)}") + return "" ignore = [] with gitignore_path.open() as gitignore: for line in gitignore: From 58ee89fc9ead4422263840b1f88ff60018d72e52 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:39:27 -0300 Subject: [PATCH 3/7] Formatting. --- src/wily/commands/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/wily/commands/build.py b/src/wily/commands/build.py index d31fb1cc..5df1a78a 100644 --- a/src/wily/commands/build.py +++ b/src/wily/commands/build.py @@ -135,7 +135,10 @@ def build(config: WilyConfig, archiver: Archiver, operators: List[Operator]) -> # Run each operator as a separate process data = pool.starmap( run_operator, - [(operator, revision, config, targets, ignore) for operator in operators], + [ + (operator, revision, config, targets, ignore) + for operator in operators + ], ) # data is a list of tuples, where for each operator, it is a tuple of length 2, operator_data_len = 2 From 2bc0662601f4e97bd8416161bbc8e63a22e0902f Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Wed, 23 Aug 2023 01:18:27 -0300 Subject: [PATCH 4/7] Add return type to gitignore_to_radon(), update module docstring. --- src/wily/commands/build.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/wily/commands/build.py b/src/wily/commands/build.py index 5df1a78a..284ee61c 100644 --- a/src/wily/commands/build.py +++ b/src/wily/commands/build.py @@ -1,8 +1,7 @@ """ -Builds a cache based on a source-control history. - -TODO : Convert .gitignore to radon ignore patterns to make the build more efficient. +Build command. +Builds a cache based on a source-control history. """ import multiprocessing import os @@ -22,7 +21,7 @@ from wily.state import State -def gitignore_to_radon(): +def gitignore_to_radon() -> str: """Convert entries in a .gitignore file to radon ignore/exclude configs.""" config = load_config(DEFAULT_CONFIG_PATH) gitignore_path = pathlib.Path(config.path) / ".gitignore" From b85c7c60fc4cb04c2dee9d3c106698748a7a33ae Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Thu, 24 Aug 2023 11:35:05 -0300 Subject: [PATCH 5/7] Pass config.path to gitignore_to_radon() instead of building a new config. --- src/wily/commands/build.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/wily/commands/build.py b/src/wily/commands/build.py index 284ee61c..8a9ff683 100644 --- a/src/wily/commands/build.py +++ b/src/wily/commands/build.py @@ -14,19 +14,21 @@ from wily import logger from wily.archivers import Archiver, FilesystemArchiver, Revision from wily.archivers.git import InvalidGitRepositoryError -from wily.config import load as load_config from wily.config.types import WilyConfig -from wily.defaults import DEFAULT_CONFIG_PATH from wily.operators import Operator, resolve_operator from wily.state import State -def gitignore_to_radon() -> str: - """Convert entries in a .gitignore file to radon ignore/exclude configs.""" - config = load_config(DEFAULT_CONFIG_PATH) - gitignore_path = pathlib.Path(config.path) / ".gitignore" +def gitignore_to_radon(path: str) -> str: + """ + Convert entries in a .gitignore file to radon ignore/exclude configs. + + :param path: The path where to find .gitignore. + :return: A comma-separated string containing glob patterns from .gitignore. + """ + gitignore_path = pathlib.Path(path) / ".gitignore" if not gitignore_path.exists(): - logger.info(f".gitignore file not found at {pathlib.Path(config.path)}") + logger.info(f".gitignore file not found at {pathlib.Path(path)}") return "" ignore = [] with gitignore_path.open() as gitignore: @@ -111,7 +113,7 @@ def build(config: WilyConfig, archiver: Archiver, operators: List[Operator]) -> state.operators = operators # Get patterns to ignore from .gitignore - ignore = gitignore_to_radon() + ignore = gitignore_to_radon(config.path) # Index all files the first time, only scan changes afterward seed = True From 79f6d1b5098da65fe95a59317cf0b212084174a3 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Thu, 24 Aug 2023 12:54:54 -0300 Subject: [PATCH 6/7] Pass empty string as ignore to run_operator in diff.py. --- src/wily/commands/diff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wily/commands/diff.py b/src/wily/commands/diff.py index f9738de6..6fe63bdd 100644 --- a/src/wily/commands/diff.py +++ b/src/wily/commands/diff.py @@ -95,7 +95,8 @@ def diff( # Build a set of operators with multiprocessing.Pool(processes=len(operators)) as pool: operator_exec_out = pool.starmap( - run_operator, [(operator, None, config, targets) for operator in operators] + run_operator, + [(operator, None, config, targets, "") for operator in operators], ) data = {} for operator_name, result in operator_exec_out: From 79591560e2c72a1425adc8dcb1ccf25c455d44f0 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Thu, 24 Aug 2023 13:52:28 -0300 Subject: [PATCH 7/7] Add tests for gitignore_to_radon(). --- test/unit/test_build_unit.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/test/unit/test_build_unit.py b/test/unit/test_build_unit.py index 49e131ef..826006b7 100644 --- a/test/unit/test_build_unit.py +++ b/test/unit/test_build_unit.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, mock_open, patch import pytest @@ -61,3 +61,36 @@ def test_build_simple(config): with patch("wily.state.resolve_archiver", return_value=MockArchiver): result = build.build(config, MockArchiver, _test_operators) assert result is None + + +def test_gitignore_to_radon(): + mock_file = mock_open(read_data="test1.py\n\n\\not_included\n#comment\ntest2/") + mock_file = mock_file() + mock_opener = MagicMock() + mock_opener.__enter__.return_value = mock_file + mock_opener.open.return_value = mock_opener + mock_opener.__truediv__.return_value = mock_opener + + mock_path = MagicMock(return_value=mock_opener) + with patch("wily.commands.build.pathlib.Path", mock_path): + result = build.gitignore_to_radon("") + assert result == "test1.py,test2/" + mock_file.__iter__.assert_called_once() + assert len(mock_opener.mock_calls) == 6 + + +def test_gitignore_to_radon_no_gitignore(): + mock_file = mock_open(read_data="test1.py\n\n\\not_included\n#comment\ntest2/") + mock_file = mock_file() + mock_opener = MagicMock() + mock_opener.__enter__.return_value = mock_file + mock_opener.open.return_value = mock_opener + mock_opener.__truediv__.return_value = mock_opener + + # Make gitignore_path.exists() return False + mock_opener.exists.return_value = False + mock_path = MagicMock(return_value=mock_opener) + with patch("wily.commands.build.pathlib.Path", mock_path): + result = build.gitignore_to_radon("") + assert result == "" + assert len(mock_opener.mock_calls) == 3