From 10d2ab492b87e402ca42a5d3e4a90f9d8545a710 Mon Sep 17 00:00:00 2001 From: Samuel Brand <48288458+SamuelBrand1@users.noreply.github.com> Date: Tue, 11 Feb 2025 18:30:20 +0000 Subject: [PATCH] Issue 66: simplify and split CI actions (#67) * switch to more basic pre-commit from pyrenew * remove concurrency * Update .pre-commit-config.yaml * Add back some hooks * Update .pre-commit-config.yaml * Delete .pre-commit-config_max.yaml * add spelling pre-commit hook * move script because not a "hook" anymore * Create render-README.yaml * Update render-README.yaml * Update README.md * Update .pre-commit-config.yaml * revert pre-commit file minus readme render and with security hook * pre-commit fixes * minor test changes in quarto readme --------- Co-authored-by: AFg6K7h4fhy2 <127630341+AFg6K7h4fhy2@users.noreply.github.com> --- .github/workflows/pre-commit.yaml | 6 -- .github/workflows/render-README.yaml | 23 +++++ .pre-commit-config.yaml | 77 ++++++++++---- .secrets.baseline | 112 +++++++++++++++++++++ README.md | 29 ------ README.qmd | 2 +- forecasttools/__init__.py | 4 +- forecasttools/daily_to_epiweekly.py | 18 +++- forecasttools/data.py | 64 +++++++++--- forecasttools/idata_w_dates_to_df.py | 5 +- forecasttools/recode_locations.py | 24 +++-- forecasttools/trajectories_to_quantiles.py | 4 +- forecasttools/utils.py | 12 ++- scripts/render-quarto-readme.sh | 42 ++++++++ tests/test_add_time_coords_to_idata.py | 6 +- tests/test_recoding_locations.py | 19 ++-- 16 files changed, 344 insertions(+), 103 deletions(-) create mode 100644 .github/workflows/render-README.yaml create mode 100644 .secrets.baseline create mode 100755 scripts/render-quarto-readme.sh diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 800e852..bfe6623 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -13,10 +13,4 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Set up Quarto - uses: quarto-dev/quarto-actions/setup@v2 - - name: Install Poetry - run: pip install poetry - - name: Install packages - run: poetry install --with dev - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/render-README.yaml b/.github/workflows/render-README.yaml new file mode 100644 index 0000000..c07803d --- /dev/null +++ b/.github/workflows/render-README.yaml @@ -0,0 +1,23 @@ +name: render-readme + +on: + pull_request: + push: + branches: [main] + +jobs: + render-readme: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + - name: Install Poetry + run: pip install poetry + - name: Install packages + run: poetry install --with dev + - name: Render README + run: poetry run bash scripts/render-quarto-readme.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fd585c3..240ade7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,6 @@ repos: - id: end-of-file-fixer - id: mixed-line-ending - id: trailing-whitespace - - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: @@ -56,41 +55,75 @@ repos: - id: check-executables-have-shebangs files: \.sh$ ################################################################################ -# QUARTO README RENDERING +# PYTHON ################################################################################ -- repo: local +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.4 hooks: - - id: render-quarto-readme - name: "render README.qmd as README.md" - entry: "./scripts/hooks/render-quarto-readme.sh" - language: script - always_run: true - files: "README.(q|)md" + # "currently, the Ruff formatter does not sort imports. + # In order to both sort imports and format, call + # the Ruff linter and then the formatter:" + - id: ruff + args: [ + "check", + "--select", + # isort + "I", + "--fix"] + # run ruff linter; the Ruff Linter is an extremely fast Python linter + # designed as a drop-in replacement for Flake8 (plus dozens of plugins), + # isort, pydocstyle, pyupgrade, autoflake, and more + - id: ruff + args: [ + # ambiguous variable name: {name} + "--ignore=E741", + # do not assign a lambda expression, use a def + "--ignore=E731", + # found useless expression. ignore since .qmd displays + "--ignore=B018", + # {name} is too complex ({complexity} > {max_complexity}) + # note: ignored on select repositories + "--ignore=C901", + # E and W: pycodestyle, standard PEP8 errors and pycodestyle warnings. + # F: pyflakes warnings (e.g., unused variables, undefined names)., + # B: flake8-bugbear (useful best practices). + # SIM: flake8-simplify + # C90: McCabe complexity (cyclomatic complexity). + # UP: pyupgrade, Python version compatibility + "--select=E,W,F,B,C90,UP,SIM", + # linter checks for lines, but doesn't fix, default is 88 + "--line-length=79", + # lint all files in the current directory, and fix any fixable errors. + "--fix"] + # run the ruff-formatter; the Ruff formatter is an extremely fast + # Python code formatter designed as a drop-in replacement for Black + - id: ruff-format + args: [ + "--line-length=79", + ] ################################################################################ -# PYTHON +# SECURITY ################################################################################ -- repo: https://github.com/PyCQA/isort - rev: 5.13.2 - hooks: - - id: isort - args: ["--profile", "black", "--line-length", "79"] -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.3 +- repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 hooks: - - id: ruff - args: ["--ignore=E741", "--ignore=E731", "--fix"] + # must first run + # detect-secrets scan > .secrets.baseline + - id: detect-secrets + args: ["--baseline", ".secrets.baseline"] + exclude: package.lock.json ################################################################################ # GITHUB ACTIONS ################################################################################ - repo: https://github.com/rhysd/actionlint - rev: v1.7.4 + rev: v1.7.7 hooks: - id: actionlint ################################################################################ # SPELLING ################################################################################ - repo: https://github.com/crate-ci/typos - rev: typos-dict-v0.11.37 + rev: typos-dict-v0.12.4 hooks: - id: typos args: ["--force-exclude"] @@ -98,7 +131,7 @@ repos: # COMMIT MESSAGES ################################################################################ - repo: https://github.com/commitizen-tools/commitizen - rev: v4.1.0 + rev: v4.1.1 hooks: - id: commitizen - repo: https://github.com/jorisroovers/gitlint diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..63f42f9 --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,112 @@ +{ + "version": "1.4.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": {}, + "generated_at": "2023-09-24T19:52:08Z" +} diff --git a/README.md b/README.md index 0c0fea0..1d4c87c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ render a GitHub README, see --> -