From a91ef101aea52587fa42b4ef5f5a471b7d5209cd Mon Sep 17 00:00:00 2001 From: Samuel Brand Date: Mon, 10 Feb 2025 11:40:09 +0000 Subject: [PATCH] switch to more basic pre-commit from pyrenew --- .github/workflows/pre-commit.yaml | 10 +-- .pre-commit-config.yaml | 139 ++++++++---------------------- .pre-commit-config_max.yaml | 108 +++++++++++++++++++++++ .secrets.baseline | 112 ++++++++++++++++++++++++ forecasttools/__init__.py | 4 +- forecasttools/data.py | 64 +++++++++++--- 6 files changed, 312 insertions(+), 125 deletions(-) create mode 100644 .pre-commit-config_max.yaml create mode 100644 .secrets.baseline diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 800e852..0b4fc2b 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -5,6 +5,10 @@ on: push: branches: [main] +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + jobs: pre-commit: runs-on: ubuntu-latest @@ -13,10 +17,4 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Set up Quarto - uses: quarto-dev/quarto-actions/setup@v2 - - name: Install Poetry - run: pip install poetry - - name: Install packages - run: poetry install --with dev - uses: pre-commit/action@v3.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fd585c3..7bf26f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,108 +1,39 @@ repos: -################################################################################ -# GENERAL -################################################################################ -- repo: https://github.com/pre-commit/pre-commit-hooks + ##### + # Basic file cleanliness + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - - id: check-added-large-files - args: ["--maxkb=10000"] - - id: check-yaml - args: ["--allow-multiple-documents"] - - id: check-toml - - id: end-of-file-fixer - - id: mixed-line-ending - - id: trailing-whitespace - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 - hooks: - # prevent giant files from being committed. - - id: check-added-large-files - args: ["--maxkb=10000"] - # simply check whether files parse as valid - # python - - id: check-ast - # check for files with names that would - # conflict on a case-insensitive filesystem - # like MacOS HFS+ or Windows FAT. - - id: check-case-conflict - # checks for a common error of placing - # code before the docstring. - - id: check-docstring-first - # attempts to load all yaml files to - # verify syntax. - - id: check-yaml - # allow yaml files which use the - # multi-document syntax - args: ["--allow-multiple-documents"] - # attempts to load all TOML files to - # verify syntax. - - id: check-toml - # makes sure files end in a newline and - # only a newline. - - id: end-of-file-fixer - # replaces or checks mixed line ending. - - id: mixed-line-ending - # verifies that test files are named - # correctly. - - id: name-tests-test - # ensure tests match test_.*\.py - args: ["--pytest-test-first"] - # trims trailing whitespace. - - id: trailing-whitespace - # checks that non-binary executables have - # a proper shebang. - - id: check-executables-have-shebangs - files: \.sh$ -################################################################################ -# QUARTO README RENDERING -################################################################################ -- repo: local - hooks: - - id: render-quarto-readme - name: "render README.qmd as README.md" - entry: "./scripts/hooks/render-quarto-readme.sh" - language: script - always_run: true - files: "README.(q|)md" -################################################################################ -# PYTHON -################################################################################ -- repo: https://github.com/PyCQA/isort - rev: 5.13.2 - hooks: - - id: isort - args: ["--profile", "black", "--line-length", "79"] -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.3 - hooks: + - id: check-added-large-files + - id: check-yaml + - id: check-toml + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + ##### + # Python + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.1 + hooks: + # Sort imports - id: ruff - args: ["--ignore=E741", "--ignore=E731", "--fix"] -################################################################################ -# GITHUB ACTIONS -################################################################################ -- repo: https://github.com/rhysd/actionlint - rev: v1.7.4 - hooks: - - id: actionlint -################################################################################ -# SPELLING -################################################################################ -- repo: https://github.com/crate-ci/typos - rev: typos-dict-v0.11.37 - hooks: - - id: typos - args: ["--force-exclude"] -################################################################################ -# COMMIT MESSAGES -################################################################################ -- repo: https://github.com/commitizen-tools/commitizen - rev: v4.1.0 - hooks: - - id: commitizen -- repo: https://github.com/jorisroovers/gitlint - rev: v0.19.1 - hooks: - - id: gitlint -################################################################################ + args: ["check", "--select", "I", "--fix"] + # Run the linter + - id: ruff + # Run the formatter + - id: ruff-format + args: ["--line-length", "79"] + ##### + # R + - repo: https://github.com/lorenzwalthert/precommit + rev: v0.4.3 + hooks: + - id: style-files + - id: lintr + # Secrets + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ["--baseline", ".secrets.baseline"] + exclude: package.lock.json diff --git a/.pre-commit-config_max.yaml b/.pre-commit-config_max.yaml new file mode 100644 index 0000000..fd585c3 --- /dev/null +++ b/.pre-commit-config_max.yaml @@ -0,0 +1,108 @@ +repos: +################################################################################ +# GENERAL +################################################################################ +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + args: ["--maxkb=10000"] + - id: check-yaml + args: ["--allow-multiple-documents"] + - id: check-toml + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + # prevent giant files from being committed. + - id: check-added-large-files + args: ["--maxkb=10000"] + # simply check whether files parse as valid + # python + - id: check-ast + # check for files with names that would + # conflict on a case-insensitive filesystem + # like MacOS HFS+ or Windows FAT. + - id: check-case-conflict + # checks for a common error of placing + # code before the docstring. + - id: check-docstring-first + # attempts to load all yaml files to + # verify syntax. + - id: check-yaml + # allow yaml files which use the + # multi-document syntax + args: ["--allow-multiple-documents"] + # attempts to load all TOML files to + # verify syntax. + - id: check-toml + # makes sure files end in a newline and + # only a newline. + - id: end-of-file-fixer + # replaces or checks mixed line ending. + - id: mixed-line-ending + # verifies that test files are named + # correctly. + - id: name-tests-test + # ensure tests match test_.*\.py + args: ["--pytest-test-first"] + # trims trailing whitespace. + - id: trailing-whitespace + # checks that non-binary executables have + # a proper shebang. + - id: check-executables-have-shebangs + files: \.sh$ +################################################################################ +# QUARTO README RENDERING +################################################################################ +- repo: local + hooks: + - id: render-quarto-readme + name: "render README.qmd as README.md" + entry: "./scripts/hooks/render-quarto-readme.sh" + language: script + always_run: true + files: "README.(q|)md" +################################################################################ +# PYTHON +################################################################################ +- repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + args: ["--profile", "black", "--line-length", "79"] +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.3 + hooks: + - id: ruff + args: ["--ignore=E741", "--ignore=E731", "--fix"] +################################################################################ +# GITHUB ACTIONS +################################################################################ +- repo: https://github.com/rhysd/actionlint + rev: v1.7.4 + hooks: + - id: actionlint +################################################################################ +# SPELLING +################################################################################ +- repo: https://github.com/crate-ci/typos + rev: typos-dict-v0.11.37 + hooks: + - id: typos + args: ["--force-exclude"] +################################################################################ +# COMMIT MESSAGES +################################################################################ +- repo: https://github.com/commitizen-tools/commitizen + rev: v4.1.0 + hooks: + - id: commitizen +- repo: https://github.com/jorisroovers/gitlint + rev: v0.19.1 + hooks: + - id: gitlint +################################################################################ diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..63f42f9 --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,112 @@ +{ + "version": "1.4.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": {}, + "generated_at": "2023-09-24T19:52:08Z" +} diff --git a/forecasttools/__init__.py b/forecasttools/__init__.py index 023c3f3..b191a96 100644 --- a/forecasttools/__init__.py +++ b/forecasttools/__init__.py @@ -32,7 +32,9 @@ "location_table.parquet" ) location_table = pl.read_parquet(location_table_path) -united_states = location_table.filter(pl.col("is_state")).get_column("long_name").to_list() +united_states = ( + location_table.filter(pl.col("is_state")).get_column("long_name").to_list() +) # load example flusight submission example_flusight_submission_path = importlib.resources.files( diff --git a/forecasttools/data.py b/forecasttools/data.py index b15bf23..8746f89 100644 --- a/forecasttools/data.py +++ b/forecasttools/data.py @@ -7,8 +7,6 @@ an example FluSight submission. """ - - import os import pathlib from urllib import error, request @@ -64,7 +62,6 @@ def check_file_save_path( raise FileExistsError(f"File already exists at: {file_save_path}") - def merge_pop_data_and_loc_data( file_save_path: str, population_file_path: str, @@ -109,16 +106,56 @@ def merge_pop_data_and_loc_data( print(f"File already exists at {save_path}. Skipping writing.") return us_states = [ - "Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", - "Connecticut", "Delaware", "Florida", "Georgia", "Hawaii", "Idaho", - "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana", - "Maine", "Maryland", "Massachusetts", "Michigan", "Minnesota", - "Mississippi", "Missouri", "Montana", "Nebraska", "Nevada", - "New Hampshire", "New Jersey", "New Mexico", "New York", - "North Carolina", "North Dakota", "Ohio", "Oklahoma", "Oregon", - "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", - "Tennessee", "Texas", "Utah", "Vermont", "Virginia", "Washington", - "West Virginia", "Wisconsin", "Wyoming" + "Alabama", + "Alaska", + "Arizona", + "Arkansas", + "California", + "Colorado", + "Connecticut", + "Delaware", + "Florida", + "Georgia", + "Hawaii", + "Idaho", + "Illinois", + "Indiana", + "Iowa", + "Kansas", + "Kentucky", + "Louisiana", + "Maine", + "Maryland", + "Massachusetts", + "Michigan", + "Minnesota", + "Mississippi", + "Missouri", + "Montana", + "Nebraska", + "Nevada", + "New Hampshire", + "New Jersey", + "New Mexico", + "New York", + "North Carolina", + "North Dakota", + "Ohio", + "Oklahoma", + "Oregon", + "Pennsylvania", + "Rhode Island", + "South Carolina", + "South Dakota", + "Tennessee", + "Texas", + "Utah", + "Vermont", + "Virginia", + "Washington", + "West Virginia", + "Wisconsin", + "Wyoming", ] pop_df = pl.read_parquet(population_path).select( [ @@ -144,7 +181,6 @@ def merge_pop_data_and_loc_data( print(f"File successfully written to {save_path}") - def make_census_dataset( file_save_path: str, ) -> None: