Merge remote-tracking branch 'origin/main' into spcb-sbc

.github/workflows/pre-commit.yaml

@@ -13,10 +13,4 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: "3.12"
-    - name: Set up Quarto
-      uses: quarto-dev/quarto-actions/setup@v2
-    - name: Install Poetry
-      run: pip install poetry
-    - name: Install packages
-      run: poetry install --with dev
     - uses: pre-commit/[email protected]

.github/workflows/render-README.yaml

@@ -0,0 +1,23 @@
+name: render-readme
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  render-readme:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Set up Quarto
+      uses: quarto-dev/quarto-actions/setup@v2
+    - name: Install Poetry
+      run: pip install poetry
+    - name: Install packages
+      run: poetry install --with dev
+    - name: Render README
+      run: poetry run bash scripts/render-quarto-readme.sh

.pre-commit-config.yaml

@@ -13,7 +13,6 @@ repos:
     -   id: end-of-file-fixer
     -   id: mixed-line-ending
     -   id: trailing-whitespace
-
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
@@ -56,49 +55,83 @@ repos:
     -   id: check-executables-have-shebangs
         files: \.sh$
 ################################################################################
-# QUARTO README RENDERING
+# PYTHON
 ################################################################################
--   repo: local
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.4
     hooks:
-    -   id: render-quarto-readme
-        name: "render README.qmd as README.md"
-        entry: "./scripts/hooks/render-quarto-readme.sh"
-        language: script
-        always_run: true
-        files: "README.(q|)md"
+    # "currently, the Ruff formatter does not sort imports.
+    # In order to both sort imports and format, call
+    # the Ruff linter and then the formatter:"
+    -   id: ruff
+        args: [
+          "check",
+          "--select",
+          # isort
+          "I",
+          "--fix"]
+    # run ruff linter; the Ruff Linter is an extremely fast Python linter
+    # designed as a drop-in replacement for Flake8 (plus dozens of plugins),
+    # isort, pydocstyle, pyupgrade, autoflake, and more
+    -   id: ruff
+        args: [
+          # ambiguous variable name: {name}
+          "--ignore=E741",
+          # do not assign a lambda expression, use a def
+          "--ignore=E731",
+          # found useless expression. ignore since .qmd displays
+          "--ignore=B018",
+          # {name} is too complex ({complexity} > {max_complexity})
+          # note: ignored on select repositories
+          "--ignore=C901",
+          # E and W: pycodestyle, standard PEP8 errors and pycodestyle warnings.
+          # F: pyflakes warnings (e.g., unused variables, undefined names).,
+          # B: flake8-bugbear (useful best practices).
+          # SIM: flake8-simplify
+          # C90: McCabe complexity (cyclomatic complexity).
+          # UP: pyupgrade, Python version compatibility
+          "--select=E,W,F,B,C90,UP,SIM",
+          # linter checks for lines, but doesn't fix, default is 88
+          "--line-length=79",
+          # lint all files in the current directory, and fix any fixable errors.
+          "--fix"]
+    # run the ruff-formatter; the Ruff formatter is an extremely fast
+    # Python code formatter designed as a drop-in replacement for Black
+    -   id: ruff-format
+        args: [
+          "--line-length=79",
+        ]
 ################################################################################
-# PYTHON
+# SECURITY
 ################################################################################
--   repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-    hooks:
-    -   id: isort
-        args: ["--profile", "black", "--line-length", "79"]
--   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.3
+-   repo: https://github.com/Yelp/detect-secrets
+    rev: v1.5.0
     hooks:
-      - id: ruff
-        args: ["--ignore=E741", "--ignore=E731", "--fix"]
+    # must first run
+    # detect-secrets scan > .secrets.baseline
+    -   id: detect-secrets
+        args: ["--baseline", ".secrets.baseline"]
+        exclude: package.lock.json
 ################################################################################
 # GITHUB ACTIONS
 ################################################################################
 -   repo: https://github.com/rhysd/actionlint
-    rev: v1.7.4
+    rev: v1.7.7
     hooks:
     -   id: actionlint
 ################################################################################
 # SPELLING
 ################################################################################
 -   repo: https://github.com/crate-ci/typos
-    rev: typos-dict-v0.11.37
+    rev: typos-dict-v0.12.4
     hooks:
     -   id: typos
         args: ["--force-exclude"]
 ################################################################################
 # COMMIT MESSAGES
 ################################################################################
 -   repo: https://github.com/commitizen-tools/commitizen
-    rev: v4.1.0
+    rev: v4.1.1
     hooks:
     -   id: commitizen
 -   repo: https://github.com/jorisroovers/gitlint

.secrets.baseline

@@ -0,0 +1,112 @@
+{
+  "version": "1.4.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
+  "results": {},
+  "generated_at": "2023-09-24T19:52:08Z"
+}

README.qmd

@@ -230,7 +230,7 @@ print(idata_w_dates["observed_data"]["obs"]["obs_dim_0"][:15])
 ```{python}
 # idata without dates as coordinates
 idata_wo_dates = forecasttools.nhsn_flu_forecast_wo_dates
-print(idata_wo_dates["observed_data"]["obs"]["obs_dim_0"][:15])
+print(idata_wo_dates["observed_data"]["obs"]["obs_dim_0"][:20])
 ```
 
 The forecast was generated following the creation of `nhsn_hosp_flu.csv` (see previous section) by running `data.py` with the following added:

forecasttools/__init__.py

@@ -32,7 +32,9 @@
     "location_table.parquet"
 )
 location_table = pl.read_parquet(location_table_path)
-united_states = location_table.filter(pl.col("is_state")).get_column("long_name").to_list()
+united_states = (
+    location_table.filter(pl.col("is_state")).get_column("long_name").to_list()
+)
 
 # load example flusight submission
 example_flusight_submission_path = importlib.resources.files(

forecasttools/daily_to_epiweekly.py

@@ -31,7 +31,7 @@ def df_aggregate_to_epiweekly(
     df: pl.DataFrame,
     value_col: str = "value",
     date_col: str = "date",
-    id_cols: str | list[str] = ["draw"],
+    id_cols: list[str] = None,
     weekly_value_name: str = "weekly_value",
     strict: bool = True,
 ) -> pl.DataFrame:
@@ -72,6 +72,9 @@ def df_aggregate_to_epiweekly(
         A dataframe with value_col aggregated
         across epiweek and epiyear.
     """
+    # set default id_cols
+    if id_cols is None:
+        id_cols = ["draw"]
     id_cols = ensure_listlike(id_cols)
     # add epiweek and epiyear columns
     df = df.with_columns(
@@ -89,14 +92,25 @@ def df_aggregate_to_epiweekly(
     n_elements = grouped_df.agg(pl.count().alias("n_elements"))
     problematic_trajectories = n_elements.filter(pl.col("n_elements") > 7)
     if not problematic_trajectories.is_empty():
+        message = (
+            f"Problematic trajectories with more than"
+            f" 7 values per epiweek per year: {problematic_trajectories}"
+        )
+        raise ValueError(
+            f"At least one trajectory has more than 7 values for a given"
+            f" epiweek of a given year.\n{message}"
+        )
+    # check if any week has more than 7 dates
+    if not n_elements["n_elements"].to_numpy().max() <= 7:
         raise ValueError(
             "At least one trajectory has more than 7 values "
             "for a given epiweek of a given epiyear.\n"
             "Problematic trajectories with more than 7 "
             "values: "
             f"{problematic_trajectories}"
         )
-
+    # if strict, filter out groups that do not have exactly 7
+    # contributing dates
     if strict:
         valid_groups = n_elements.filter(pl.col("n_elements") == 7)
         df = df.join(