Merge pull request #1 from reichlab/initial-setup

Initial setup
reichlab · Jan 21, 2025 · 4aa551d · 4aa551d
2 parents f195a5c + f0aeed8
commit 4aa551d
Show file tree

Hide file tree

Showing 10 changed files with 296 additions and 74 deletions.
diff --git a/.github/.gitignore b/.github/.gitignore
@@ -0,0 +1 @@
+*.html
diff --git a/.github/workflows/cache-hubval-deps.yaml b/.github/workflows/cache-hubval-deps.yaml
@@ -0,0 +1,34 @@
+name: Build dependency cache on main branch
+on:
+  push:
+    branches: [main, master]
+    paths:
+      - '.github/workflows/cache-hubval-deps.yaml'
+  schedule:
+    - cron: "10 0 * * *"
+
+jobs:
+  build-deps-cache-on-main:
+    if: github.event.repository.fork != true
+    runs-on: ubuntu-22.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          install-r: false
+          use-public-rspm: true
+          extra-repositories: 'https://hubverse-org.r-universe.dev'
+
+      - name: Update R
+        run: |
+          sudo apt-get update
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          packages: |
+            any::hubValidations
+            any::sessioninfo
diff --git a/.github/workflows/validate-config.yaml b/.github/workflows/validate-config.yaml
@@ -0,0 +1,55 @@
+name: Hub Config Validation (R)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: main
+    paths:
+      - 'hub-config/**'
+      - '!**README**'
+
+jobs:
+  validate-hub-config:
+    runs-on: ubuntu-22.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      PR_NUMBER: ${{ github.event.number }}
+      HUB_PATH: ${{ github.workspace }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          install-r: false
+          use-public-rspm: true
+          extra-repositories: 'https://hubverse-org.r-universe.dev'
+
+      - name: Update R
+        run: |
+          sudo apt-get update
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          cache: 'always'
+          packages: |
+            any::hubAdmin
+            any::sessioninfo
+
+      - name: Run validations
+        id: validate
+        run: hubAdmin::ci_validate_hub_config(diff = "${{ runner.temp }}/diff.md")
+        shell: Rscript {0}
+      - name: "Comment on PR"
+        id: comment-diff
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        uses: carpentries/actions/comment-diff@main
+        with:
+          pr: ${{ env.PR_NUMBER }}
+          path: ${{ runner.temp }}/diff.md
+      - name: Error on Failure
+        if: ${{ steps.validate.outputs.result == 'false' }}
+        run: |
+          echo "::error title=Invalid Configuration::Errors were detected in one or more config files in 'hub-config/'"
+          exit 1
+
diff --git a/.github/workflows/validate-submission.yaml b/.github/workflows/validate-submission.yaml
@@ -0,0 +1,48 @@
+name: Hub Submission Validation (R)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: main
+    paths:
+      - 'model-output/**'
+      - 'model-metadata/*'
+      - '!**README**'
+
+jobs:
+  validate-submission:
+    runs-on: ubuntu-22.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          install-r: false
+          use-public-rspm: true
+          extra-repositories: 'https://hubverse-org.r-universe.dev'
+
+      - name: Update R
+        run: |
+          sudo apt-get update
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          packages: |
+            any::hubValidations
+            any::sessioninfo
+
+      - name: Run validations
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          library("hubValidations")
+          v <- hubValidations::validate_pr(
+              gh_repo = Sys.getenv("GITHUB_REPOSITORY"),
+              pr_number = Sys.getenv("PR_NUMBER"),
+              skip_submit_window_check = FALSE
+          )
+          hubValidations::check_for_errors(v, verbose = TRUE)
+        shell: Rscript {0}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,6 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata
+
+*.DS_Store
diff --git a/README.md b/README.md
@@ -1,27 +1,27 @@
-<mark style="background-color: #FFE331">**Below is a template of the README.md file for your hub. Italics in brackets are placeholders for information about your hub.  **</mark>
-
-
-# *[Insert title of hub]*
-*[Describe overall purpose of hub.]* Anyone interested in using these data for additional research or publications is requested to contact *[insert email]* for information regarding attribution of the source forecasts.
-
-## *[Insert title of forecasts]*
-*[Describe forecasts ]*
-
-**Dates:** The Challenge Period will begin *[insert start date]* and will run until *[insert start date]*. Participants are currently asked to submit *[insert description of forecasts]*  by *[insert timing]* .(herein referred to as the Forecast Due Date). In the event that timelines of data availability change, *[insert name of hub]*  may change the day of week that forecasts are due. In this case, participants would be notified at least one week in advance. *[insert temporal period]*  submissions (including file names) will be specified in terms of the reference date, which is the Saturday following the Forecast Due Date. The reference date is the last day of the epidemiological week (EW) (Sunday to Saturday) containing the Forecast Due Date.
-
-**Prediction Targets:**
-Participating teams are asked to provide *[insert geographical requirements]*  predictions for  targets *[insert name of target(s)]*. 
-
-Teams will submit  *[insert description of forecasts]* for the epidemiological week (EW) ending on the reference date as well as  *[insert horizons]*. Teams can but are not required to submit forecasts for all *[insert temporal period]* horizons or for all locations. The evaluation data for forecasts will be the *[insert temporal period]* aggregate of *[insert description of evaluation data]*  We will use the specification of EWs defined by the [CDC](https://wwwn.cdc.gov/nndss/document/MMWR_Week_overview.pdf), which run Sunday through Saturday. The target end date for a prediction is the Saturday that ends an EW of interest, and can be calculated using the expression: 
-**target end date = reference date + horizon * (*[insert # days in temporal period]* days)**.
-
-There are standard software packages to convert from dates to epidemic weeks and vice versa (e.g. [MMWRweek](https://cran.r-project.org/web/packages/MMWRweek/) and [lubridate](https://lubridate.tidyverse.org/reference/week.html) for R and [pymmwr](https://pypi.org/project/pymmwr/) and [epiweeks](https://pypi.org/project/epiweeks/) for Python). 
-
-
-If you have questions aboutthis target, please reach out to *[insert name]*  (*[insert email]* ). 
-
-## Acknowledgments
-This repository follows the guidelines and standards outlined by the [hubverse]([url](https://hubverse.io/en/latest/)), which provides a set of data formats and open source tools for modeling hubs.
-
-
-<mark style="background-color: #FFE331">**As an example, here is the link to [Flusight-Forecast_Hub  README](https://github.com/cdcepi/FluSight-forecast-hub/blob/master/README.md).  **</mark>
+# Flu MetroCast Hub
+
+The Flu MetroCast Hub is a modeling hub with the goal of collecting city- and county-level forecasts of influenza activity. It is a project led by the epiENGAGE team from UT-Austin and UMass-Amherst, as a part of the CDC Insight Net program. Anyone interested in using these data for additional research or publications is requested to contact [Dongah Kim](mailto:[email protected]) for information regarding attribution of the source forecasts.
+
+The Flu MetroCast Hub makes public forecasts for New York City, and is open to adding forecasts for other jurisdictions that have public data. For New York City, the influenza-related data are Emergency Department (ED) visits due to Influenza-like Illness (ILI), but other data sources could be modeled as well, such as the percentage of outpatient visits with a primary complaint of ILI or count of all hospitalizations due to influenza.
+
+## City and county level forecasts
+
+**Dates:** The Challenge Period will begin on January 22, 2025 and will run until May 2025. Participants are currently asked to submit forecasts by Wednesday evenings at 8pm ET (herein referred to as the Forecast Due Date). In the event that timelines of data availability change, Flu MetroCast may change the day of week that forecasts are due. In this case, participants would be notified at least one week in advance. Weekly submissions will be specified in terms of the reference date, which is the Saturday following the Forecast Due Date. The reference date is the last day of the epidemiological week (EW) (Sunday to Saturday) containing the Forecast Due Date. The reference date must be included in the file name for any model submission.
+
+**Prediction Targets:** Participating teams are asked to provide city- or county-level predictions for the target relevant to each jurisdiction.
+
+| Jurisdiction | Target name | Target description |
+|------------------------|------------------------|------------------------|
+| New York City (NYC) | ILI ED visits | Number of Emergency Department visits due to influenza |
+
+For NYC, teams will submit predictions of new ED visits due to ILI for the epidemiological week (EW) ending on the reference date (horizon = 0) as well as for horizons 1 through 4. Teams can but are not required to submit forecasts for all horizons.
+
+The evaluation data for forecasts will be the weekly aggregate of daily ED visits for each jurisdiction. We will use the specification of EWs defined by the [CDC](https://wwwn.cdc.gov/nndss/document/MMWR_Week_overview.pdf), which run Sunday through Saturday. The target end date for a prediction is the Saturday that ends an EW of interest, and can be calculated using the expression: **target end date = reference date + horizon \* (7 days)**.
+
+There are standard software packages to convert from dates to epidemic weeks and vice versa (e.g. [MMWRweek](https://cran.r-project.org/web/packages/MMWRweek/) and [lubridate](https://lubridate.tidyverse.org/reference/week.html) for R and [pymmwr](https://pypi.org/project/pymmwr/) and [epiweeks](https://pypi.org/project/epiweeks/) for Python).
+
+If you have questions about this target, please reach out to [Dongah Kim](mailto:[email protected]).
+
+## Acknowledgments
+
+This repository follows the guidelines and standards outlined by the [hubverse](%5Burl%5D(https://hubverse.io/en/latest/)), which provides a set of data formats and open source tools for modeling hubs.
diff --git a/flu-metrocast.Rproj b/flu-metrocast.Rproj
@@ -0,0 +1,17 @@
+Version: 1.0
+ProjectId: 01426fba-1997-482c-b607-6166c0fe643b
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX
+
+AutoAppendNewline: Yes
+StripTrailingWhitespace: Yes
diff --git a/hub-config/admin.json b/hub-config/admin.json
@@ -1,16 +1,16 @@
 {
     "schema_version": "https://raw.githubusercontent.com/hubverse-org/schemas/main/v5.0.0/admin-schema.json",
-    "name": "Template Forecast Hub",
-    "maintainer": "Consortium of Infectious Disease Modeling Hubs",
+    "name": "Flu MetroCast",
+    "maintainer": "epiENGAGE",
     "contact": {
-        "name": "Joe Bloggs",
-        "email": "j.bloggs@cidmh.com"
+        "name": "Dongah Kim",
+        "email": "donga0223@gmail.com"
     },
     "repository": {
         "host": "github",
-        "owner": "hubverse-org",
-        "name": "hubTemplate"
+        "owner": "reichlab",
+        "name": "flu-metrocast"
     },
-    "file_format": ["csv", "parquet"],
+    "file_format": ["csv"],
     "timezone": "US/Eastern"
 }
diff --git a/hub-config/model-metadata-schema.json b/hub-config/model-metadata-schema.json
@@ -1,6 +1,6 @@
 {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "title": "Schema for Modeling Hub model metadata",
+    "title": "Schema for Flu MetroCast Hub model metadata",
     "description": "This is the schema for model metadata files, please refer to https://github.com/covid19-forecast-hub-europe/covid19-forecast-hub-europe/wiki/Metadata for more information.",
     "type": "object",
     "properties": {
@@ -47,18 +47,10 @@
                         "type": "string",
                         "format": "email"
                     },
-                    "twitter": {
-                        "type": "string"
-                    },
                     "additionalProperties": false
                 }
             }
         },
-        "website_url": {
-            "description": "Public facing website for the model",
-            "type": "string",
-            "format": "uri"
-        },
         "repo_url": {
             "description": "Repository containing code for the model",
             "type": "string",
@@ -91,40 +83,34 @@
                 "National Institutes of General Medical Sciences (R01GM123456). The content is solely the responsibility of the authors and does not necessarily represent the official views of NIGMS."
             ]
         },
-        "model_details": {
-            "description": "Structured information about the model",
-            "type": "object",
-            "properties": {
-                "data_inputs": {
-                    "description": "List or description of data inputs used by the model",
-                    "type": "string"
-                },
-                "methods": {
-                    "description": "A brief (200 char.) description of the methods used by this model",
-                    "type": "string",
-                    "maxLength": 200
-                },
-                "methods_long": {
-                    "description": "A full description of the methods used by this model.",
-                    "type": "string"
-                }
-            },
-            "additionalProperties": false,
-            "required": ["data_inputs", "methods"]
+        "methods": {
+            "description": "A summary of the methods used by this model (5000 character limit).",
+            "type": "string",
+            "maxLength": 5000
+        },
+        "methods_url": {
+            "description": "A link to a complete write-up of the model specification, with mathematical details. This could be a peer-reviewed article, preprint, or an unpublished PDF or webpage stored at a public url somewhere.",
+            "type": "string",
+            "format": "uri"
+        },
+        "data_sources": {
+            "description": "List or description of data inputs used by the model. For example: public data from NYC DOHMH, CDC NSSP surveillance data, etc...",
+            "type": "string"
         },
         "ensemble_of_hub_models": {
             "description": "Indicator for whether this model is an ensemble of other Hub models",
             "type": "boolean"
         }
     },
-    "additionalProperties": true,
+    "additionalProperties": false,
     "required": [
         "team_name",
         "team_abbr",
         "model_name",
         "model_abbr",
         "model_contributors",
         "license",
-        "model_details"
+        "methods",
+        "data_sources"
     ]
 }