Merge pull request #1178 from cmu-delphi/release/delphi-epidata-4.1.0

* API keys (#1115)

* chore: release delphi-epidata 4.1.0

---------

Co-authored-by: Katie Mazaitis <[email protected]>
Co-authored-by: Dmytro Trotsko <[email protected]>
Co-authored-by: Brian Clark <[email protected]>
Co-authored-by: george haff <[email protected]>

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.12
+current_version = 4.1.0
 commit = False
 tag = False
 

.dockerignore

@@ -1,6 +1,7 @@
 /delphi-epidata
-/.mypy_cache
+**/.mypy_cache
 /.github
 /docs
-__pycache__
-/node_modules
+**/__pycache__
+**/.pytest_cache
+**/node_modules

.env.example

@@ -1,18 +1,6 @@
 FLASK_DEBUG=True
 SQLALCHEMY_DATABASE_URI=sqlite:///test.db
 FLASK_SECRET=abc
-SECRET_TWITTER=abc
-SECRET_GHT=abc
-SECRET_FLUVIEW=abc
-SECRET_CDC=abc
-SECRET_SENSORS=abc
-SECRET_SENSOR_TWTR=abc
-SECRET_SENSOR_GFT=abc
-SECRET_SENSOR_GHT=abc
-SECRET_SENSOR_GHTJ=abc
-SECRET_SENSOR_CDC=abc
-SECRET_SENSOR_QUID=abc
-SECRET_SENSOR_WIKI=abc
-SECRET_QUIDEL=abc
-SECRET_NOROSTAT=abc
-SECRET_AFHSB=abc
+#API_KEY_REQUIRED_STARTING_AT=2021-07-30
+API_KEY_ADMIN_PASSWORD=abc
+API_KEY_REGISTER_WEBHOOK_TOKEN=abc

.github/workflows/ci.yaml

@@ -53,24 +53,31 @@ jobs:
         run: |
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
           docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
+          sudo docker build -t delphi_redis -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
           cd ./repos/delphi/delphi-epidata
           docker build -t delphi_web_epidata -f ./devops/Dockerfile .
           cd ../../../
 
       # MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
       # see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
-      - name: Start services
+      - name: Start database and Redis services
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
-          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
-          docker ps
+          docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis
+          
 
       - run: |
           wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for
           chmod +x wait-for
           ./wait-for localhost:13306 -- echo 'ready'
           sleep 10s
+
+      - name: Start delphi_web_epidata
+        run: |
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker ps
+
       - name: Run Unit Tests
         run: |
           docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests
@@ -81,7 +88,7 @@ jobs:
 
       - name: Clean Up
         run: |
-          docker stop delphi_database_epidata delphi_web_epidata
+          docker stop delphi_database_epidata delphi_web_epidata delphi_redis
           docker network remove delphi-net
 
   build_js_client:
@@ -108,7 +115,8 @@ jobs:
   image:
     needs: build
     # only on main and dev branch
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    # TODO: #1112 Remove `|| github.ref == 'refs/heads/api-keys'` after transition to production status.
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/api-keys'
 
     runs-on: ubuntu-latest
     steps:

dev/docker/python/Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.8-buster
 
+RUN apt-get update && apt-get install -y r-base && Rscript -e "install.packages(c('httr','xml2'))"
+
 WORKDIR /usr/src/app
 
 COPY repos repos

dev/docker/redis/Dockerfile

@@ -0,0 +1,5 @@
+FROM redis
+
+ENV REDIS_PASSWORD=$REDIS_PASSWORD
+
+CMD ["sh", "-c", "exec redis-server --requirepass \"$REDIS_PASSWORD\""]

dev/local/Makefile

@@ -64,8 +64,10 @@ CWD:=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 NOW:=$(shell date "+%Y-%m-%d")
 LOG_WEB:=delphi_web_epidata_$(NOW).log
 LOG_DB:=delphi_database_epidata_$(NOW).log
+LOG_REDIS:=delphi_redis_instance_$(NOW).log
 WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
 DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
+REDIS_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_redis')
 
 M1=
 ifeq ($(shell uname -smp), Darwin arm64 arm)
@@ -98,6 +100,10 @@ web:
 		--env "MODULE_NAME=delphi.epidata.server.main" \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
+		--env "REDIS_HOST=delphi_redis" \
+		--env "REDIS_PASSWORD=1234" \
+		--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
+		--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
 		--network delphi-net --name delphi_web_epidata \
 		delphi_web_epidata >$(LOG_WEB) 2>&1 &
 
@@ -136,8 +142,25 @@ py:
 		$(M1) \
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 
+.PHONY=redis
+redis:
+	@# Stop container if running
+	@if [ $(REDIS_CONTAINER_ID) ]; then\
+		docker stop $(REDIS_CONTAINER_ID);\
+	fi
+
+	@docker build -t delphi_redis \
+		$(M1) \
+		-f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
+
+	@docker run --rm -d -p 127.0.0.1:6379:6379 \
+		$(M1) \
+		--network delphi-net \
+		--env "REDIS_PASSWORD=1234" \
+		--name delphi_redis delphi_redis >$(LOG_REDIS) 2>&1 &
+
 .PHONY=all
-all: db web py
+all: db web py redis
 
 .PHONY=test
 test:
@@ -149,6 +172,17 @@ test:
 		--env "FLASK_SECRET=abc" \
 		delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
 
+.PHONY=r-test
+r-test:
+	@docker run -i --rm --network delphi-net \
+		$(M1) \
+		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
+		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
+		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
+		--env "FLASK_SECRET=abc" \
+		delphi_web_python Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log
+
+
 .PHONY=bash
 bash:
 	@docker run -it --rm --network delphi-net \

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 0.4.12
+version = 4.1.0
 
 [options]
 packages =
@@ -28,6 +28,8 @@ packages =
     delphi.epidata.acquisition.wiki
     delphi.epidata.client
     delphi.epidata.server
+    delphi.epidata.server.admin
+    delphi.epidata.server.admin.templates
     delphi.epidata.server.covidcast_issues_migration
     delphi.epidata.server.endpoints
     delphi.epidata.server.endpoints.covidcast_utils

docs/api/api_keys.md

@@ -16,11 +16,11 @@ change as we learn more about their impact:
 
 For example, a query for three signals on one date across all counties can be
 submitted anonymously, but a query for three signals on a period of four weeks
-across all counties requires an API key. 
+across all counties requires an API key.
 
 An API key is a pseudonymous access token that grants privileged access to the
-Epidata API. You can request an API key by 
-[registering with us](https://forms.gle/hkBr5SfQgxguAfEt7).
+Epidata API. You can request an API key by
+[registering with us](https://api.delphi.cmu.edu/epidata/admin/registration_form).
 Privileges of registration may include:
 
 1. no rate limit
@@ -36,7 +36,8 @@ store the information you provide us at registration time, see our
 
 ## Usage
 
-If you choose to [register for an API key](https://forms.gle/hkBr5SfQgxguAfEt7), 
+If you choose to
+[register for an API key](https://api.delphi.cmu.edu/epidata/admin/registration_form),
 there are several ways to use your key to authenticate your requests:
 
 ### Via request parameter

docs/api/privacy_statement.md

@@ -29,7 +29,7 @@ purposes:
 * to identify excessive or abnormal usage patterns which may harm our system
 
 The logs are only available to members of our operations team, and are expunged
-at or before they reach five years in age.  
+at or before they reach five years in age.
 
 If you provide us with your email address, we will only use it to contact you in
 the following scenarios:
@@ -47,9 +47,9 @@ security practices to help us keep your information secure. We only retrieve
 this mapping to resolve cases of excessive or abnormal usage. We automatically
 disassociate an email address from its API key when the API key has not been
 used for six months, or upon user request. You can request that your
-email address be removed from our records by filling out a 
-[deletion request](https://forms.gle/GucFmZHTMgEFjH197).
+email address be removed from our records by filling out a
+[deletion request](https://api.delphi.cmu.edu/epidata/admin/removal_request).
 
-For more information, see 
+For more information, see
 [Carnegie Mellon’s privacy notice](https://www.cmu.edu/legal/privacy-notice.html).
 Further questions can be directed to [email protected].

docs/index.md

@@ -14,14 +14,14 @@ group](https://delphi.cmu.edu/). The Epidata API includes:
   quick access to COVID data are available.
 - [Data about other diseases](api/README.md), including influenza, dengue, and
   other diseases tracked by Delphi through various data streams.
-  
+
 Anyone may access the Epidata API anonymously without providing any personal
 data. Anonymous API access is currently rate-limited and restricted to public
 datasets with a maximum of two of the requested parameters having multiple
 selections (signals, dates, versions, regions, etc).
 
 To request access with no rate limit and unlimited multiple
-selections, you can [request a registered API key](https://forms.gle/hkBr5SfQgxguAfEt7).
+selections, you can [request a registered API key](https://api.delphi.cmu.edu/epidata/admin/registration_form).
 For policy and usage details, consult the [Epidata API keys documentation](api/api_keys.md).
 
 If you regularly or frequently use our system, please consider using an API key
@@ -30,10 +30,11 @@ us understand who and how others are using our Delphi Epidata API, which may in
 turn inform our future research, data partnerships, and funding.
 
 For more information about how we use the data you provide us through your
-registration and API request activity, see our 
-[Privacy Statement](api/privacy_statement.md). At any time, you may submit a 
-[Deletion Request](https://forms.gle/GucFmZHTMgEFjH197) to have us deactivate your key and destroy all
-information associating that key with your identity.
+registration and API request activity, see our
+[Privacy Statement](api/privacy_statement.md). At any time, you may submit a
+[Deletion Request](https://api.delphi.cmu.edu/epidata/admin/removal_request) to
+have us deactivate your key and destroy all information associating that key
+with your identity.
 
 The Delphi group is extremely grateful to Pedrito Maynard-Zhang for all his
 help with the Epidata API [documentation](api/README.md).

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -29,6 +29,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -40,6 +41,8 @@ def setUp(self):
         cur.execute('truncate table covid_hosp_facility')
         cur.execute('truncate table covid_hosp_facility_key')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, email) values ("key", "emai")')
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -33,6 +33,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -43,6 +44,8 @@ def setUp(self):
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_state_timeseries')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, email) values("key", "email")')
 
   @freeze_time("2021-03-16")
   def test_acquire_dataset(self):

integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py

@@ -29,6 +29,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
     secrets.db.host = 'delphi_database_epidata'
@@ -39,6 +40,8 @@ def setUp(self):
       with db.new_cursor() as cur:
         cur.execute('truncate table covid_hosp_state_timeseries')
         cur.execute('truncate table covid_hosp_meta')
+        cur.execute('delete from api_user')
+        cur.execute('insert into api_user(api_key, email) values ("key", "email")')
 
   @freeze_time("2021-03-17")
   def test_acquire_dataset(self):

integrations/acquisition/covidcast/test_covidcast_meta_caching.py

@@ -60,12 +60,20 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = BASE_URL
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""
     self.cur.close()
     self.cnx.close()
 
+  @staticmethod
+  def _make_request():
+    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
+    response = requests.get(Epidata.BASE_URL, params=params, auth=Epidata.auth)
+    response.raise_for_status()
+    return response.json()
+
   def test_caching(self):
     """Populate, query, cache, query, and verify the cache."""
 
@@ -147,10 +155,7 @@ def test_caching(self):
     self.cnx.commit()
 
     # fetch the cached version (manually)
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
-    response = requests.get(BASE_URL, params=params)
-    response.raise_for_status()
-    epidata4 = response.json()
+    epidata4 = self._make_request()
 
     # make sure the cache was actually served
     self.assertEqual(epidata4, {
@@ -170,10 +175,7 @@ def test_caching(self):
     self.cnx.commit()
 
     # fetch the cached version (manually)
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
-    response = requests.get(BASE_URL, params=params)
-    response.raise_for_status()
-    epidata5 = response.json()
+    epidata5 = self._make_request()
 
     # make sure the cache was returned anyhow
     self.assertEqual(epidata4, epidata5)

integrations/acquisition/covidcast/test_csv_uploading.py

@@ -57,6 +57,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""

integrations/acquisition/covidcast_nowcast/test_csv_uploading.py

@@ -41,6 +41,8 @@ def setUp(self):
         database='epidata')
     cur = cnx.cursor()
     cur.execute('truncate table covidcast_nowcast')
+    cur.execute('delete from api_user')
+    cur.execute('insert into api_user(api_key, email) values ("key", "email")')
     cnx.commit()
     cur.close()
 
@@ -54,6 +56,7 @@ def setUp(self):
 
     # use the local instance of the Epidata API
     Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
     """Perform per-test teardown."""