Merge pull request #199 from catalyst-cooperative/save-daily-metrics-fix

Fix `save_daily_metrics.yml` import error

src/usage_metrics/helpers.py

@@ -5,12 +5,10 @@
 import os
 from datetime import UTC, datetime, timezone
 from pathlib import Path
-from urllib.error import HTTPError
 from urllib.parse import urlparse
 
 import ipinfo
 import pandas as pd
-import requests
 from dagster import OutputContext, RetryPolicy, op
 from joblib import Memory
 
@@ -156,28 +154,3 @@ def get_table_name_from_context(context: OutputContext) -> str:
     if context.has_asset_key:
         return context.asset_key.to_python_identifier()
     return context.get_identifier()
-
-
-def make_request(
-    url: str, headers: str | None = None, params: str | None = None, timeout: int = 100
-) -> requests.models.Response:
-    """Makes a request with some error handling.
-
-    Args:
-        query (str): A request url.
-        headers (str): Header to include in the request.
-        params (str): Params of request.
-        timeout (int): Timeout of request (in seconds).
-
-    Returns:
-        response (requests.models.Response): the request response.
-    """
-    try:
-        response = requests.get(url, headers=headers, params=params, timeout=timeout)
-
-        response.raise_for_status()
-    except HTTPError as http_err:
-        raise HTTPError(
-            f"HTTP error occurred: {http_err}\n\tResponse text: {response.text}"
-        )
-    return response

src/usage_metrics/scripts/save_github_metrics.py

@@ -8,10 +8,9 @@
 from dataclasses import dataclass
 from datetime import date
 
+import requests
 from google.cloud import storage
 
-from usage_metrics.helpers import make_request
-
 logger = logging.getLogger()
 logging.basicConfig(level="INFO")
 
@@ -39,7 +38,7 @@ def get_biweekly_metrics(owner: str, repo: str, token: str, metric: str) -> str:
         "Accept": "application/vnd.github.v3+json",
     }
 
-    response = make_request(url=url, headers=headers)
+    response = requests.get(url, headers=headers, timeout=100)
     return json.dumps(response.json())
 
 
@@ -66,7 +65,9 @@ def get_persistent_metrics(owner: str, repo: str, token: str, metric: str) -> st
 
     while time.time() < timeout_start + timeout:
         params = {"page": page}
-        metrics_json = make_request(url=url, headers=headers, params=params).json()
+        metrics_json = requests.get(
+            url=url, headers=headers, params=params, timeout=100
+        ).json()
 
         if len(metrics_json) <= 0:
             break

src/usage_metrics/scripts/save_zenodo_metrics.py

@@ -7,11 +7,10 @@
 from typing import Annotated
 
 import pandas as pd
+import requests
 from google.cloud import storage
 from pydantic import BaseModel, StringConstraints
 
-from usage_metrics.helpers import make_request
-
 Doi = Annotated[str, StringConstraints(pattern=r"10\.5281/zenodo\.\d+")]
 SandboxDoi = Annotated[str, StringConstraints(pattern=r"10\.5072/zenodo\.\d+")]
 
@@ -55,7 +54,7 @@ def save_zenodo_logs() -> pd.DataFrame():
 
     community_url = "https://zenodo.org/api/communities/14454015-63f1-4f05-80fd-1a9b07593c9e/records"
     # First, get metadata on all the datasets in the Catalyst Cooperative community
-    community_records = make_request(url=community_url)
+    community_records = requests.get(url=community_url, timeout=100)
     dataset_records = community_records.json()["hits"]["hits"]
     dataset_records = [CommunityMetadata(**record) for record in dataset_records]
 
@@ -64,7 +63,7 @@ def save_zenodo_logs() -> pd.DataFrame():
         # For each dataset in the community, get all archived versions and their
         # corresponding metrics.
         versions_url = f"https://zenodo.org/api/records/{record.recid}/versions"
-        record_versions = make_request(url=versions_url).json()
+        record_versions = requests.get(url=versions_url, timeout=100).json()
         versions_metadata = json.dumps(record_versions)
         blob_name = f"zenodo/{date.today().strftime('%Y-%m-%d')}-{record.recid}.json"
         upload_to_bucket(bucket=bucket, blob_name=blob_name, data=versions_metadata)