Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DISCO-3185 Add metrics for manifest #770

Merged
merged 1 commit into from
Jan 29, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions merino/providers/manifest/backends/filemanager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from merino.providers.manifest.backends.protocol import ManifestData, GetManifestResultCode
from merino.utils.gcs.gcp_uploader import GcsUploader
from merino.utils.metrics import get_metrics_client

logger = logging.getLogger(__name__)

Expand All @@ -31,30 +32,38 @@ def __init__(self, gcs_project_path: str, gcs_bucket_path: str, blob_name: str)

def get_file(self) -> tuple[GetManifestResultCode, ManifestData | None]:
"""Fetch the remote manifest file from GCS"""
metrics_client = get_metrics_client()

try:
blob = self.gcs_client.get_file_by_name(self.blob_name, self.blob_generation)

if blob is not None:
blob.reload()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice add!

metrics_client.gauge("manifest.size", value=blob.size)

blob_data = blob.download_as_text()

try:
manifest_content = ManifestData.model_validate(json.loads(blob_data))
metrics_client.gauge(
"manifest.domains.count", value=len(manifest_content.domains)
)
except JSONDecodeError as json_error:
logger.error("Failed to decode manifest JSON: %s", json_error)
return (GetManifestResultCode.FAIL, None)
return GetManifestResultCode.FAIL, None
except ValidationError as val_err:
logger.error(f"Invalid manifest content: {val_err}")
return (GetManifestResultCode.FAIL, None)
return GetManifestResultCode.FAIL, None

logger.info("Successfully loaded remote manifest file: %s", self.blob_name)
return (GetManifestResultCode.SUCCESS, manifest_content)
return GetManifestResultCode.SUCCESS, manifest_content

# If `get_file_by_name` returned None, that usually means no new generation (SKIP).
logger.info(
f"No new GCS generation for {self.blob_name}; returning SKIP or blob doesn't exist.",
)
return (GetManifestResultCode.SKIP, None)
return GetManifestResultCode.SKIP, None

except Exception as e:
logger.error(f"Error fetching remote manifest file {self.blob_name}: {e}")
return (GetManifestResultCode.FAIL, None)
return GetManifestResultCode.FAIL, None
6 changes: 3 additions & 3 deletions merino/providers/manifest/backends/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,12 @@ def fetch_manifest_data(self) -> tuple[GetManifestResultCode, ManifestData | Non
match GetManifestResultCode(result_code):
case GetManifestResultCode.SUCCESS:
logger.info("Manifest data loaded remotely from GCS.")
return (result_code, manifest_data)
return result_code, manifest_data

case GetManifestResultCode.SKIP:
logger.info("Manifest data was not updated (SKIP).")
return (result_code, None)
return result_code, None

case GetManifestResultCode.FAIL:
logger.error("Failed to fetch manifest from GCS (FAIL).")
return (result_code, None)
return result_code, None