diff --git a/brozzler/metrics.py b/brozzler/metrics.py index 81239bcc..c2c7665c 100644 --- a/brozzler/metrics.py +++ b/brozzler/metrics.py @@ -17,13 +17,18 @@ from prometheus_client import Counter, Gauge, Histogram, start_http_server # fmt: off -brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler") +brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler") brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler") +brozzler_in_progress_headers = Gauge("brozzler_in_progress_headers", "number of headers currently processing with brozzler") +brozzler_header_processing_duration_seconds = Histogram("brozzler_header_processing_duration_seconds", "time spent processing one page's headers in brozzler") +brozzler_in_progress_browses = Gauge("brozzler_in_progress_browses", "number of pages currently browsing with brozzler") +brozzler_browsing_duration_seconds = Histogram("brozzler_browsing_duration_seconds", "time spent browsing a page in brozzler") +brozzler_in_progress_ytdlps = Gauge("brozzler_in_progress_ytdlps", "number of ytdlp sessions currently in progress with brozzler") +brozzler_ytdlp_duration_seconds = Histogram("brozzler_ytdlp_duration_seconds", "time spent running ytdlp for a page in brozzler") +brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler") brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler") brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch") -brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler") brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp") -brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"]) # fmt: on diff --git a/brozzler/worker.py b/brozzler/worker.py index 6eb58720..116bb75b 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -251,6 +251,8 @@ def thumb_jpeg(self, full_jpeg): img.save(out, "jpeg", quality=95) return out.getbuffer() + @metrics.brozzler_page_processing_duration_seconds.time() + @metrics.brozzler_in_progress_pages.track_inprogress() def brozzle_page( self, browser, @@ -315,6 +317,8 @@ def brozzle_page( ) return outlinks + @metrics.brozzler_header_processing_duration_seconds.time() + @metrics.brozzler_in_progress_headers.track_inprogress() def _get_page_headers(self, page): # bypassing warcprox, requests' stream=True defers downloading the body of the response # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow @@ -334,8 +338,8 @@ def _needs_browsing(self, page_headers): return False return True - @metrics.brozzler_page_processing_duration_seconds.time() - @metrics.brozzler_in_progress_pages.track_inprogress() + @metrics.brozzler_browsing_duration_seconds.time() + @metrics.brozzler_in_progress_browses.track_inprogress() def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None): def update_page_metrics(page, outlinks): """Update page-level Prometheus metrics.""" diff --git a/brozzler/ydl.py b/brozzler/ydl.py index f3f21bea..09593f10 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -334,7 +334,6 @@ def _try_youtube_dl(worker, ydl, site, page): # ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid) # if ydl.is_youtube_host and ie_result: # download_url = ie_result.get("url") - metrics.brozzler_ydl_extract_attempts.labels(ydl.is_youtube_host).inc(1) with brozzler.thread_accept_exceptions(): # we do whatwg canonicalization here to avoid "" resulting in ProxyError @@ -406,6 +405,8 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result +@metrics.brozzler_ytdlp_duration_seconds.time() +@metrics.brozzler_in_progress_ytdlps.track_inprogress() def do_youtube_dl(worker, site, page): """ Runs yt-dlp configured for `worker` and `site` to download videos from