From 4f31fbc02665e8971cd8a0fc03ce883ac09ff0de Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Fri, 25 Oct 2024 09:36:54 -0400
Subject: [PATCH 01/11] log exporter code

---
 .../config/proxy/docker-compose-extra.yml     | 13 ++++
 .../config/thredds/docker-compose-extra.yml   |  5 ++
 .../config/thredds/prometheus-log-exporter.py | 44 +++++++++++++
 .../prometheus-log-parser/default.env         | 11 ++++
 .../docker-compose-extra.yml                  | 24 +++++++
 .../prometheus-log-parser/entrypoint.sh       |  5 ++
 .../prometheus-log-exporter.py                | 65 +++++++++++++++++++
 .../prometheus-log-parser/requirements.txt    |  2 +
 8 files changed, 169 insertions(+)
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/config/proxy/docker-compose-extra.yml
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/config/thredds/docker-compose-extra.yml
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/default.env
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/entrypoint.sh
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/requirements.txt

diff --git a/birdhouse/optional-components/prometheus-log-parser/config/proxy/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-log-parser/config/proxy/docker-compose-extra.yml
new file mode 100644
index 000000000..0b0308c40
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/config/proxy/docker-compose-extra.yml
@@ -0,0 +1,13 @@
+version: "3.4"
+services:
+  proxy:
+    volumes:
+      - proxy-logs:${PROXY_LOG_DIR}
+  prometheus-log-parser:
+    volumes:
+      - proxy-logs:/var/log/proxy
+    environment:
+      - PROXY_LOG_FILE=${PROXY_LOG_FILE}
+
+volumes:
+  proxy-logs:
diff --git a/birdhouse/optional-components/prometheus-log-parser/config/thredds/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-log-parser/config/thredds/docker-compose-extra.yml
new file mode 100644
index 000000000..0b344f0f8
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/config/thredds/docker-compose-extra.yml
@@ -0,0 +1,5 @@
+version: "3.4"
+services:
+  prometheus-log-parser:
+    volumes:
+      - ./optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py:${PROMETHEUS_LOG_PARSER_EXPORTERS_DIR}/thredds-prometheus-log-exporter.py:ro
diff --git a/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py b/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
new file mode 100644
index 000000000..cb9fa5aba
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
@@ -0,0 +1,44 @@
+import os
+import re
+
+import prometheus_client
+
+
+# This matches a request to the THREDDS data services as defined in birdhouse/components/thredds/catalog.xml.template 
+THREDDS_REQ_URI_REGEX = r'\/[^\s]+\/thredds\/(?P<tds_service>dodsC|fileServer|ncss)\/(?P<dataset>[^\s]*)(?:\?(?P<variable>\w+))?'
+
+# This matches the nginx log_fomat as defined in birdhouse/components/proxy/nginx.conf.template
+REGEX = re.compile(
+    r'(?P<remote_addr>(?:^|\b(?<!\.))(?:1?\d\d?|2[0-4]\d|25[0-5])(?:\.(?:1?\d\d?|2[0-4]\d|25[0-5])){3}(?=$|[^\w.]))\s'
+    r'-\s'
+    r'(?P<remote_usr>-|[a-z_][a-z0-9_]{0,30})\s'
+    r'(?P<date_time>\[(?P<date>\d\d\d\d-\d\d-\d\d)T(?P<time>\d\d:\d\d:\d\d).*\])\s'
+    r'(?P<request>\"'
+        r'(?P<req_method>GET|POST|HEAD|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH)\s'
+        fr'(?P<req_uri>{THREDDS_REQ_URI_REGEX})\s'
+        r'(?P<http_ver>HTTP/\d\.\d)'
+        r'\")\s'
+    r'(?P<status>\d{3})\s'
+    r'(?P<body_byte_sent>\d+)\s'
+    r'\"(?P<http_referer>[^\s]+)\"\s'
+    r'\"(?P<user_agent>[^\"]+)\"\s'
+    r'\"(?P<forward_for>[^\"]+)\"')
+
+LABEL_KEYS = ("remote_addr", "date", "tds_service", "dataset", "variable")
+
+Counter = prometheus_client.Counter(
+    name="thredds_transfer_size_kb",
+    documentation="THREDDS data transferred",
+    labelnames=LABEL_KEYS,
+    unit="kb",
+)
+
+def parse_line(line):
+    match = REGEX.match(line)
+    if match:
+        labels = {label: match.group(label) or "" for label in LABEL_KEYS}
+        if body_byte_sent := match.group("body_byte_sent") is not None:
+            body_byte_sent = int(body_byte_sent) / 1024
+            Counter.labels(**labels).inc(body_byte_sent)
+
+PROMETHEUS_LOG_EXPORTER_CONFIG = {f"/var/log/proxy/{os.getenv('PROXY_LOG_FILE')}": [parse_line]}
diff --git a/birdhouse/optional-components/prometheus-log-parser/default.env b/birdhouse/optional-components/prometheus-log-parser/default.env
new file mode 100644
index 000000000..e6ff4086e
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/default.env
@@ -0,0 +1,11 @@
+export PROMETHEUS_LOG_PARSER_DOCKER=python
+export PROMETHEUS_LOG_PARSER_VERSION=3.13-alpine
+export PROMETHEUS_LOG_PARSER_IMAGE='${PROMETHEUS_LOG_PARSER_DOCKER}:${PROMETHEUS_LOG_PARSER_VERSION}'
+export PROMETHEUS_LOG_PARSER_CLIENT_PORT=8000
+export PROMETHEUS_LOG_PARSER_EXPORTERS_DIR=/exporters.d
+export PROMETHEUS_LOG_PARSER_POLL_DELAY=1 # time in seconds
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  PROMETHEUS_LOG_PARSER_IMAGE
+"
diff --git a/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
new file mode 100644
index 000000000..9c2101eb7
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
@@ -0,0 +1,24 @@
+version: "3.4"
+
+x-logging: &default-logging
+  driver: "json-file"
+  options:
+    max-size: "50m"
+    max-file: "10"
+
+services:
+  prometheus-log-parser:
+    image: ${PROMETHEUS_LOG_PARSER_IMAGE}
+    container_name: prometheus-log-parser
+    entrypoint: ["sh", "/entrypoint.sh"]
+    command: ["python", "/prometheus-log-exporter.py"]
+    volumes:
+      - ./optional-components/prometheus-log-parser/requirements.txt:/requirements.txt:ro
+      - ./optional-components/prometheus-log-parser/entrypoint.sh:/entrypoint.sh:ro
+      - ./optional-components/prometheus-log-parser/prometheus-log-exporter.py:/prometheus-log-exporter.py:ro
+    environment:
+      - PROMETHEUS_LOG_PARSER_CLIENT_PORT=${PROMETHEUS_LOG_PARSER_CLIENT_PORT}
+      - PROMETHEUS_LOG_PARSER_EXPORTERS_DIR=${PROMETHEUS_LOG_PARSER_EXPORTERS_DIR}
+      - PROMETHEUS_LOG_PARSER_POLL_DELAY=${PROMETHEUS_LOG_PARSER_POLL_DELAY}
+    restart: always
+    logging: *default-logging
diff --git a/birdhouse/optional-components/prometheus-log-parser/entrypoint.sh b/birdhouse/optional-components/prometheus-log-parser/entrypoint.sh
new file mode 100644
index 000000000..dd9100ae8
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/entrypoint.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+python -m pip install -r /requirements.txt
+
+exec "$@"
diff --git a/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py b/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
new file mode 100644
index 000000000..75f511484
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
@@ -0,0 +1,65 @@
+import argparse
+import asyncio
+from collections import defaultdict
+from contextlib import AsyncExitStack
+import glob
+import importlib
+import os
+import sys
+from typing import Callable
+import prometheus_client
+from anyio import AsyncFile, open_file, run
+
+
+def load_exporter_configs(exporters_dir: str) -> dict[str, list[Callable]]:
+    configs = defaultdict(list)
+    sys.path.append(exporters_dir)
+    for file_path in glob.glob(os.path.join(exporters_dir, "*.py")):
+        exporter_module = importlib.import_module(os.path.basename(os.path.splitext(file_path)[0]))
+        for log_file, line_parsers in exporter_module.PROMETHEUS_LOG_EXPORTER_CONFIG.items():
+            configs[log_file].extend(line_parsers)
+    return dict(configs)
+
+
+async def track_file(log_io: AsyncFile[str], line_parsers: list[Callable], poll_delay: int) -> None:
+    while True:
+        async for line in log_io:
+            for line_parser in line_parsers:
+                line_parser(line)
+        await asyncio.sleep(poll_delay)
+
+
+async def export_metrics(configs: dict[str, list[Callable]], poll_delay: int) -> None:
+    tasks = []
+    async with AsyncExitStack() as stack:
+        for log_file, line_parsers in configs.items():
+            f = await stack.enter_async_context(await open_file(log_file))
+            tasks.append(asyncio.ensure_future(track_file(f, line_parsers, poll_delay)))
+
+        await asyncio.gather(*tasks)
+
+
+def main(port: int, exporters_dir: str, poll_delay: int) -> None:
+    prometheus_client.start_http_server(port)
+    run(export_metrics, load_exporter_configs(exporters_dir), poll_delay)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--port", type=int, default=int(os.getenv("PROMETHEUS_LOG_PARSER_CLIENT_PORT", 8000))
+    )
+    parser.add_argument(
+        "--exporters-dir",
+        default=os.getenv(
+            "PROMETHEUS_LOG_PARSER_EXPORTERS_DIR", os.path.join(os.getcwd(), "exporters.d")
+        ),
+    )
+    parser.add_argument(
+        "--poll-delay", type=int, default=int(os.getenv("PROMETHEUS_LOG_PARSER_POLL_DELAY", 1))
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main(**vars(parse_args()))
diff --git a/birdhouse/optional-components/prometheus-log-parser/requirements.txt b/birdhouse/optional-components/prometheus-log-parser/requirements.txt
new file mode 100644
index 000000000..2a6f7754a
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/requirements.txt
@@ -0,0 +1,2 @@
+anyio==4.6.2.post1
+prometheus-client==0.21.0

From 620125035f5d5371705a7656b2b37e74d3b40dc7 Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Fri, 25 Oct 2024 15:35:30 -0400
Subject: [PATCH 02/11] support log file rotation

---
 .../prometheus-log-exporter.py                | 55 +++++++++++++++----
 1 file changed, 43 insertions(+), 12 deletions(-)

diff --git a/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py b/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
index 75f511484..0feeb4803 100644
--- a/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
+++ b/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
@@ -1,7 +1,7 @@
 import argparse
 import asyncio
 from collections import defaultdict
-from contextlib import AsyncExitStack
+from enum import Enum
 import glob
 import importlib
 import os
@@ -11,6 +11,13 @@
 from anyio import AsyncFile, open_file, run
 
 
+class FileStates(Enum):
+    NOCHANGE = 0
+    TRUNCATED = 1
+    DELETED = 2
+    DIFFERENT = 3
+
+
 def load_exporter_configs(exporters_dir: str) -> dict[str, list[Callable]]:
     configs = defaultdict(list)
     sys.path.append(exporters_dir)
@@ -21,22 +28,46 @@ def load_exporter_configs(exporters_dir: str) -> dict[str, list[Callable]]:
     return dict(configs)
 
 
-async def track_file(log_io: AsyncFile[str], line_parsers: list[Callable], poll_delay: int) -> None:
-    while True:
-        async for line in log_io:
-            for line_parser in line_parsers:
-                line_parser(line)
-        await asyncio.sleep(poll_delay)
+async def check_file_state(log_io: AsyncFile[str]):
+    try:
+        same_name_file_stat = os.stat(log_io.name)
+    except FileNotFoundError:
+        return FileStates.DELETED
+    file_stat = os.stat(log_io.fileno())
+    if same_name_file_stat == file_stat:
+        if await log_io.tell() > file_stat.st_size:
+            return FileStates.TRUNCATED
+        else:
+            return FileStates.NOCHANGE
+    return FileStates.DIFFERENT
+
+
+async def track_file(log_file: str, line_parsers: list[Callable], poll_delay: int) -> None:
+    try:
+        log_io = await open_file(log_file)
+        while True:
+            file_state = await check_file_state(log_io)
+            if file_state == FileStates.NOCHANGE:
+                async for line in log_io:
+                    for line_parser in line_parsers:
+                        line_parser(line)
+            elif file_state == FileStates.TRUNCATED:
+                await log_io.seek(0)
+            elif file_state == FileStates.DIFFERENT:
+                await log_io.aclose()
+                log_io = await open_file(log_file)
+            # if file is deleted, do nothing and wait to see if it is recreated later on
+            await asyncio.sleep(poll_delay)
+    finally:
+        await log_io.aclose()
 
 
 async def export_metrics(configs: dict[str, list[Callable]], poll_delay: int) -> None:
     tasks = []
-    async with AsyncExitStack() as stack:
-        for log_file, line_parsers in configs.items():
-            f = await stack.enter_async_context(await open_file(log_file))
-            tasks.append(asyncio.ensure_future(track_file(f, line_parsers, poll_delay)))
+    for log_file, line_parsers in configs.items():
+        tasks.append(asyncio.ensure_future(track_file(log_file, line_parsers, poll_delay)))
 
-        await asyncio.gather(*tasks)
+    await asyncio.gather(*tasks)
 
 
 def main(port: int, exporters_dir: str, poll_delay: int) -> None:

From 5a099bf963310d60bfdc6dce82906638d75c4aee Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Thu, 31 Oct 2024 11:04:01 -0400
Subject: [PATCH 03/11] config updates

---
 .../config/thredds/prometheus-log-exporter.py                | 2 +-
 .../optional-components/prometheus-log-parser/default.env    | 1 +
 .../prometheus-log-parser/docker-compose-extra.yml           | 5 +++--
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py b/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
index cb9fa5aba..3729ce3b2 100644
--- a/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
+++ b/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
@@ -41,4 +41,4 @@ def parse_line(line):
             body_byte_sent = int(body_byte_sent) / 1024
             Counter.labels(**labels).inc(body_byte_sent)
 
-PROMETHEUS_LOG_EXPORTER_CONFIG = {f"/var/log/proxy/{os.getenv('PROXY_LOG_FILE')}": [parse_line]}
+LOG_PARSER_CONFIG = {f"/var/log/proxy/{os.getenv('PROXY_LOG_FILE')}": [parse_line]}
diff --git a/birdhouse/optional-components/prometheus-log-parser/default.env b/birdhouse/optional-components/prometheus-log-parser/default.env
index e6ff4086e..f3152dd2a 100644
--- a/birdhouse/optional-components/prometheus-log-parser/default.env
+++ b/birdhouse/optional-components/prometheus-log-parser/default.env
@@ -4,6 +4,7 @@ export PROMETHEUS_LOG_PARSER_IMAGE='${PROMETHEUS_LOG_PARSER_DOCKER}:${PROMETHEUS
 export PROMETHEUS_LOG_PARSER_CLIENT_PORT=8000
 export PROMETHEUS_LOG_PARSER_EXPORTERS_DIR=/exporters.d
 export PROMETHEUS_LOG_PARSER_POLL_DELAY=1 # time in seconds
+export PROMETHEUS_LOG_PARSER_TAIL=t
 
 export DELAYED_EVAL="
   $DELAYED_EVAL
diff --git a/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
index 9c2101eb7..9f8137d4d 100644
--- a/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
+++ b/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
@@ -18,7 +18,8 @@ services:
       - ./optional-components/prometheus-log-parser/prometheus-log-exporter.py:/prometheus-log-exporter.py:ro
     environment:
       - PROMETHEUS_LOG_PARSER_CLIENT_PORT=${PROMETHEUS_LOG_PARSER_CLIENT_PORT}
-      - PROMETHEUS_LOG_PARSER_EXPORTERS_DIR=${PROMETHEUS_LOG_PARSER_EXPORTERS_DIR}
-      - PROMETHEUS_LOG_PARSER_POLL_DELAY=${PROMETHEUS_LOG_PARSER_POLL_DELAY}
+      - LOG_PARSER_EXPORTERS_DIR=${PROMETHEUS_LOG_PARSER_EXPORTERS_DIR}
+      - LOG_PARSER_POLL_DELAY=${PROMETHEUS_LOG_PARSER_POLL_DELAY}
+      - LOG_PARSER_TAIL=${PROMETHEUS_LOG_PARSER_TAIL}
     restart: always
     logging: *default-logging

From d4e8402e57b6a97fcce33145999dec53269f829e Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Mon, 4 Nov 2024 15:47:06 -0500
Subject: [PATCH 04/11] use dockerhub image

---
 .../config/thredds/docker-compose-extra.yml   |  2 +-
 .../prometheus-log-parser/default.env         |  6 +-
 .../docker-compose-extra.yml                  |  9 +-
 .../prometheus-log-parser/entrypoint.sh       |  5 -
 .../prometheus-log-exporter.py                | 96 -------------------
 .../prometheus-log-parser/requirements.txt    |  2 -
 6 files changed, 6 insertions(+), 114 deletions(-)
 delete mode 100644 birdhouse/optional-components/prometheus-log-parser/entrypoint.sh
 delete mode 100644 birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
 delete mode 100644 birdhouse/optional-components/prometheus-log-parser/requirements.txt

diff --git a/birdhouse/optional-components/prometheus-log-parser/config/thredds/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-log-parser/config/thredds/docker-compose-extra.yml
index 0b344f0f8..eea8c0b8d 100644
--- a/birdhouse/optional-components/prometheus-log-parser/config/thredds/docker-compose-extra.yml
+++ b/birdhouse/optional-components/prometheus-log-parser/config/thredds/docker-compose-extra.yml
@@ -2,4 +2,4 @@ version: "3.4"
 services:
   prometheus-log-parser:
     volumes:
-      - ./optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py:${PROMETHEUS_LOG_PARSER_EXPORTERS_DIR}/thredds-prometheus-log-exporter.py:ro
+      - ./optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py:${PROMETHEUS_LOG_PARSER_PARSERS_DIR}/thredds-prometheus-log-exporter.py:ro
diff --git a/birdhouse/optional-components/prometheus-log-parser/default.env b/birdhouse/optional-components/prometheus-log-parser/default.env
index f3152dd2a..ec7c0820d 100644
--- a/birdhouse/optional-components/prometheus-log-parser/default.env
+++ b/birdhouse/optional-components/prometheus-log-parser/default.env
@@ -1,8 +1,8 @@
-export PROMETHEUS_LOG_PARSER_DOCKER=python
-export PROMETHEUS_LOG_PARSER_VERSION=3.13-alpine
+export PROMETHEUS_LOG_PARSER_DOCKER=marbleclimate/log-parser
+export PROMETHEUS_LOG_PARSER_VERSION=v1.0.1
 export PROMETHEUS_LOG_PARSER_IMAGE='${PROMETHEUS_LOG_PARSER_DOCKER}:${PROMETHEUS_LOG_PARSER_VERSION}'
 export PROMETHEUS_LOG_PARSER_CLIENT_PORT=8000
-export PROMETHEUS_LOG_PARSER_EXPORTERS_DIR=/exporters.d
+export PROMETHEUS_LOG_PARSER_PARSERS_DIR=/parsers.d
 export PROMETHEUS_LOG_PARSER_POLL_DELAY=1 # time in seconds
 export PROMETHEUS_LOG_PARSER_TAIL=t
 
diff --git a/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
index 9f8137d4d..b08d321aa 100644
--- a/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
+++ b/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
@@ -10,15 +10,10 @@ services:
   prometheus-log-parser:
     image: ${PROMETHEUS_LOG_PARSER_IMAGE}
     container_name: prometheus-log-parser
-    entrypoint: ["sh", "/entrypoint.sh"]
-    command: ["python", "/prometheus-log-exporter.py"]
-    volumes:
-      - ./optional-components/prometheus-log-parser/requirements.txt:/requirements.txt:ro
-      - ./optional-components/prometheus-log-parser/entrypoint.sh:/entrypoint.sh:ro
-      - ./optional-components/prometheus-log-parser/prometheus-log-exporter.py:/prometheus-log-exporter.py:ro
+    entrypoint: ["log-parser-prometheus"]
     environment:
       - PROMETHEUS_LOG_PARSER_CLIENT_PORT=${PROMETHEUS_LOG_PARSER_CLIENT_PORT}
-      - LOG_PARSER_EXPORTERS_DIR=${PROMETHEUS_LOG_PARSER_EXPORTERS_DIR}
+      - LOG_PARSER_PARSERS=${PROMETHEUS_LOG_PARSER_PARSERS_DIR}
       - LOG_PARSER_POLL_DELAY=${PROMETHEUS_LOG_PARSER_POLL_DELAY}
       - LOG_PARSER_TAIL=${PROMETHEUS_LOG_PARSER_TAIL}
     restart: always
diff --git a/birdhouse/optional-components/prometheus-log-parser/entrypoint.sh b/birdhouse/optional-components/prometheus-log-parser/entrypoint.sh
deleted file mode 100644
index dd9100ae8..000000000
--- a/birdhouse/optional-components/prometheus-log-parser/entrypoint.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env sh
-
-python -m pip install -r /requirements.txt
-
-exec "$@"
diff --git a/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py b/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
deleted file mode 100644
index 0feeb4803..000000000
--- a/birdhouse/optional-components/prometheus-log-parser/prometheus-log-exporter.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import argparse
-import asyncio
-from collections import defaultdict
-from enum import Enum
-import glob
-import importlib
-import os
-import sys
-from typing import Callable
-import prometheus_client
-from anyio import AsyncFile, open_file, run
-
-
-class FileStates(Enum):
-    NOCHANGE = 0
-    TRUNCATED = 1
-    DELETED = 2
-    DIFFERENT = 3
-
-
-def load_exporter_configs(exporters_dir: str) -> dict[str, list[Callable]]:
-    configs = defaultdict(list)
-    sys.path.append(exporters_dir)
-    for file_path in glob.glob(os.path.join(exporters_dir, "*.py")):
-        exporter_module = importlib.import_module(os.path.basename(os.path.splitext(file_path)[0]))
-        for log_file, line_parsers in exporter_module.PROMETHEUS_LOG_EXPORTER_CONFIG.items():
-            configs[log_file].extend(line_parsers)
-    return dict(configs)
-
-
-async def check_file_state(log_io: AsyncFile[str]):
-    try:
-        same_name_file_stat = os.stat(log_io.name)
-    except FileNotFoundError:
-        return FileStates.DELETED
-    file_stat = os.stat(log_io.fileno())
-    if same_name_file_stat == file_stat:
-        if await log_io.tell() > file_stat.st_size:
-            return FileStates.TRUNCATED
-        else:
-            return FileStates.NOCHANGE
-    return FileStates.DIFFERENT
-
-
-async def track_file(log_file: str, line_parsers: list[Callable], poll_delay: int) -> None:
-    try:
-        log_io = await open_file(log_file)
-        while True:
-            file_state = await check_file_state(log_io)
-            if file_state == FileStates.NOCHANGE:
-                async for line in log_io:
-                    for line_parser in line_parsers:
-                        line_parser(line)
-            elif file_state == FileStates.TRUNCATED:
-                await log_io.seek(0)
-            elif file_state == FileStates.DIFFERENT:
-                await log_io.aclose()
-                log_io = await open_file(log_file)
-            # if file is deleted, do nothing and wait to see if it is recreated later on
-            await asyncio.sleep(poll_delay)
-    finally:
-        await log_io.aclose()
-
-
-async def export_metrics(configs: dict[str, list[Callable]], poll_delay: int) -> None:
-    tasks = []
-    for log_file, line_parsers in configs.items():
-        tasks.append(asyncio.ensure_future(track_file(log_file, line_parsers, poll_delay)))
-
-    await asyncio.gather(*tasks)
-
-
-def main(port: int, exporters_dir: str, poll_delay: int) -> None:
-    prometheus_client.start_http_server(port)
-    run(export_metrics, load_exporter_configs(exporters_dir), poll_delay)
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--port", type=int, default=int(os.getenv("PROMETHEUS_LOG_PARSER_CLIENT_PORT", 8000))
-    )
-    parser.add_argument(
-        "--exporters-dir",
-        default=os.getenv(
-            "PROMETHEUS_LOG_PARSER_EXPORTERS_DIR", os.path.join(os.getcwd(), "exporters.d")
-        ),
-    )
-    parser.add_argument(
-        "--poll-delay", type=int, default=int(os.getenv("PROMETHEUS_LOG_PARSER_POLL_DELAY", 1))
-    )
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    main(**vars(parse_args()))
diff --git a/birdhouse/optional-components/prometheus-log-parser/requirements.txt b/birdhouse/optional-components/prometheus-log-parser/requirements.txt
deleted file mode 100644
index 2a6f7754a..000000000
--- a/birdhouse/optional-components/prometheus-log-parser/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-anyio==4.6.2.post1
-prometheus-client==0.21.0

From c78408fe910598fd7db85db8ccb8b412ff9e6fea Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Tue, 5 Nov 2024 09:38:44 -0500
Subject: [PATCH 05/11] main prometheus listens to log parser client

---
 birdhouse/components/monitoring/default.env                 | 2 +-
 birdhouse/components/monitoring/prometheus.yml.template     | 3 +++
 .../prometheus-log-parser/config/monitoring/.gitignore      | 1 +
 .../config/monitoring/docker-compose-extra.yml              | 5 +++++
 .../config/monitoring/scrape_configs.yml.template           | 6 ++++++
 .../optional-components/prometheus-log-parser/default.env   | 5 +++++
 6 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/config/monitoring/.gitignore
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/config/monitoring/docker-compose-extra.yml
 create mode 100644 birdhouse/optional-components/prometheus-log-parser/config/monitoring/scrape_configs.yml.template

diff --git a/birdhouse/components/monitoring/default.env b/birdhouse/components/monitoring/default.env
index c6f781fde..76188073d 100644
--- a/birdhouse/components/monitoring/default.env
+++ b/birdhouse/components/monitoring/default.env
@@ -8,7 +8,7 @@ export GRAFANA_VERSION="7.0.3"
 export GRAFANA_DOCKER=grafana/grafana
 export GRAFANA_IMAGE='${GRAFANA_DOCKER}:${GRAFANA_VERSION}'
 
-export PROMETHEUS_VERSION="v2.19.0"
+export PROMETHEUS_VERSION="v2.53.3"
 export PROMETHEUS_DOCKER=prom/prometheus
 export PROMETHEUS_IMAGE='${PROMETHEUS_DOCKER}:${PROMETHEUS_VERSION}'
 
diff --git a/birdhouse/components/monitoring/prometheus.yml.template b/birdhouse/components/monitoring/prometheus.yml.template
index f5d7cd09e..737331edd 100644
--- a/birdhouse/components/monitoring/prometheus.yml.template
+++ b/birdhouse/components/monitoring/prometheus.yml.template
@@ -20,6 +20,9 @@ scrape_configs:
   - targets:
     - ${BIRDHOUSE_FQDN}:9100
 
+scrape_config_files:
+  - "/etc/prometheus/scrape_config.d/*.yml"
+
 rule_files:
 - "/etc/prometheus/*.rules"
 
diff --git a/birdhouse/optional-components/prometheus-log-parser/config/monitoring/.gitignore b/birdhouse/optional-components/prometheus-log-parser/config/monitoring/.gitignore
new file mode 100644
index 000000000..add998b9b
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/config/monitoring/.gitignore
@@ -0,0 +1 @@
+scrape_configs.yml
diff --git a/birdhouse/optional-components/prometheus-log-parser/config/monitoring/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-log-parser/config/monitoring/docker-compose-extra.yml
new file mode 100644
index 000000000..c16e2f8ef
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/config/monitoring/docker-compose-extra.yml
@@ -0,0 +1,5 @@
+version: "3.4"
+services:
+  prometheus:
+    volumes:
+      - ./optional-components/prometheus-log-parser/config/monitoring/scrape_configs.yml:/etc/prometheus/scrape_config.d/log_parser.yml:ro
diff --git a/birdhouse/optional-components/prometheus-log-parser/config/monitoring/scrape_configs.yml.template b/birdhouse/optional-components/prometheus-log-parser/config/monitoring/scrape_configs.yml.template
new file mode 100644
index 000000000..025cadc67
--- /dev/null
+++ b/birdhouse/optional-components/prometheus-log-parser/config/monitoring/scrape_configs.yml.template
@@ -0,0 +1,6 @@
+scrape_configs:
+  - job_name: log_parser
+    honor_labels: true
+    static_configs:
+      - targets:
+        - prometheus-log-parser:${PROMETHEUS_LOG_PARSER_CLIENT_PORT}
diff --git a/birdhouse/optional-components/prometheus-log-parser/default.env b/birdhouse/optional-components/prometheus-log-parser/default.env
index ec7c0820d..35e268b44 100644
--- a/birdhouse/optional-components/prometheus-log-parser/default.env
+++ b/birdhouse/optional-components/prometheus-log-parser/default.env
@@ -6,6 +6,11 @@ export PROMETHEUS_LOG_PARSER_PARSERS_DIR=/parsers.d
 export PROMETHEUS_LOG_PARSER_POLL_DELAY=1 # time in seconds
 export PROMETHEUS_LOG_PARSER_TAIL=t
 
+OPTIONAL_VARS="
+  $OPTIONAL_VARS
+  \$PROMETHEUS_LOG_PARSER_CLIENT_PORT
+"
+
 export DELAYED_EVAL="
   $DELAYED_EVAL
   PROMETHEUS_LOG_PARSER_IMAGE

From 3f052e4c86206c6f555201e42886a320e11481f4 Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:40:26 -0500
Subject: [PATCH 06/11] update documentation and changes

---
 CHANGES.md                               | 27 +++++++++++++++++-
 birdhouse/env.local.example              |  4 +++
 birdhouse/optional-components/README.rst | 36 ++++++++++++++++++++++++
 3 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 2059da724..e0bc3cfc8 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,7 +15,32 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Add the `prometheus-log-parser` optional component
+
+  This component parses log files from other components and converts their logs to prometheus
+  metrics that are then ingested by the monitoring Prometheus instance (the one created by the
+  `components/monitoring` component).
+
+  For more information on how this component reads log files and converts them to prometheus components see
+  the [log-parser](https://github.com/DACCS-Climate/log-parser/) documentation.
+
+  To configure this component:
+
+  * set the `PROMETHEUS_LOG_PARSER_POLL_DELAY` variable to a number of seconds to set how often the log parser
+    checks if new lines have been added to log files (default: 1)
+  * set the `PROMETHEUS_LOG_PARSER_TAIL` variable to `"true"` to only parse new lines in log files. If unset,
+    this will parse all existing lines in the log file as well (default: `"true"`)
+
+  To view all metrics exported by the log parser:
+
+  * Navigate to the `https://<BIRDHOUSE_FQDN>/prometheus/graph` search page
+  * Put `{job="log_parser"}` in the search bar and click the "Execute" button
+
+- Update the prometheus version to the current latest `v2.53.3`. This is required to support   
+  loading multiple prometheus scrape configuration files with the `scrape_config_files`
+  configuration option.
 
 [2.5.4](https://github.com/bird-house/birdhouse-deploy/tree/2.5.4) (2024-10-31)
 ------------------------------------------------------------------------------------------------------------------
diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
index 8ca61f137..72bf5df30 100644
--- a/birdhouse/env.local.example
+++ b/birdhouse/env.local.example
@@ -632,6 +632,10 @@ export THREDDS_ADDITIONAL_CATALOG=''
 #export ALERTMANAGER_EXTRA_INHIBITION=""
 #export ALERTMANAGER_EXTRA_RECEIVERS=""
 
+# Below are for the prometheus-log-parser optional component
+#export PROMETHEUS_LOG_PARSER_POLL_DELAY=1 # time in seconds
+#export PROMETHEUS_LOG_PARSER_TAIL=t
+
 #############################################################################
 # Emu optional vars
 #############################################################################
diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst
index b1c0df792..3f6e238fd 100644
--- a/birdhouse/optional-components/README.rst
+++ b/birdhouse/optional-components/README.rst
@@ -443,3 +443,39 @@ How to enable X-Robots-Tag Header in ``env.local`` (a copy from `env.local.examp
 
     .. seealso::
         See the `env.local.example`_ file for more details about this ``BIRDHOUSE_PROXY_ROOT_LOCATION`` behaviour.
+
+.. _prometheus-log-parser
+
+Prometheus Log Parser
+---------------------
+
+Parses log files from other components and converts their logs to prometheus metrics that are then ingested by the
+monitoring Prometheus instance (the one created by the ``components/monitoring`` component).
+
+For more information on how this component reads log files and converts them to prometheus components see
+the log-parser_ documentation.
+
+To configure this component:
+
+    * set the ``PROMETHEUS_LOG_PARSER_POLL_DELAY`` variable to a number of seconds to set how often the log parser
+      checks if new lines have been added to log files (default: 1)
+    * set the ``PROMETHEUS_LOG_PARSER_TAIL`` variable to ``"true"`` to only parse new lines in log files. If unset,
+      this will parse all existing lines in the log file as well (default: ``"true"``)
+
+To view all metrics exported by the log parser:
+
+    * Navigate to the ``https://<BIRDHOUSE_FQDN>/prometheus/graph`` search page
+    * Put ``{job="log_parser"}`` in the search bar and click the "Execute" button
+
+For developers, to create a new parser that can be used to track log files:
+
+    1. create a python file that is can be mounted as a volume to the ``PROMETHEUS_LOG_PARSER_PARSERS_DIR``
+       directory on the ``prometheus-log-parser`` container.
+    2. mount any log files that you want to parse as a volume on the ``prometheus-log-parser`` container.
+    3. the python script must contain a variable named ``LOG_PARSER_CONFIG`` which is a
+       dictionary where keys are paths to log files and values are a list of "line parser" functions.
+       * a "line parser" is any function that takes a string as a single argument (a single line from a
+         log file). These functions are where you'd write the code that parses the line and converts it
+         into a prometheus metric.
+
+.. _log-parser: https://github.com/DACCS-Climate/log-parser/

From ffaf3d3150679e601952284da2002f49ba3995ea Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Fri, 8 Nov 2024 13:02:30 -0500
Subject: [PATCH 07/11] Apply suggestions from code review

Co-authored-by: David Huard <huard.david@ouranos.ca>
---
 birdhouse/optional-components/README.rst                      | 2 +-
 .../config/thredds/prometheus-log-exporter.py                 | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst
index 3f6e238fd..8dc8877d1 100644
--- a/birdhouse/optional-components/README.rst
+++ b/birdhouse/optional-components/README.rst
@@ -469,7 +469,7 @@ To view all metrics exported by the log parser:
 
 For developers, to create a new parser that can be used to track log files:
 
-    1. create a python file that is can be mounted as a volume to the ``PROMETHEUS_LOG_PARSER_PARSERS_DIR``
+    1. create a python file that can be mounted as a volume to the ``PROMETHEUS_LOG_PARSER_PARSERS_DIR``
        directory on the ``prometheus-log-parser`` container.
     2. mount any log files that you want to parse as a volume on the ``prometheus-log-parser`` container.
     3. the python script must contain a variable named ``LOG_PARSER_CONFIG`` which is a
diff --git a/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py b/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
index 3729ce3b2..8869ff183 100644
--- a/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
+++ b/birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py
@@ -26,7 +26,7 @@
 
 LABEL_KEYS = ("remote_addr", "date", "tds_service", "dataset", "variable")
 
-Counter = prometheus_client.Counter(
+counter = prometheus_client.Counter(
     name="thredds_transfer_size_kb",
     documentation="THREDDS data transferred",
     labelnames=LABEL_KEYS,
@@ -39,6 +39,6 @@ def parse_line(line):
         labels = {label: match.group(label) or "" for label in LABEL_KEYS}
         if body_byte_sent := match.group("body_byte_sent") is not None:
             body_byte_sent = int(body_byte_sent) / 1024
-            Counter.labels(**labels).inc(body_byte_sent)
+            counter.labels(**labels).inc(body_byte_sent)
 
 LOG_PARSER_CONFIG = {f"/var/log/proxy/{os.getenv('PROXY_LOG_FILE')}": [parse_line]}

From db2f825732996f9cb72ac44d06054475666149ae Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Fri, 8 Nov 2024 13:29:45 -0500
Subject: [PATCH 08/11] review suggestions

---
 birdhouse/components/README.rst                          | 1 +
 birdhouse/env.local.example                              | 2 +-
 birdhouse/optional-components/README.rst                 | 9 +++++++--
 .../prometheus-log-parser/default.env                    | 2 +-
 4 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst
index f67a9767e..c9c1ffebe 100644
--- a/birdhouse/components/README.rst
+++ b/birdhouse/components/README.rst
@@ -276,6 +276,7 @@ Here is a sample setup to test autodeploy:
   # then remove the running scheduler container
   birdhouse compose up -d --remove-orphans
 
+.. _Monitoring:
 
 Monitoring
 ==========
diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
index 72bf5df30..ee4beaf91 100644
--- a/birdhouse/env.local.example
+++ b/birdhouse/env.local.example
@@ -634,7 +634,7 @@ export THREDDS_ADDITIONAL_CATALOG=''
 
 # Below are for the prometheus-log-parser optional component
 #export PROMETHEUS_LOG_PARSER_POLL_DELAY=1 # time in seconds
-#export PROMETHEUS_LOG_PARSER_TAIL=t
+#export PROMETHEUS_LOG_PARSER_TAIL=true
 
 #############################################################################
 # Emu optional vars
diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst
index 8dc8877d1..19290977c 100644
--- a/birdhouse/optional-components/README.rst
+++ b/birdhouse/optional-components/README.rst
@@ -450,7 +450,7 @@ Prometheus Log Parser
 ---------------------
 
 Parses log files from other components and converts their logs to prometheus metrics that are then ingested by the
-monitoring Prometheus instance (the one created by the ``components/monitoring`` component).
+monitoring Prometheus instance (the one created by the :ref:`Monitoring` component).
 
 For more information on how this component reads log files and converts them to prometheus components see
 the log-parser_ documentation.
@@ -473,9 +473,14 @@ For developers, to create a new parser that can be used to track log files:
        directory on the ``prometheus-log-parser`` container.
     2. mount any log files that you want to parse as a volume on the ``prometheus-log-parser`` container.
     3. the python script must contain a variable named ``LOG_PARSER_CONFIG`` which is a
-       dictionary where keys are paths to log files and values are a list of "line parser" functions.
+       dictionary where keys are paths to log files (mounted in the container) and values are a list of
+       "line parser" functions.
        * a "line parser" is any function that takes a string as a single argument (a single line from a
          log file). These functions are where you'd write the code that parses the line and converts it
          into a prometheus metric.
 
+    For an example of a working log parser, see
+    `optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py`_
+    (:download:`download <birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py>`).
+
 .. _log-parser: https://github.com/DACCS-Climate/log-parser/
diff --git a/birdhouse/optional-components/prometheus-log-parser/default.env b/birdhouse/optional-components/prometheus-log-parser/default.env
index 35e268b44..9b33425e7 100644
--- a/birdhouse/optional-components/prometheus-log-parser/default.env
+++ b/birdhouse/optional-components/prometheus-log-parser/default.env
@@ -4,7 +4,7 @@ export PROMETHEUS_LOG_PARSER_IMAGE='${PROMETHEUS_LOG_PARSER_DOCKER}:${PROMETHEUS
 export PROMETHEUS_LOG_PARSER_CLIENT_PORT=8000
 export PROMETHEUS_LOG_PARSER_PARSERS_DIR=/parsers.d
 export PROMETHEUS_LOG_PARSER_POLL_DELAY=1 # time in seconds
-export PROMETHEUS_LOG_PARSER_TAIL=t
+export PROMETHEUS_LOG_PARSER_TAIL=true
 
 OPTIONAL_VARS="
   $OPTIONAL_VARS

From d332cf25a655652f167f1763246b0464622abc09 Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Fri, 8 Nov 2024 13:44:46 -0500
Subject: [PATCH 09/11] review suggestions [ci skip]

---
 birdhouse/optional-components/README.rst | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst
index 19290977c..2210bbe1b 100644
--- a/birdhouse/optional-components/README.rst
+++ b/birdhouse/optional-components/README.rst
@@ -472,15 +472,18 @@ For developers, to create a new parser that can be used to track log files:
     1. create a python file that can be mounted as a volume to the ``PROMETHEUS_LOG_PARSER_PARSERS_DIR``
        directory on the ``prometheus-log-parser`` container.
     2. mount any log files that you want to parse as a volume on the ``prometheus-log-parser`` container.
-    3. the python script must contain a variable named ``LOG_PARSER_CONFIG`` which is a
-       dictionary where keys are paths to log files (mounted in the container) and values are a list of
-       "line parser" functions.
+    3. the python script should create at least one `prometheus metric using the prometheus_client 
+       library <prometheus_python_metrics>`_ and must contain a global constant named ``LOG_PARSER_CONFIG`` 
+       which is a dictionary where keys are paths to log files (mounted in the container) and values are a 
+       list of "line parser" functions.
        * a "line parser" is any function that takes a string as a single argument (a single line from a
          log file). These functions are where you'd write the code that parses the line and converts it
          into a prometheus metric.
+       * your line parser function should update one of the prometheus metrics you created previously. 
 
     For an example of a working log parser, see
-    `optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py`_
+    `birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py`_
     (:download:`download <birdhouse/optional-components/prometheus-log-parser/config/thredds/prometheus-log-exporter.py>`).
 
 .. _log-parser: https://github.com/DACCS-Climate/log-parser/
+.. _prometheus_python_metrics: https://prometheus.github.io/client_python/instrumenting/

From 0d8678131cb75f76f34045d5abbbd49261128add Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Thu, 14 Nov 2024 09:33:44 -0500
Subject: [PATCH 10/11] review updates: set log level and extra comments

---
 birdhouse/components/proxy/nginx.conf.template                  | 2 ++
 birdhouse/optional-components/prometheus-log-parser/default.env | 1 +
 .../prometheus-log-parser/docker-compose-extra.yml              | 1 +
 3 files changed, 4 insertions(+)

diff --git a/birdhouse/components/proxy/nginx.conf.template b/birdhouse/components/proxy/nginx.conf.template
index 46479d035..a7acb6e8a 100755
--- a/birdhouse/components/proxy/nginx.conf.template
+++ b/birdhouse/components/proxy/nginx.conf.template
@@ -15,6 +15,8 @@ http {
     include       /etc/nginx/mime.types;
     default_type  application/octet-stream;
 
+    # NOTE: if this log_format ever changes, make sure to update the relevant code in the
+    #       prometheus-log-parser component as well to match the change.
     log_format  main  '$remote_addr - $remote_user [$time_iso8601] "$request" '
                       '$status $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for"';
diff --git a/birdhouse/optional-components/prometheus-log-parser/default.env b/birdhouse/optional-components/prometheus-log-parser/default.env
index 9b33425e7..ee6067976 100644
--- a/birdhouse/optional-components/prometheus-log-parser/default.env
+++ b/birdhouse/optional-components/prometheus-log-parser/default.env
@@ -5,6 +5,7 @@ export PROMETHEUS_LOG_PARSER_CLIENT_PORT=8000
 export PROMETHEUS_LOG_PARSER_PARSERS_DIR=/parsers.d
 export PROMETHEUS_LOG_PARSER_POLL_DELAY=1 # time in seconds
 export PROMETHEUS_LOG_PARSER_TAIL=true
+export PROMETHEUS_LOG_PARSER_LOG_LEVEL=INFO
 
 OPTIONAL_VARS="
   $OPTIONAL_VARS
diff --git a/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
index b08d321aa..c9baf62b7 100644
--- a/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
+++ b/birdhouse/optional-components/prometheus-log-parser/docker-compose-extra.yml
@@ -16,5 +16,6 @@ services:
       - LOG_PARSER_PARSERS=${PROMETHEUS_LOG_PARSER_PARSERS_DIR}
       - LOG_PARSER_POLL_DELAY=${PROMETHEUS_LOG_PARSER_POLL_DELAY}
       - LOG_PARSER_TAIL=${PROMETHEUS_LOG_PARSER_TAIL}
+      - LOG_PARSER_LOG_LEVEL=${PROMETHEUS_LOG_PARSER_LOG_LEVEL}
     restart: always
     logging: *default-logging

From bbef0daf2c259c0d861397b3c640ad374a34812c Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Tue, 19 Nov 2024 08:53:14 -0500
Subject: [PATCH 11/11] =?UTF-8?q?Bump=20version:=202.5.5=20=E2=86=92=202.6?=
 =?UTF-8?q?.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg                                          | 6 +++---
 CHANGES.md                                                | 5 +++++
 Makefile                                                  | 2 +-
 README.rst                                                | 8 ++++----
 RELEASE.txt                                               | 2 +-
 .../canarie-api/docker_configuration.py.template          | 8 ++++----
 docs/source/conf.py                                       | 4 ++--
 7 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index b9d85e110..2bac4e005 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.5.5
+current_version = 2.6.0
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2024-11-14T21:04:17Z
+search = {current_version} 2024-11-19T13:53:14Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2024-11-14T21:04:17Z
+values = 2024-11-19T13:53:14Z
 
 [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'
diff --git a/CHANGES.md b/CHANGES.md
index 97e049405..b20880015 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,6 +15,11 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
+[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+
+[2.6.0](https://github.com/bird-house/birdhouse-deploy/tree/2.6.0) (2024-11-19)
+------------------------------------------------------------------------------------------------------------------
+
 ## Changes
 
 - Add the `prometheus-log-parser` optional component
diff --git a/Makefile b/Makefile
index 085051c8d..178116bd2 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 2.5.5
+override APP_VERSION := 2.6.0
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")
diff --git a/README.rst b/README.rst
index e291e85dc..3d0a9963c 100644
--- a/README.rst
+++ b/README.rst
@@ -18,13 +18,13 @@ for a full-fledged production platform.
     * - citation
       - | |citation|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.5.5.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.6.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.5.5...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.6.0...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-2.5.5-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-2.6.0-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.5.5
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.6.0
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)
diff --git a/RELEASE.txt b/RELEASE.txt
index 26d8c76b4..9200ea1cd 100644
--- a/RELEASE.txt
+++ b/RELEASE.txt
@@ -1 +1 @@
-2.5.5 2024-11-14T21:04:17Z
+2.6.0 2024-11-19T13:53:14Z
diff --git a/birdhouse/components/canarie-api/docker_configuration.py.template b/birdhouse/components/canarie-api/docker_configuration.py.template
index c0a3248e6..fc82b7d34 100644
--- a/birdhouse/components/canarie-api/docker_configuration.py.template
+++ b/birdhouse/components/canarie-api/docker_configuration.py.template
@@ -108,8 +108,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.5.5',
-            'releaseTime': '2024-11-14T21:04:17Z',
+            'version': '2.6.0',
+            'releaseTime': '2024-11-19T13:53:14Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',
@@ -141,8 +141,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.5.5',
-            'releaseTime': '2024-11-14T21:04:17Z',
+            'version': '2.6.0',
+            'releaseTime': '2024-11-19T13:53:14Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',
diff --git a/docs/source/conf.py b/docs/source/conf.py
index b484a165f..a6e0fba86 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.5.5'
+version = '2.6.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.5.5'
+release = '2.6.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.