From 782e30e1ecbd70eb2df674e65cd15aa025da8c08 Mon Sep 17 00:00:00 2001 From: Scotsoo Date: Thu, 19 Aug 2021 10:18:29 +0100 Subject: [PATCH 1/3] adding netacea-v2 generator --- log_generator/generators/__init__.py | 3 ++- log_generator/generators/netaceaV2.py | 29 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 log_generator/generators/netaceaV2.py diff --git a/log_generator/generators/__init__.py b/log_generator/generators/__init__.py index 4575cef..e1876ee 100644 --- a/log_generator/generators/__init__.py +++ b/log_generator/generators/__init__.py @@ -6,5 +6,6 @@ from generators.apache import Apache from generators.cloudflare import Cloudflare from generators.cloudfront import Cloudfront +from generators.netaceaV2 import NetaceaV2 -__all__ = [Apache, Cloudfront, Cloudflare] +__all__ = [Apache, Cloudfront, Cloudflare, NetaceaV2] diff --git a/log_generator/generators/netaceaV2.py b/log_generator/generators/netaceaV2.py new file mode 100644 index 0000000..72d5bb8 --- /dev/null +++ b/log_generator/generators/netaceaV2.py @@ -0,0 +1,29 @@ +from generators.base import LogRender +import json + + +class NetaceaV2(LogRender): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.timestamp_format = "%FT%T%z" + + def generate(self, d: dict) -> str: + request = f"""{d.method} {d.uri_path}{d.query_params} ${d.http_protocol}""" + log_line: dict = { + "status": str(d.http_status), + "referrer": d.referer, + "request": request, + "request_time": d.request_time, + "integration_type": "lumbererType", + "integration_version": "lumbererVersion", + "client": d.ip_address, + "user_agent": d.user_agent, + "hour": d.timestamp.tm_hour, + "minute": d.timestamp.tm_min, + "@timestamp": d.timestamp.strftime(self.timestamp_format), + "path": d.uri_path, + "protocol": d.http_protocol, + "query": d.query_params, + "user_id": d.uuid, + } + return json.dumps(log_line) From 3b5eb509443bb4114daded5c2bdb632a8c123e7b Mon Sep 17 00:00:00 2001 From: Scotsoo Date: Thu, 19 Aug 2021 10:35:33 +0100 Subject: [PATCH 2/3] fix timetuple --- log_generator/generators/netaceaV2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/log_generator/generators/netaceaV2.py b/log_generator/generators/netaceaV2.py index 72d5bb8..36366aa 100644 --- a/log_generator/generators/netaceaV2.py +++ b/log_generator/generators/netaceaV2.py @@ -9,6 +9,7 @@ def __init__(self, **kwargs): def generate(self, d: dict) -> str: request = f"""{d.method} {d.uri_path}{d.query_params} ${d.http_protocol}""" + time_tuple = d.timestamp.timetuple() log_line: dict = { "status": str(d.http_status), "referrer": d.referer, @@ -18,8 +19,8 @@ def generate(self, d: dict) -> str: "integration_version": "lumbererVersion", "client": d.ip_address, "user_agent": d.user_agent, - "hour": d.timestamp.tm_hour, - "minute": d.timestamp.tm_min, + "hour": time_tuple.tm_hour, + "minute": time_tuple.tm_min, "@timestamp": d.timestamp.strftime(self.timestamp_format), "path": d.uri_path, "protocol": d.http_protocol, From 14f80416e7868fc15e9ddd1da6c68e0fd97c9c9d Mon Sep 17 00:00:00 2001 From: Scotsoo Date: Thu, 19 Aug 2021 13:24:37 +0100 Subject: [PATCH 3/3] batch the logs, as that's how they come through --- log_generator/generators/netaceaV2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/log_generator/generators/netaceaV2.py b/log_generator/generators/netaceaV2.py index 36366aa..51902c2 100644 --- a/log_generator/generators/netaceaV2.py +++ b/log_generator/generators/netaceaV2.py @@ -27,4 +27,5 @@ def generate(self, d: dict) -> str: "query": d.query_params, "user_id": d.uuid, } - return json.dumps(log_line) + # v2 log lines come through as batches + return json.dumps([log_line])