v1.8.4 (#79)

# Version History ## v1.8.4 | | | |--------|---------------------| | Date | 2025-01-07 | | Kind | MINOR release | | Author | [email protected] | - **Minor improvements** - [docker] bumped Python container version to 3.12.8 - [docker] bumped EAA container version to RC0.6.11 - **Bugfixes** - Bugfix for JSON Log Escaping (massive thx to @sethumadhav07 for the provded PR)
akamai · Jan 7, 2025 · 1122750 · 1122750
1 parent f283ff0
commit 1122750
Show file tree

Hide file tree

Showing 14 changed files with 176 additions and 27 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,8 @@
-FROM            python:3.12.4-slim-bookworm
+FROM            python:3.12.8-slim-bookworm
 LABEL           MAINTAINER="Mike Schiessl - [email protected]"
-LABEL	        APP_LONG="Akamai Universal Log Streamer"
+LABEL	        APP_LONG="Akamai Unified Log Streamer"
 LABEL           APP_SHORT="ULS"
-LABEL           VENDOR="Akamai Technologies"
+LABEL           VENDOR="Akamai Technologies Inc"
 
 
 # CONFIGURATION ARGS
@@ -11,7 +11,7 @@ ARG             ULS_DIR="$HOMEDIR/uls"
 ARG             EXT_DIR="$ULS_DIR/ext"
 
 ARG             ETP_CLI_VERSION="0.4.8"
-ARG             EAA_CLI_VERSION="0.6.10"
+ARG             EAA_CLI_VERSION="rc0.6.11"
 ARG             MFA_CLI_VERSION="0.1.1"
 ARG             GC_CLI_VERSION="v0.0.6"
 ARG             LINODE_CLI_VERSION="dev"

diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py
@@ -449,27 +449,37 @@ def send_data(self, data):
                             single_event_data = f"{single_event_data}{self.http_out_format % json.dumps(logline)}"
                         request = requests.Request('POST', url=self.http_url, data=(single_event_data))
 
-                    prepped = self.httpSession.prepare_request(request)
-                    payload_length = prepped.headers["Content-Length"]
-
+                    # Send the HTTP request
                     response = None
                     try:
+                        aka_log.log.debug(f"{self.name} Sending HTTP Request")
+                        prepped = self.httpSession.prepare_request(request)
+                        payload_length = prepped.headers["Content-Length"]
                         response = self.httpSession.send(prepped, verify=self.http_verify_tls, timeout=self.http_timeout)
-                    except Exception as bluu:
-                        print(f"bluu {bluu}")
-                        return False
+                    except Exception as error:
+                        aka_log.log.critical(f"{self.name} HTTP POST of {len(self.aggregateList)} event(s) went wrong. Error: {error}")
+
                     finally:
                         if response:
-
                             response.close()  # Free up the underlying TCP connection in the connection pool
 
-                    aka_log.log.info(f"{self.name} HTTP POST of {len(self.aggregateList)} event(s) "
+                    if not response:
+                        aka_log.log.warning(
+                            f"{self.name} HTTP POST of {len(self.aggregateList)} was NOT successful. Statuscode: we have not even received a response - see above logs for more details")
+                        return False
+
+                    elif response.status_code == uls_config.output_http_expected_status_code:
+                        aka_log.log.info(f"{self.name} HTTP POST of {len(self.aggregateList)} event(s) "
                                      f"completed in {(response.elapsed.total_seconds()*1000):.3f} ms, "
                                      f"payload={payload_length} bytes, HTTP response {response.status_code}, "
                                      f"response={response.text} ")
-                    if response.status_code != uls_config.output_http_expected_status_code:
+                        self.aggregateList.clear()
+                        return True
+
+                    elif response.status_code != uls_config.output_http_expected_status_code:
+                        aka_log.log.warning(f"{self.name} HTTP POST of {len(self.aggregateList)} was NOT successful. Statuscode: {response.status_code}, Error: {response.text}")
                         return False
-                    self.aggregateList.clear()
+
                 else:
                     aka_log.log.info(f"{self.name} Data not sent, but added to HTTP aggregation. Size: "
                                      f"({len(self.aggregateList)}/{self.http_out_aggregate_count})")
@@ -490,7 +500,7 @@ def send_data(self, data):
                 aka_log.log.critical(f"{self.name} target was not defined {self.output_type} ")
                 sys.exit(1)
 
-            aka_log.log.debug(f"{self.name} Data successfully sent via {self.output_type}")
+            aka_log.log.info(f"{self.name} Data successfully sent via {self.output_type}")
             return True
 
         except Exception as my_error:

diff --git a/bin/uls.py b/bin/uls.py
@@ -193,7 +193,7 @@ def main():
         try:
             input_data = event_q.get(block=True, timeout=0.05)
             if uls_args.debugloglines:
-                escaped_data = input_data.decode('utf-8').replace('"', '\\"')
+                escaped_data = input_data.rstrip().decode('utf-8').replace('"', '\\"')
                 aka_log.log.debug(f"<IN> {escaped_data}")
             for log_line in input_data.splitlines():
 

diff --git a/bin/uls_config/global_config.py b/bin/uls_config/global_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import sys
 # Common global variables / constants
-__version__ = "1.8.3"
+__version__ = "1.8.4"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -1,4 +1,20 @@
 # Version History
+## v1.8.4
+|        |                     |
+|--------|---------------------|
+| Date   | 2025-01-07          |
+| Kind   | MINOR release       |
+| Author | [email protected] |
+
+- **Minor improvements**
+  - [docker] bumped Python container version to 3.12.8
+  - [docker] bumped EAA container version to RC0.6.11
+
+- **Bugfixes**
+  - Bugfix for  JSON Log Escaping (massive thx to @sethumadhav07 for the provded PR)
+
+
+---
 
 ## v1.8.3
 |        |                     |
@@ -11,8 +27,14 @@
   - [docker] bumped CLI-GC version to 0.0.6 
 
 - **Bugfixes**
-  - Improved JSON Log Escaping (massive thx to @sethumadhav07 for the privded PR)
+  - Improved JSON Log Escaping (massive thx to @sethumadhav07 for the provded PR)
+
+- **Docfixes**
+  - Introduced "var" directory mount for docker & docker compose usage (allows autoresume within docker)
 
+- **Housekeeping**
+  - improved python version testing (sampling py3.9 to 3.13)
+  - 
 ---
 
 ## v1.8.2

diff --git a/docs/DOCKER_USAGE.md b/docs/DOCKER_USAGE.md
@@ -59,6 +59,7 @@ Below are two examples with our Enterprise Threat Protector product:
     ```bash 
     docker run -d --name uls_etp-threat -ti \
         --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly \
+        --mount type=bind,source="/path/to/your/uls-var/",target="/opt/akamai-uls/uls/var",readonly \
         akamai/uls \
         --input etp --feed threat --output tcp --host 10.10.10.10 --port 9091
     ```
@@ -67,6 +68,7 @@ Below are two examples with our Enterprise Threat Protector product:
     ```bash 
     docker run -d --name uls_etp-threat -ti \
         --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly \
+        --mount type=bind,source="/path/to/your/uls-var/",target="/opt/akamai-uls/uls/var",readonly \
         --env ULS_INPUT=ETP \
         --env ULS_FEED=THREAT \
         --env ULS_OUTPUT=TCP \

diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md
@@ -26,21 +26,22 @@ Here are some examples (per product) and links to additional information.
   - [Guardicore](#guardicore)
     - [NETLOG](#netlog)
     - [INCIDENT](#incident)
-    - [AUDIT](#gc-audit)
+    - [GC Audit](#gc-audit)
   - [Linode](#linode)
     - [AUDIT Logs](#audit-logs)
-  - [AKAMAI ACC](#akamai-acc)
-    - [Events](#events)
+  - [Akamai ACC](#akamai-acc)
+    - [EVENTS](#events)
 
 ---
 ## Enterprise Application Access (EAA)
 When configuring ULS to access EAA these feed, set `input` argument/variable to `EAA` and `feed` as indicated below in parathesis.
 
 ### Access Logs (ACCESS)
 
-Additional information regarding the log fields can be found on [here](https://techdocs.akamai.com/eaa/docs/data-feed-siem#access-logs)
+Latest information regarding the log fields can be found on [here](https://techdocs.akamai.com/eaa/docs/data-feed-siem#access-logs).
+
 <details>
-    <summary>EAA access logs example (JSON)</summary>
+    <summary>User access log example (JSON)</summary>
 
 ```json
 {

diff --git a/docs/examples/docker-compose/etp-tenants/docker-compose.yml b/docs/examples/docker-compose/etp-tenants/docker-compose.yml
@@ -9,6 +9,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir1/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
   etp-threat-tenant-2:
     image: akamai/uls:latest
     restart: always
@@ -17,4 +21,8 @@ services:
       - type: bind
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
-        read_only: true
+        read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir2/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
diff --git a/docs/examples/docker-compose/examples/all_services_docker-compose.yml b/docs/examples/docker-compose/examples/all_services_docker-compose.yml
@@ -10,6 +10,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
   # AUP
   etp-aup:
     image: akamai/uls:latest
@@ -20,6 +24,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
   # DNS
   etp-aup:
     image: akamai/uls:latest
@@ -30,6 +38,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
   # PROXY
   etp-aup:
     image: akamai/uls:latest
@@ -40,6 +52,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
 # EAA
   # ACCESS
   eaa-access:
@@ -51,6 +67,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
   # ADMIN
   eaa-admin:
     image: akamai/uls:latest
@@ -61,6 +81,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
 # MFA
   # AUTH
   mfa-auth:
@@ -72,6 +96,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
   # POLICY
   mfa-policy:
     image: akamai/uls:latest
@@ -81,4 +109,8 @@ services:
       - type: bind
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
-        read_only: true
+        read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
diff --git a/docs/examples/docker-compose/simple/docker-compose.yml b/docs/examples/docker-compose/simple/docker-compose.yml
@@ -8,4 +8,8 @@ services:
       - type: bind
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
-        read_only: true
+        read_only: true
+      - type: bind
+        source: /path/to/your/uls-var-dir/
+        target: /opt/akamai-uls/uls/var/
+        read_only: false
diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
@@ -4,4 +4,4 @@ description: Akamai Universal Log Streamer Helm installation
 
 type: application
 version: 2.0.0
-appVersion: "1.8.3"
+appVersion: "1.8.4"
diff --git a/test/basic_test.bats b/test/basic_test.bats
@@ -73,6 +73,12 @@ current_version=$(cat docs/CHANGELOG.md | grep "##" | head -n 1 | sed 's/.* v//'
 	[ "$status" -eq 0 ]
 }
 
+@test "python3.13 uls.py --version" {
+	run /opt/homebrew/bin/python3.13 $uls_bin --version
+	assert_output --partial "Akamai Unified Log Streamer Version information"
+	[ "$status" -eq 0 ]
+}
+
 @test "cat bin/config/global - Version output should be ($current_version) according to CHANGELOG" {
 	run echo $(cat bin/uls_config/global_config.py | grep "__version__ =" | cut -d " " -f 3)
 	assert_output --partial "$current_version"

diff --git a/test/opt/README.md b/test/opt/README.md
@@ -0,0 +1,8 @@
+# Additional Testing tools
+
+## webserver.py
+```text
+webserver.py <port>
+```
+
+Runs a local webserver on the given <port> where ULS can deliver data against
diff --git a/test/opt/webserver.py b/test/opt/webserver.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""
+License: MIT License
+Copyright (c) 2023 Miel Donkers, amended 2024 by Mike Schiessl (to suite our ULS needs)
+Very simple HTTP server in python for logging requests
+Usage::
+    ./server.py [<port>]
+"""
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import logging
+
+class S(BaseHTTPRequestHandler):
+    def _set_response(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    def do_OPTIONS(self):
+        logging.info("OPTIONS request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers))
+        self._set_response()
+        self.wfile.write("GET request for {}".format(self.path).encode('utf-8'))
+
+    def do_GET(self):
+        logging.info("GET request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers))
+        self._set_response()
+        self.wfile.write("GET request for {}".format(self.path).encode('utf-8'))
+
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
+        post_data = self.rfile.read(content_length) # <--- Gets the data itself
+        logging.info("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n",
+                str(self.path), str(self.headers), post_data.decode('utf-8'))
+
+        self._set_response()
+        self.wfile.write("POST request for {}".format(self.path).encode('utf-8'))
+
+def run(server_class=HTTPServer, handler_class=S, port=8080):
+    logging.basicConfig(level=logging.INFO)
+    server_address = ('', port)
+    httpd = server_class(server_address, handler_class)
+    logging.info('Starting httpd...\n')
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    httpd.server_close()
+    logging.info('Stopping httpd...\n')
+
+if __name__ == '__main__':
+    from sys import argv
+
+    if len(argv) == 2:
+        run(port=int(argv[1]))
+    else:
+        run()
+