v1.8.5 (#83)

akamai · Mar 4, 2025 · 1745fe3 · 1745fe3
1 parent 1122750
commit 1745fe3
Show file tree

Hide file tree

Showing 11 changed files with 76 additions and 19 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM            python:3.12.8-slim-bookworm
+FROM            python:3.13.2-slim-bookworm
 LABEL           MAINTAINER="Mike Schiessl - [email protected]"
 LABEL	        APP_LONG="Akamai Unified Log Streamer"
 LABEL           APP_SHORT="ULS"
@@ -14,7 +14,7 @@ ARG             ETP_CLI_VERSION="0.4.8"
 ARG             EAA_CLI_VERSION="rc0.6.11"
 ARG             MFA_CLI_VERSION="0.1.1"
 ARG             GC_CLI_VERSION="v0.0.6"
-ARG             LINODE_CLI_VERSION="dev"
+ARG             LINODE_CLI_VERSION="v0.0.4"
 ARG             ACC_CLI_VERSION="v0.0.1-alpha"
 
 # ENV VARS

diff --git a/README.md b/README.md
@@ -150,3 +150,9 @@ This software is released under the "Apache License". Please read the [frequentl
 
 [The ULS Project on GitHub](https://github.com/akamai/uls)  
 [The ULS Project on Dockerhub](https://hub.docker.com/r/akamai/uls)
+
+## Stargazers over time
+A small graph to show how many stars we collected over the time of this project.
+Feel free to add your star as well.  
+[![Stargazers over time](https://starchart.cc/akamai/uls.svg?variant=adaptive)](https://starchart.cc/akamai/uls)
+
diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py
@@ -312,11 +312,17 @@ def proc_create(self):
                     aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
                                         f"not available for {product_path}, continuing with JSON.")
                 if not self.rawcmd:
+
                     self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
                                               self.credentials_file_section,
                                               uls_config.edgerc_linode)
                     my_feed = self._feed_selector(self.feed, product_feeds)
-                    cli_command = [self.bin_python, '-u', product_path, 'events', my_feed.lower(), '-f']
+
+                    if my_feed == "AUDIT":
+                        cli_command = [self.bin_python, '-u', product_path, 'events', my_feed.lower(), '-f']
+                    elif my_feed == "UTILIZATION":
+                        cli_command = [self.bin_python, '-u', product_path, 'utilization', '-f']
+
                     cli_command[3:3] = self._uls_useragent(self.product, my_feed)
                     cli_command[3:3] = edgegrid_auth
                     cli_command[3:3] = self._prep_proxy(self.inproxy)

diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py
@@ -463,6 +463,7 @@ def send_data(self, data):
                         if response:
                             response.close()  # Free up the underlying TCP connection in the connection pool
 
+                    response_text_escaped = response.text.replace('"', '\\"')
                     if not response:
                         aka_log.log.warning(
                             f"{self.name} HTTP POST of {len(self.aggregateList)} was NOT successful. Statuscode: we have not even received a response - see above logs for more details")
@@ -472,12 +473,13 @@ def send_data(self, data):
                         aka_log.log.info(f"{self.name} HTTP POST of {len(self.aggregateList)} event(s) "
                                      f"completed in {(response.elapsed.total_seconds()*1000):.3f} ms, "
                                      f"payload={payload_length} bytes, HTTP response {response.status_code}, "
-                                     f"response={response.text} ")
+                                     f"response={response_text_escaped} ")
                         self.aggregateList.clear()
-                        return True
+                        # Removed the return here - Issue #82
+                        # return True
 
                     elif response.status_code != uls_config.output_http_expected_status_code:
-                        aka_log.log.warning(f"{self.name} HTTP POST of {len(self.aggregateList)} was NOT successful. Statuscode: {response.status_code}, Error: {response.text}")
+                        aka_log.log.warning(f"{self.name} HTTP POST of {len(self.aggregateList)} was NOT successful. Statuscode: {response.status_code}, Error: {response_text_escaped}")
                         return False
 
                 else:

diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py
@@ -228,8 +228,10 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin
                         aka_log.log.debug(f"Autoresume Checkpoint successfully loaded. Checkpoint Time: {data['checkpoint']}, Creation_time: {data['creation_time']}")
                         creation_time = data['creation_time']
                         # Convert the Checkpoint to "epoch Timestamp", depending on the input
+                        is_unixtimestamp=False
                         if data['input'] == "ETP" or data['input'] == "SIA":
                             mytime = data['checkpoint'].split("Z")[0]
+                            is_unixtimestamp=True
                         elif data['input'] == "EAA":
                             mytime = data['checkpoint'].split("+")[0]
                         elif data['input'] == "GC"  or data['input'] == "ACC":
@@ -238,14 +240,22 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin
                             aka_log.log.critical(
                                 f"Unhandeled input data in checkpointfile  \'{checkpoint_full}\' --> {input} / {feed} - Exiting.")
                             sys.exit(1)
-                        my_timestamp = datetime.datetime(year=int(mytime.split("T")[0].split("-")[0]),
-                                                         month=int(mytime.split("T")[0].split("-")[1]),
-                                                         day=int(mytime.split("T")[0].split("-")[2]),
-                                                         hour=int(mytime.split("T")[1].split(":")[0]),
-                                                         minute=int(mytime.split("T")[1].split(":")[1]),
-                                                         second=int(mytime.split("T")[1].split(":")[2]),
-                                                         )
-                        checkpoint = int(my_timestamp.replace(tzinfo=datetime.timezone.utc).timestamp())
+
+                        # Render unixTimeStamp
+                        if not is_unixtimestamp:
+                            my_timestamp = datetime.datetime(year=int(mytime.split("T")[0].split("-")[0]),
+                                                             month=int(mytime.split("T")[0].split("-")[1]),
+                                                             day=int(mytime.split("T")[0].split("-")[2]),
+                                                             hour=int(mytime.split("T")[1].split(":")[0]),
+                                                             minute=int(mytime.split("T")[1].split(":")[1]),
+                                                             second=int(mytime.split("T")[1].split(":")[2]),
+                                                             )
+                            print(f"my_timestamp {my_timestamp}")
+                            checkpoint = int(my_timestamp.replace(tzinfo=datetime.timezone.utc).timestamp())
+                        elif is_unixtimestamp:
+                            my_timestamp = mytime
+                            checkpoint = int(mytime)
+
 
                         aka_log.log.debug(f"Checkpoint timestamp {data['checkpoint']} converted to epoch time {checkpoint}")
                     else:

diff --git a/bin/uls_config/global_config.py b/bin/uls_config/global_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import sys
 # Common global variables / constants
-__version__ = "1.8.4"
+__version__ = "1.8.5"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -33,7 +33,7 @@
 
     # LINODE
 bin_linode_cli = "ext/cli-linode/bin/akamai-linode"         # Path to the LINODE CLI Executable
-linode_cli_feeds = ['AUDIT']                        # Available LINODE CLI feeds
+linode_cli_feeds = ['AUDIT', 'UTILIZATION']                        # Available LINODE CLI feeds
 
     # ACC-LOGS
 bin_acc_logs = "ext/acc-logs/bin/akamai-acc"

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -1,4 +1,29 @@
 # Version History
+## v1.8.5
+|        |                     |
+|--------|---------------------|
+| Date   | 2025-03-04          |
+| Kind   | BUGFIX release      |
+| Author | [email protected] |
+
+- **Features**
+  - New ULS feed for Linode: UTILIZATION
+    ```json
+    {"time": "2025-02-06T15:23:54.048512+00:00", "account": "Akamai Technologies - DEMO", "linode": 17, "lke_cluster": 5, "vpc": 9, "vlan": 21, "cloud_firewall": 41, "node_balancer": 15, "object_storage": 46, "volume": 127}
+    ```
+    This feed helps to understand and track the Linode account usage
+
+- **Minor improvements**
+  - [docker] bumped Python container version to 3.13.2
+
+
+- **Bugfixes**
+  - fixed a bug caused by newer ETP/SIA CLI when using the autoresume function (This is prone to fail again as we were tackling an API issue) 
+  - manually merged changes from PR#78  - issue #81 - something went wrong when merging the original PR - sorry for the inconvenience @vasantbala
+  - Fixed a bug where the successful sending of HTTP was not properly logged - issue #82 - thanks again @vasantbala
+
+---
+
 ## v1.8.4
 |        |                     |
 |--------|---------------------|

diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md
@@ -2710,6 +2710,15 @@ Additional information regarding the log fields can be found on [here](https://w
 ```
 </details>
 
+<details>
+    <summary>Linode Utilization example (JSON)</summary>
+
+```json
+{"time": "2025-02-06T15:23:54.048512+00:00", "account": "Akamai Technologies - DEMO", "linode": 17, "lke_cluster": 5, "vpc": 9, "vlan": 21, "cloud_firewall": 41, "node_balancer": 15, "object_storage": 46, "volume": 127}
+
+```
+</details>
+
 
 ---
 ## Akamai ACC

diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
@@ -4,4 +4,4 @@ description: Akamai Universal Log Streamer Helm installation
 
 type: application
 version: 2.0.0
-appVersion: "1.8.4"
+appVersion: "1.8.5"
diff --git a/misc/install_uls.sh b/misc/install_uls.sh
diff --git a/test/positive_test.bats b/test/positive_test.bats
@@ -10,7 +10,7 @@ uls_bin=bin/uls.py
 mocked_edgerc=FALSE
 
 # TIMEOUT
-  # How much time is timeout alklowed to run
+  # How much time is timeout allowed to run
   uls_test_timeout=90
   # Send a kill signal after
   uls_kill_timeout=110