fix(lm-logs): fix timed out message from concat plugin causing data l…

…oss (#131) Use labes to reuse the route for common processing on logs
logicmonitor · Feb 6, 2024 · 16ad0f7 · 16ad0f7
1 parent e3ce836
commit 16ad0f7
Show file tree

Hide file tree

Showing 6 changed files with 100 additions and 56 deletions.
diff --git a/charts/lm-logs/Chart.yaml b/charts/lm-logs/Chart.yaml
@@ -1,10 +1,10 @@
 apiVersion: v2
 description: A Helm chart for sending k8s logs to Logic Monitor
 name: lm-logs
-icon: https://logicmonitor.github.io/helm-charts/lm_logo.png
-version: 0.4.0
+version: 0.5.1
 maintainers:
   - email: [email protected]
     name: LogicMonitor
-appVersion: 1.0.5
+icon: https://logicmonitor.github.io/helm-charts/lm_logo.png
+appVersion: 1.1.0
 home: https://logicmonitor.github.io/helm-charts
diff --git a/charts/lm-logs/Dockerfile b/charts/lm-logs/Dockerfile
@@ -1,4 +1,4 @@
 FROM fluent/fluentd-kubernetes-daemonset:v1.16-debian-forward-1
 USER root
-RUN gem install fluent-plugin-lm-logs -v 1.0.5
+RUN gem install fluent-plugin-lm-logs -v 1.1.0
 RUN gem install fluent-plugin-multi-format-parser -v 1.0.0
diff --git a/charts/lm-logs/README.md b/charts/lm-logs/README.md
@@ -40,7 +40,9 @@ The following tables lists the configurable parameters of the lm-logs chart and
 | `affinity`                  | Affinity for pod assignment		                | `{}`  (evaluated as a template)                         |
 | `env`                       | Map to add extra environment variables	        | `{}`                                                    |
 | `kubernetes.multiline_start_regexp` | Regexp to match beginning of multiline	| `/^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/` |
-| `kubernetes.cluster_name`       | ClusterName given while adding k8s cluster  	| `""`                                                    |
+| `kubernetes.cluster_name`       | ClusterName given while adding k8s cluster  	| `""`                                                |
+| `kubernetes.multiline_concat_key`       | Key to look for fluentD to concatenate multiline logs   	| `"log"`                     |
+
 
 ### Avaialble Environment variables
 For descriptions see: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
@@ -76,3 +78,19 @@ Anomaly detection will be done on `namespace` and `service`
 #### Multiline log support for k8s lm logs
 To use regexp to match beginning of multiline set `kubernetes.multiline_start_regexp=<some-regex-pattern>`
 by default the regex is set to `/^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/`
+
+### Logs appearing in cri format
+If conatiner runtime is containerD or cri-o, on lm-logs ui you might see logs with prefix eg.
+```
+2016-10-06T00:17:09.669794202Z stdout F The content of the log entry 1
+```
+To solve this we need to install lm-logs with following command :
+```
+helm upgrade --install -n <namespace> \
+--set lm_company_name="<comapny>" \
+--set lm_access_id="<access_id>" \
+--set lm_access_key="<access_key"> \
+--set env.FLUENT_CONTAINER_TAIL_PARSER_TYPE="cri" \
+--set kubernetes.multiline_concat_key="message" \
+lm-logs logicmonitor/lm-logs
+```
diff --git a/charts/lm-logs/templates/configmap.yaml b/charts/lm-logs/templates/configmap.yaml
@@ -14,40 +14,61 @@ data:
       log_level "#{ENV['FLUENT_LOG_LEVEL'] || 'warn'}"
     </system>
 
-    <filter kubernetes.**>
-      @type record_transformer
-      enable_ruby
-      <record>
-        message ${record["log"]} ${record["message"]}
-        timestamp ${record["time"]}
-        {{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
-        {{ include "kubernetes.cluster_name" . | nindent 8 }}
-        {{- end}}
-        {{- if  .Values.fluent.device_less_logs }}
-        resource.service.name ${record.dig("kubernetes","labels","app") != nil ? record.dig("kubernetes","labels","app") : record.dig("kubernetes","labels","app_kubernetes_io/name") != nil ? record.dig("kubernetes","labels","app_kubernetes_io/name") :  record.dig("kubernetes","container_name") != nil ?  record.dig("kubernetes","container_name") : record.dig("kubernetes","pod_name") != nil ? record.dig("kubernetes","pod_name") :  "unknown" }
-        resource.service.namespace ${record["kubernetes"]["namespace_name"]}
+    <label @PROCESS_AFTER_CONCAT>
+      <filter kubernetes.**>
+        @type kubernetes_metadata
+        @id filter_kube_metadata
+        kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+        verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+        ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+        skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+        skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+        skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+        skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
+      </filter>
+
+      <filter kubernetes.**>
+        @type record_transformer
+        enable_ruby
+        <record>
+          message ${record["log"]} ${record["message"]}
+          timestamp ${record["time"]}
+          {{- if or .Values.kubernetes.cluster_name .Values.global.clusterName }}
+          {{ include "kubernetes.cluster_name" . | nindent 8 }}
+          {{- end}}
+          {{- if  .Values.fluent.device_less_logs }}
+          resource.service.name ${record.dig("kubernetes","labels","app") != nil ? record.dig("kubernetes","labels","app") : record.dig("kubernetes","labels","app_kubernetes_io/name") != nil ? record.dig("kubernetes","labels","app_kubernetes_io/name") :  record.dig("kubernetes","container_name") != nil ?  record.dig("kubernetes","container_name") : record.dig("kubernetes","pod_name") != nil ? record.dig("kubernetes","pod_name") :  "unknown" }
+          resource.service.namespace ${record["kubernetes"]["namespace_name"]}
+          {{- end}}
+        </record>
+        remove_keys log
+      </filter>
+
+      <match kubernetes.**>
+        @type lm
+        company_name {{ if .Values.lm_company_name }} {{ .Values.lm_company_name }} {{ else }} {{ required "A valid .Values.lm_company_name or .Values.global.account entry is required!" .Values.global.account }} {{ end }}
+        resource_mapping {"kubernetes.pod_name": "auto.name"}
+        {{- if  and ( or .Values.lm_access_id .Values.global.accessID ) ( or .Values.lm_access_key .Values.global.accessKey) }}
+        access_id {{ .Values.lm_access_id | default .Values.global.accessID }}
+        access_key {{ .Values.lm_access_key | default .Values.global.accessKey }}
+        {{- else if .Values.lm_bearer_token }}
+        bearer_token {{ .Values.lm_bearer_token }}
+        {{- else }} {{ required "Either specify valid lm_access_id and lm_access_key both or lm_bearer_token for authentication with LogicMonitor." .Values.lm_bearer_token }}
         {{- end}}
-      </record>
-      remove_keys log
-    </filter>
+        debug false
+        compression gzip
+        include_metadata {{ hasKey .Values.fluent "include_metadata" | ternary .Values.fluent.include_metadata true }}
+        device_less_logs {{ .Values.fluent.device_less_logs | default false }}
+        <buffer>
+          @type memory
+          flush_interval {{ .Values.fluent.buffer.memory.flush_interval | default "1s" }}
+          chunk_limit_size {{ .Values.fluent.buffer.memory.chunk_limit_size | default "8m" }}
+          flush_thread_count {{ .Values.fluent.buffer.memory.flush_thread_count | default "8"}}
+        </buffer>
+      </match>
+    </label>
+
 
-    <match kubernetes.**>
-      @type lm
-      company_name {{ if .Values.lm_company_name }} {{ .Values.lm_company_name }} {{ else }} {{ required "A valid .Values.lm_company_name or .Values.global.account entry is required!" .Values.global.account }} {{ end }}
-      resource_mapping {"kubernetes.pod_name": "auto.name"}
-      access_id {{ .Values.lm_access_id | default .Values.global.accessID }}
-      access_key {{ .Values.lm_access_key | default .Values.global.accessKey }}
-      debug false
-      compression gzip
-      include_metadata {{ hasKey .Values.fluent "include_metadata" | ternary .Values.fluent.include_metadata true }}
-      device_less_logs {{ .Values.fluent.device_less_logs | default false }}
-      <buffer>
-        @type memory
-        flush_interval {{ .Values.fluent.buffer.memory.flush_interval | default "1s" }}
-        chunk_limit_size {{ .Values.fluent.buffer.memory.chunk_limit_size | default "8m" }}
-        flush_thread_count {{ .Values.fluent.buffer.memory.flush_thread_count | default "8"}}
-      </buffer>
-    </match>
   kubernetes.conf: |
     <source>
       @type tail
@@ -74,26 +95,15 @@ data:
 
     <filter kubernetes.**>
       @type concat
-      key log
+      key {{ .Values.kubernetes.multiline_concat_key }}
       seperator ""
       multiline_start_regexp {{ .Values.kubernetes.multiline_start_regexp }}
-      timeout_label @NORMAL
+      timeout_label @PROCESS_AFTER_CONCAT
     </filter>
 
-    <label @NORMAL>
-      <match kubernetes.**>
-        @type stdout
-      </match>
-    </label>
+    <match kubernetes.**>
+      @type relabel
+      @label @PROCESS_AFTER_CONCAT
+    </match>
+
 
-    <filter kubernetes.**>
-      @type kubernetes_metadata
-      @id filter_kube_metadata
-      kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
-      verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
-      ca_file "#{ENV['KUBERNETES_CA_FILE']}"
-      skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
-      skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
-      skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
-      skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
-    </filter>
diff --git a/charts/lm-logs/values.schema.json b/charts/lm-logs/values.schema.json
@@ -73,6 +73,17 @@
       ],
       "$comment": "ui:lm_access_key-ignore tf:optional"
     },
+    "lm_bearer_token": {
+      "$id": "#/properties/lm_bearer_token",
+      "type": "string",
+      "title": "Logicmonitor API Bearer Token",
+      "description": "The LogicMonitor API Bearer Token.\nNOTE: Ensure to add surrounding double quotes to avoid special character parsing errors.",
+      "default": "",
+      "examples": [
+        ""
+      ],
+      "$comment": "ui:lm_bearer_token-ignore tf:optional"
+    },
     "lm_company_name": {
       "$id": "#/properties/lm_company_name",
       "type": "string",
@@ -302,6 +313,9 @@
         },
         "cluster_name" : {
           "type": "string"
+        },
+        "multiline_concat_key" : {
+          "type": "string"
         }
       }
     },

diff --git a/charts/lm-logs/values.yaml b/charts/lm-logs/values.yaml
@@ -1,11 +1,12 @@
 image:
   repository: logicmonitor/lm-logs-k8s-fluentd
   pullPolicy: Always
-  tag: "1.0.5"
+  tag: "1.1.0"
 
 lm_access_id: ""
 lm_access_key: ""
 lm_company_name: ""
+lm_bearer_token: ""
 
 global:
   accessID: ""
@@ -38,6 +39,7 @@ fluent:
 
 kubernetes:
   multiline_start_regexp: /^\[(\d{4}-)?\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}.*\]/
+  multiline_concat_key: log
 
 nodeSelector: {}
 affinity: {}