Merge pull request #2 from logicmonitor/feature/resource_mapping

feature/resource_mapping: Modified: Changing the resource mapping to…
logicmonitor · May 5, 2021 · f51b8f0 · f51b8f0
2 parents 02fc130 + 42ace5b
commit f51b8f0
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 237 deletions.
diff --git a/fluent-plugin-lm-logs-gcp.gemspec b/fluent-plugin-lm-logs-gcp.gemspec
@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |spec|
   spec.name                           = "fluent-plugin-lm-logs-gcp"
-  spec.version                        = '1.0.0'
+  spec.version                        = '1.0.1'
   spec.authors                        = ["LogicMonitor"]
   spec.email                          = "[email protected]"
   spec.summary                        = "LogicMonitor with GCP logs fluentd filter plugin"

diff --git a/lib/fluent/plugin/filter_gcplm.rb b/lib/fluent/plugin/filter_gcplm.rb
@@ -27,6 +27,7 @@ def filter(tag, time, record)
       resourceMap = Hash.new
       project_id = record.dig("resource","labels", "project_id")
       region = record.dig("resource","labels", "region")
+      filteredRecord = Hash.new
 
       case
       when record['textPayload']
@@ -42,23 +43,24 @@ def filter(tag, time, record)
       case resourceType
       when 'gce_instance'
         if (record.dig("resource","labels", "instance_id"))
-            resourceMap = {"system.gcp.resourceid" => record.dig("resource","labels", "instance_id")}
+            resourceMap = {"system.gcp.resourceid" => record.dig("resource","labels", "instance_id"), "system.cloud.category" => 'GCP/ComputeEngine'}
         elsif (record.dig("labels", "compute.googleapis.com/resource_name"))
-            resourceMap = {"system.gcp.resourcename" => record.dig("labels", "compute.googleapis.com/resource_name")}
+            resourceMap = {"system.gcp.resourcename" => record.dig("labels", "compute.googleapis.com/resource_name"), "system.cloud.category" => 'GCP/ComputeEngine'}
         end
       when 'cloud_function'
-        resourceMap = {"system.gcp.resourcename" => "projects/" + project_id + "/locations/" + region +"/functions/" + record.dig("resource","labels", "function_name")}
+        resourceMap = {"system.gcp.resourcename" => "projects/" + project_id + "/locations/" + region +"/functions/" + record.dig("resource","labels", "function_name"), "system.cloud.category" => 'GCP/CloudFunction'}
       when 'cloudsql_database'
-        resourceMap = {"system.gcp.resourceid" => record.dig("resource","labels", "database_id")}
+        resourceMap = {"system.gcp.resourceid" => record.dig("resource","labels", "database_id"), "system.cloud.category" => 'GCP/CloudSQL'}
       end
 
       if(record.key?("protoPayload") && record.dig('protoPayload','@type') == "type.googleapis.com/google.cloud.audit.AuditLog")
         resourceMap = {"system.gcp.projectId" => project_id, "system.cloud.category" => 'GCP/LMAccount'}
       end
 
-      record['message'] = message
-      record['_lm.resourceId'] = resourceMap
-      record
+      filteredRecord['message'] = message
+      filteredRecord['_lm.resourceId'] = resourceMap
+      filteredRecord['timestamp'] = record['timestamp']
+      filteredRecord
     end
   end
 end
diff --git a/test/plugin/test_filter_gcplm_auditlogs.rb b/test/plugin/test_filter_gcplm_auditlogs.rb
@@ -104,75 +104,12 @@ def filter(messages, conf = '')
                         "system.cloud.category" => 'GCP/LMAccount'
                         },
                     "message" =>"{\"@type\": \"type.googleapis.com/google.cloud.audit.AuditLog\",\"status\": {\"message\": \"Instance Group Manager 'projects/272452262635/zones/us-central1-c/instanceGroupManagers/aef-backend-20180822t164014-00' initiated recreateInstance on instance 'projects/272452262635/zones/us-central1-c/instances/aef-backend-20180822t164014-b75j'. Reason: instance's current action is VERIFYING, but health status is TIMEOUT.\"},\"authenticationInfo\": {\"principalEmail\": \"[email protected]\"},\"serviceName\": \"compute.googleapis.com\",\"methodName\": \"compute.instances.repair.recreateInstance\",\"resourceName\": \"projects/development-198123/zones/us-central1-c/instances/aef-backend-20180822t164014-b75j\",\"request\": {\"@type\": \"type.googleapis.com/compute.instances.repair.recreateInstance\"}}",
-                    "insertId" => "bdl1yydsszy",
-                    "resource" => {
-                        "type" => "gce_instance",
-                        "labels" => {
-                            "zone" => "us-central1-c",
-                            "project_id" => "development-198123",
-                            "instance_id" => "5229353360185505344"
-                        }
-                    },
-                    "timestamp" => "2021-04-27T06:44:51.535974Z",
-                    "severity" => "INFO",
-                    "logName" => "projects/development-198123/logs/cloudaudit.googleapis.com%2Fsystem_event",
-                    "operation" => {
-                        "id" => "repair-1619505891524-5c0ee99603e9a-862341d5-24923c3c",
-                        "producer" => "compute.instances.repair.recreateInstance",
-                        "first" => true,
-                        "last" => true
-                        },
-                    "receiveTimestamp" => "2021-04-27T06:44:51.904218698Z",
-                   "protoPayload" => {
-                     "@type" => "type.googleapis.com/google.cloud.audit.AuditLog",
-                     "status" => {
-                       "message" => "Instance Group Manager 'projects/272452262635/zones/us-central1-c/instanceGroupManagers/aef-backend-20180822t164014-00' initiated recreateInstance on instance 'projects/272452262635/zones/us-central1-c/instances/aef-backend-20180822t164014-b75j'. Reason: instance's current action is VERIFYING, but health status is TIMEOUT."
-                       },
-                     "authenticationInfo" => {
-                       "principalEmail" => "[email protected]"
-                       },
-                     "serviceName" => "compute.googleapis.com",
-                     "methodName" => "compute.instances.repair.recreateInstance",
-                     "resourceName" => "projects/development-198123/zones/us-central1-c/instances/aef-backend-20180822t164014-b75j",
-                     "request" => {
-                       "@type" => "type.googleapis.com/compute.instances.repair.recreateInstance"
-                       }
-                     }
+                    "timestamp" => "2021-04-27T06:44:51.535974Z"
                     },
                     {
                     "_lm.resourceId" => {},
                     "message" =>"{\"@type\": \"type.googleapis.com/google.cloud.audit.AuditLog\",\"status\": {\"message\": \"Instance Group Manager 'projects/272452262635/zones/us-central1-c/instanceGroupManagers/aef-backend-20180822t164014-00' initiated recreateInstance on instance 'projects/272452262635/zones/us-central1-c/instances/aef-backend-20180822t164014-b75j'. Reason: instance's current action is VERIFYING, but health status is TIMEOUT.\"},\"authenticationInfo\": {\"principalEmail\": \"[email protected]\"},\"serviceName\": \"compute.googleapis.com\",\"methodName\": \"compute.instances.repair.recreateInstance\",\"resourceName\": \"projects/development-198123/zones/us-central1-c/instances/aef-backend-20180822t164014-b75j\",\"request\": {\"@type\": \"type.googleapis.com/compute.instances.repair.recreateInstance\"}}",
-                    "resource" => {
-                        "type" => "gce_instance",
-                        "labels" => {
-                            "project_id" => "development-198123"
-                            }
-                        },
-                   "protoPayload" => {
-                     "status" => {
-                       "message" => "Instance Group Manager 'projects/272452262635/zones/us-central1-c/instanceGroupManagers/aef-backend-20180822t164014-00' initiated recreateInstance on instance 'projects/272452262635/zones/us-central1-c/instances/aef-backend-20180822t164014-b75j'. Reason: instance's current action is VERIFYING, but health status is TIMEOUT."
-                       },
-                     "authenticationInfo" => {
-                       "principalEmail" => "[email protected]"
-                       },
-                     "serviceName" => "compute.googleapis.com",
-                     "methodName" => "compute.instances.repair.recreateInstance",
-                     "resourceName" => "projects/development-198123/zones/us-central1-c/instances/aef-backend-20180822t164014-b75j",
-                     "request" => {
-                       "@type" => "type.googleapis.com/compute.instances.repair.recreateInstance"
-                       }
-                     },
-                   "insertId" => "bdl1yydsszy",
-                   "timestamp" => "2021-04-27T06:44:51.535974Z",
-                   "severity" => "INFO",
-                   "logName" => "projects/development-198123/logs/cloudaudit.googleapis.com%2Fsystem_event",
-                   "operation" => {
-                    "id" => "repair-1619505891524-5c0ee99603e9a-862341d5-24923c3c",
-                    "producer" => "compute.instances.repair.recreateInstance",
-                    "first" => true,
-                    "last" => true
-                     },
-                   "receiveTimestamp" => "2021-04-27T06:44:51.904218698Z"
+                    "timestamp" => "2021-04-27T06:44:51.535974Z"
                     }]
                 actual = filter(messages)
                 actual.each_with_index { |val, index| assert_equal(expected[index]["_lm.resourceId"], actual[index]["_lm.resourceId"]) }

diff --git a/test/plugin/test_filter_gcplm_cf.rb b/test/plugin/test_filter_gcplm_cf.rb
@@ -50,27 +50,11 @@ def filter(messages, conf = '')
 
                 expected = [{
                     "_lm.resourceId" => {
-                        "system.gcp.resourcename" =>"projects/logicmonitor.com:api-project-650342240768/locations/asia-south1/functions/testfunctionlogsToPubsub"
+                        "system.gcp.resourcename" =>"projects/logicmonitor.com:api-project-650342240768/locations/asia-south1/functions/testfunctionlogsToPubsub",
+                        "system.cloud.category" => 'GCP/CloudFunction'
                         },
                     "message" =>"Function execution took 200 ms, finished with status: 'ok'",
-                    "insertId" => "000000-d8f6e03b-271c-4c7d-9780-4c63d37ccda7",
-                    "resource" => {
-                      "type" => "cloud_function",
-                      "labels" => {
-                        "function_name" => "testfunctionlogsToPubsub",
-                        "region" => "asia-south1",
-                        "project_id" => "logicmonitor.com:api-project-650342240768"
-                        }
-                      },
-                    "timestamp" => "2021-04-15T06:14:10.997438123Z",
-                    "severity" => "DEBUG",
-                    "labels" => {
-                      "execution_id" => "d4mdjbva6jz2"
-                      },
-                    "logName" => "projects/logicmonitor.com:api-project-650342240768/logs/cloudfunctions.googleapis.com%2Fcloud-functions",
-                    "trace" => "projects/logicmonitor.com:api-project-650342240768/traces/72e2cdf9e92a630b981f61e9e498f2e0",
-                    "receiveTimestamp" => "2021-04-08T09:59:25.195011779Z",
-                    "textPayload" => "Function execution took 200 ms, finished with status: 'ok'",
+                    "timestamp" => "2021-04-15T06:14:10.997438123Z"
                 }]
                 actual = filter(messages)
                 actual.each_with_index { |val, index| assert_equal(expected[index]["_lm.resourceId"], actual[index]["_lm.resourceId"]) }

diff --git a/test/plugin/test_filter_gcplm_cloudsql.rb b/test/plugin/test_filter_gcplm_cloudsql.rb
@@ -46,24 +46,12 @@ def filter(messages, conf = '')
 
                 expected = [{
                     "_lm.resourceId" => {
-                        "system.gcp.resourceid" =>"development-198123:rwaver-sql-1"
+                        "system.gcp.resourceid" =>"development-198123:rwaver-sql-1",
+                        "system.cloud.category" => 'GCP/CloudSQL'
                         },
                     "message" =>"2021-04-27T14:34:16.352086Z 0 [ERROR] Could not successfully notify external app of semisync status change before timeout.",
-                    "insertId" => "s=0f48625de67a4c96a1b506ffc3851ca5;i=1e47;b=1d5c153c19964ece83df6787dd571159;m=74f53eb;t=5c0f528216aa5;x=be395e5383cf012f-0-0@a1",
-                    "resource" => {
-                      "type" => "cloudsql_database",
-                      "labels" => {
-                        "region" => "asia",
-                        "database_id" => "development-198123:rwaver-sql-1",
-                        "project_id" => "development-198123"
-                      }
-                    },
-                    "timestamp" => "2021-04-27T14:34:16.352421Z",
-                    "severity" => "ERROR",
-                    "logName" => "projects/development-198123/logs/cloudsql.googleapis.com%2Fmysql.err",
-                    "receiveTimestamp" => "2021-04-27T14:34:17.079988744Z",
-                    "textPayload" => "2021-04-27T14:34:16.352086Z 0 [ERROR] Could not successfully notify external app of semisync status change before timeout.",
-                    }]
+                    "timestamp" => "2021-04-27T14:34:16.352421Z"
+                      }]
                 actual = filter(messages)
                 actual.each_with_index { |val, index| assert_equal(expected[index]["_lm.resourceId"], actual[index]["_lm.resourceId"]) }
         end