From 01b2776fdb18b135e93c2f36f19a94ad4912fdc1 Mon Sep 17 00:00:00 2001 From: Nando <78201305+fprats@users.noreply.github.com> Date: Thu, 11 Feb 2021 14:45:43 +0100 Subject: [PATCH] Fprats new golden metrics (#64) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adding goldenmetrics * Adding goldenmetrics for gcpstorabebucket * Adding goldenmetrics for gcpspannerinstance * golden metrics for gcphttploadbalancer * golden metrics for gcpcloudfunction * goldenmetrics definition & summary for gcpstoragebucket * definition for goldenmetrics for gcpspannerinstance * goldenmetrics definition and summary metrics for gcpinternalloadbalancer * fix golden_metrics file for gcpstoragebucket * goldenmetrics ยก& summaryMetric changes for gcpbigquerytable * goldenmetrics &summary metrics changes for gcpbigquerydataset * goldenmetrics added for gcpappengineservice * added sessions to goldenmetrics in gcpspannerinstance * fix goldenmetrics for gcphttploadbalancer * fix goldenmetrics for gcpappengineservice * added instances to summary_metrics of gcpappengineservice * fix definitions for gcpbigquerydataset * fix definition for gcpappengineservice * fix definition for gcpbigquerytable * fix definition for gcpcloudfunction * fix definition gcpappengineservice * fix definition gcpbigquerydataset * fix definition gcpbigquerytable * fix definition gcpcloudfunction * fix definition for gcpinternalloadbalancer * fix definition gcpspannerinstance * fix definition gcpstoragebucket * fix definition gcpstoragebucket * fix definition for gcpspannerinstance * fix definition gcpinternalloadbalancer * fix definition for gcphttploadbalancer * fix definition for gcpcloudsql * fix definition gcphttploadbalancer * fix definition gcpcloudfunction * fix definition gcpbigquerytable * fix definition gcpbigquerydataset * fix definition gcpappengineservice * fix definition gcpspannerdatabase * fix goldenmetrics gcpspannerinstance * fix defnition for gcpspannerdatabase Co-authored-by: fprats --- .../infra-gcpappengineservice/definition.yml | 5 ++++ .../golden_metrics.yml | 28 +++++++++++++++++++ .../summary_metrics.yml | 7 +++++ .../infra-gcpbigquerydataset/definition.yml | 6 ++++ .../golden_metrics.yml | 14 ++++++++++ .../summary_metrics.yml | 15 ++++++++++ .../infra-gcpbigquerytable/definition.yml | 5 ++++ .../infra-gcpbigquerytable/golden_metrics.yml | 14 ++++++++++ .../summary_metrics.yml | 14 ++++++++++ .../infra-gcpcloudfunction/definition.yml | 10 +++++-- .../infra-gcpcloudfunction/golden_metrics.yml | 21 ++++++++++++++ .../summary_metrics.yml | 8 ++++++ definitions/infra-gcpcloudsql/definition.yml | 4 +-- .../infra-gcphttploadbalancer/definition.yml | 4 +-- .../golden_metrics.yml | 10 +++---- .../definition.yml | 8 ++++++ .../golden_metrics.yml | 21 ++++++++++++++ .../summary_metrics.yml | 26 +++++++++++++++++ .../infra-gcpspannerdatabase/definition.yml | 3 ++ .../infra-gcpspannerinstance/definition.yml | 5 ++++ .../golden_metrics.yml | 28 +++++++++++++++++++ .../infra-gcpstoragebucket/definition.yml | 5 ++++ .../infra-gcpstoragebucket/golden_metrics.yml | 21 ++++++++++++++ .../summary_metrics.yml | 7 +++++ 24 files changed, 277 insertions(+), 12 deletions(-) create mode 100644 definitions/infra-gcpappengineservice/golden_metrics.yml create mode 100644 definitions/infra-gcpbigquerydataset/golden_metrics.yml create mode 100644 definitions/infra-gcpbigquerytable/golden_metrics.yml create mode 100644 definitions/infra-gcpcloudfunction/golden_metrics.yml create mode 100644 definitions/infra-gcpinternalloadbalancer/golden_metrics.yml create mode 100644 definitions/infra-gcpinternalloadbalancer/summary_metrics.yml create mode 100644 definitions/infra-gcpspannerinstance/golden_metrics.yml create mode 100644 definitions/infra-gcpstoragebucket/golden_metrics.yml diff --git a/definitions/infra-gcpappengineservice/definition.yml b/definitions/infra-gcpappengineservice/definition.yml index e8a255af8..cc622cd5d 100644 --- a/definitions/infra-gcpappengineservice/definition.yml +++ b/definitions/infra-gcpappengineservice/definition.yml @@ -1,5 +1,10 @@ domain: INFRA type: GCPAPPENGINESERVICE +goldenTags: +- gcp.projectId +- gcp.zone compositeMetrics: summaryMetrics: - summary_metrics.yml + goldenMetrics: + - golden_metrics.yml diff --git a/definitions/infra-gcpappengineservice/golden_metrics.yml b/definitions/infra-gcpappengineservice/golden_metrics.yml new file mode 100644 index 000000000..d5b8ae91d --- /dev/null +++ b/definitions/infra-gcpappengineservice/golden_metrics.yml @@ -0,0 +1,28 @@ +Instances: + title: Number of instances + query: + select: max(`system.Instances`) + from: GcpAppEngineServiceSample + facet: entityName + eventId: entityGuid +Responses: + title: Number of http responses + query: + select: sum(`server.Responses`) + from: GcpAppEngineServiceSample + facet: entityName + eventId: entityGuid +ErrorRate: + title: Error rate + query: + select: filter(sum(`server.Responses`), WHERE response_code NOT LIKE '2%') * 100 + from: GcpAppEngineServiceSample + facet: entityName + eventId: entityGuid +latency: + title: Latency + query: + select: (average(`server.ResponseLatenciesMilliseconds`)) / 1000 + from: GcpAppEngineServiceSample + facet: entityName + eventId: entityGuid diff --git a/definitions/infra-gcpappengineservice/summary_metrics.yml b/definitions/infra-gcpappengineservice/summary_metrics.yml index 32e5ad6d2..8200fbbfc 100644 --- a/definitions/infra-gcpappengineservice/summary_metrics.yml +++ b/definitions/infra-gcpappengineservice/summary_metrics.yml @@ -3,6 +3,13 @@ providerAccountName: key: providerAccountName title: GCP Account unit: STRING +Instances: + query: + eventId: entityGuid + select: max(`system.Instances`) + from: GcpAppEngineServiceSample + unit: COUNT + title: Number of instances Service responses: query: eventId: entityGuid diff --git a/definitions/infra-gcpbigquerydataset/definition.yml b/definitions/infra-gcpbigquerydataset/definition.yml index 0bc43aa92..c384e6102 100644 --- a/definitions/infra-gcpbigquerydataset/definition.yml +++ b/definitions/infra-gcpbigquerydataset/definition.yml @@ -1,5 +1,11 @@ domain: INFRA type: GCPBIGQUERYDATASET +goldenTags: +- gcp.projectId +- gcp.zone compositeMetrics: summaryMetrics: - summary_metrics.yml + goldenMetrics: + - golden_metrics.yml + diff --git a/definitions/infra-gcpbigquerydataset/golden_metrics.yml b/definitions/infra-gcpbigquerydataset/golden_metrics.yml new file mode 100644 index 000000000..2254f5b31 --- /dev/null +++ b/definitions/infra-gcpbigquerydataset/golden_metrics.yml @@ -0,0 +1,14 @@ +storedBytes: + title: Stored bytes + query: + select: sum(`storage.StoredBytes`) + from: GcpBigQueryDatasetSample + facet: entityName + eventId: entityGuid +tables: + title: Number of Tables + query: + select: sum(`storage.Tables`) + from: GcpBigQueryDatasetSample + facet: entityName + eventId: entityGuid diff --git a/definitions/infra-gcpbigquerydataset/summary_metrics.yml b/definitions/infra-gcpbigquerydataset/summary_metrics.yml index 6bdead924..5a650cf59 100644 --- a/definitions/infra-gcpbigquerydataset/summary_metrics.yml +++ b/definitions/infra-gcpbigquerydataset/summary_metrics.yml @@ -8,3 +8,18 @@ zone: tag: key: zone title: Zone +storedBytes: + query: + eventId: entityGuid + select: sum(`storage.StoredBytes`) + from: GcpBigQueryDatasetSample + unit: BYTES + title: storedBytes +tables: + query: + eventId: entityGuid + select: sum(`storage.Tables`) + from: GcpBigQueryDatasetSample + unit: COUNT + title: tables + diff --git a/definitions/infra-gcpbigquerytable/definition.yml b/definitions/infra-gcpbigquerytable/definition.yml index 6ff0f5bb9..e8e7f5afc 100644 --- a/definitions/infra-gcpbigquerytable/definition.yml +++ b/definitions/infra-gcpbigquerytable/definition.yml @@ -1,5 +1,10 @@ domain: INFRA type: GCPBIGQUERYTABLE +goldenTags: +- gcp.projectId +- gcp.zone compositeMetrics: summaryMetrics: - summary_metrics.yml + goldenMetrics: + - golden_metrics.yml diff --git a/definitions/infra-gcpbigquerytable/golden_metrics.yml b/definitions/infra-gcpbigquerytable/golden_metrics.yml new file mode 100644 index 000000000..c4f622ffb --- /dev/null +++ b/definitions/infra-gcpbigquerytable/golden_metrics.yml @@ -0,0 +1,14 @@ +uploadedBytes: + title: Uploaded bytes + query: + select: sum(`storage.UploadedBytes`) + from: GcpBigQueryTableSample + facet: entityName + eventId: entityGuid +uploadedRows: + title: Uploaded Rows + query: + select: sum(`storage.UploadedRowCount`) + from: GcpBigQueryTableSample + facet: entityName + eventId: entityGuid diff --git a/definitions/infra-gcpbigquerytable/summary_metrics.yml b/definitions/infra-gcpbigquerytable/summary_metrics.yml index 00bfd8b00..c9596731d 100644 --- a/definitions/infra-gcpbigquerytable/summary_metrics.yml +++ b/definitions/infra-gcpbigquerytable/summary_metrics.yml @@ -13,3 +13,17 @@ zone: tag: key: zone title: Zone +uploadedBytes: + query: + eventId: entityGuid + select: sum(`storage.UploadedBytes`) + from: GcpBigQueryTableSample + unit: BYTES + title: uploadedBytes +uploadedRows: + query: + eventId: entityGuid + select: sum(`storage.UploadedRowCount`) + from: GcpBigQueryTableSample + unit: COUNT + title: uploadedRows diff --git a/definitions/infra-gcpcloudfunction/definition.yml b/definitions/infra-gcpcloudfunction/definition.yml index de32ddd7c..fc4c40bed 100644 --- a/definitions/infra-gcpcloudfunction/definition.yml +++ b/definitions/infra-gcpcloudfunction/definition.yml @@ -1,6 +1,10 @@ domain: INFRA type: GCPCLOUDFUNCTION - +goldenTags: +- gcp.projectId +- gcp.zone compositeMetrics: - summaryMetrics: - - summary_metrics.yml \ No newline at end of file + summaryMetrics: + - summary_metrics.yml + goldenMetrics: + - golden_metrics.yml diff --git a/definitions/infra-gcpcloudfunction/golden_metrics.yml b/definitions/infra-gcpcloudfunction/golden_metrics.yml new file mode 100644 index 000000000..90667227b --- /dev/null +++ b/definitions/infra-gcpcloudfunction/golden_metrics.yml @@ -0,0 +1,21 @@ +Executions: + title: Number of function executions + query: + select: sum(`api.Requests`) + from: GcpCloudFunctionSample + facet: entityName + eventId: entityGuid +ErrorRate: + title: Error rate + query: + select: filter(sum(`function.Executions`), WHERE status != 'ok') * 100 / sum(`function.Executions`) + from: GcpCloudFunctionSample + facet: entityName + eventId: entityGuid +latency: + title: Latency + query: + select: (average(`function.ExecutionTimeNanos`)) / 1000000000 + from: GcpCloudFunctionSample + facet: entityName + eventId: entityGuid diff --git a/definitions/infra-gcpcloudfunction/summary_metrics.yml b/definitions/infra-gcpcloudfunction/summary_metrics.yml index 5e035e8b9..92755a45a 100644 --- a/definitions/infra-gcpcloudfunction/summary_metrics.yml +++ b/definitions/infra-gcpcloudfunction/summary_metrics.yml @@ -17,3 +17,11 @@ errorRate: from: GcpCloudFunctionSample unit: PERCENTAGE title: Error rate +latency: + query: + eventId: entityGuid + select: (average(`function.ExecutionTimeNanos`)) / 1000000000 + from: GcpCloudFunctionSample + unit: SECONDS + title: Latency + diff --git a/definitions/infra-gcpcloudsql/definition.yml b/definitions/infra-gcpcloudsql/definition.yml index c0fb0d1ac..92a7248d4 100644 --- a/definitions/infra-gcpcloudsql/definition.yml +++ b/definitions/infra-gcpcloudsql/definition.yml @@ -1,8 +1,8 @@ domain: INFRA type: GCPCLOUDSQL goldenTags: -- gcp.regionName -- gcp.subscriptionId +- gcp.projectId +- gcp.zone compositeMetrics: goldenMetrics: - golden_metrics.yml diff --git a/definitions/infra-gcphttploadbalancer/definition.yml b/definitions/infra-gcphttploadbalancer/definition.yml index 9872a7417..d08d341cf 100644 --- a/definitions/infra-gcphttploadbalancer/definition.yml +++ b/definitions/infra-gcphttploadbalancer/definition.yml @@ -1,8 +1,8 @@ domain: INFRA type: GCPHTTPLOADBALANCER goldenTags: -- gcp.regionName -- gcp.subscriptionId +- gcp.projectId +- gcp.zone compositeMetrics: goldenMetrics: - golden_metrics.yml diff --git a/definitions/infra-gcphttploadbalancer/golden_metrics.yml b/definitions/infra-gcphttploadbalancer/golden_metrics.yml index c852bd1f1..d3c3d7754 100644 --- a/definitions/infra-gcphttploadbalancer/golden_metrics.yml +++ b/definitions/infra-gcphttploadbalancer/golden_metrics.yml @@ -1,14 +1,14 @@ -requests: - title: Requests +requestsRate: + title: Requests per second query: - select: sum(https.Requests) + select: (rate(sum(`https.Requests`), 1 minute)) / 60 from: GcpHttpLoadBalancerSample facet: entityName eventId: entityGuid totalLatencyMs: - title: Total Latency (ms) + title: Total Latency query: - select: average(https.TotalLatencies) + select: (average(`https.TotalLatencies`)) / 1000 from: GcpHttpLoadBalancerSample facet: entityName eventId: entityGuid diff --git a/definitions/infra-gcpinternalloadbalancer/definition.yml b/definitions/infra-gcpinternalloadbalancer/definition.yml index 080319297..29f244100 100644 --- a/definitions/infra-gcpinternalloadbalancer/definition.yml +++ b/definitions/infra-gcpinternalloadbalancer/definition.yml @@ -1,2 +1,10 @@ domain: INFRA type: GCPINTERNALLOADBALANCER +goldenTags: + - gcp.projectId + - gcp.zone +compositeMetrics: + summaryMetrics: + - summary_metrics.yml + goldenMetrics: + - golden_metrics.yml diff --git a/definitions/infra-gcpinternalloadbalancer/golden_metrics.yml b/definitions/infra-gcpinternalloadbalancer/golden_metrics.yml new file mode 100644 index 000000000..20d21874e --- /dev/null +++ b/definitions/infra-gcpinternalloadbalancer/golden_metrics.yml @@ -0,0 +1,21 @@ +Sent: + title: Sent bytes + query: + select: sum(`l3.internal.EgressBytes`) + from: GcpInternalLoadBalancerSample + facet: entityName + eventId: entityGuid +Received: + title: Received bytes + query: + select: sum(`l3.internal.IngressBytes`) + from: GcpInternalLoadBalancerSample + facet: entityName + eventId: entityGuid +latency: + title: Latency + query: + select: average(`l3.internal.RttLatencies`) / 1000 + from: GcpInternalLoadBalancerSample + facet: entityName + eventId: entityGuid diff --git a/definitions/infra-gcpinternalloadbalancer/summary_metrics.yml b/definitions/infra-gcpinternalloadbalancer/summary_metrics.yml new file mode 100644 index 000000000..b2fd13774 --- /dev/null +++ b/definitions/infra-gcpinternalloadbalancer/summary_metrics.yml @@ -0,0 +1,26 @@ +providerAccountName: + tag: + key: providerAccountName + title: GCP Account + unit: STRING +sent: + query: + eventId: entityGuid + select: sum(`l3.internal.EgressBytes`) + from: GcpInternalLoadBalancerSample + unit: BYTES + title: Sent bytes +received: + query: + eventId: entityGuid + select: sum(`l3.internal.IngressBytes`) + from: GcpInternalLoadBalancerSample + unit: BYTES + title: Received bytes +latency: + query: + eventId: entityGuid + select: average(`l3.internal.RttLatencies`)/1000 + from: GcpInternalLoadBalancerSample + unit: SECONDS + title: Latency diff --git a/definitions/infra-gcpspannerdatabase/definition.yml b/definitions/infra-gcpspannerdatabase/definition.yml index c9ef35107..41c439723 100644 --- a/definitions/infra-gcpspannerdatabase/definition.yml +++ b/definitions/infra-gcpspannerdatabase/definition.yml @@ -1,5 +1,8 @@ domain: INFRA type: GCPSPANNERDATABASE +goldenTags: +- gcp.projectid +- gcp.zone compositeMetrics: summaryMetrics: - summary_metrics.yml diff --git a/definitions/infra-gcpspannerinstance/definition.yml b/definitions/infra-gcpspannerinstance/definition.yml index 8be337296..7811435bb 100644 --- a/definitions/infra-gcpspannerinstance/definition.yml +++ b/definitions/infra-gcpspannerinstance/definition.yml @@ -1,5 +1,10 @@ domain: INFRA type: GCPSPANNERINSTANCE +goldenTags: + - gcp.projectId + - gcp.zone compositeMetrics: summaryMetrics: - summary_metrics.yml + goldenMetrics: + - golden_metrics.yml diff --git a/definitions/infra-gcpspannerinstance/golden_metrics.yml b/definitions/infra-gcpspannerinstance/golden_metrics.yml new file mode 100644 index 000000000..00ef2885c --- /dev/null +++ b/definitions/infra-gcpspannerinstance/golden_metrics.yml @@ -0,0 +1,28 @@ +Nodes: + title: Number of nodes + query: + select: latest(`instance.nodes`) + from: GcpSpannerInstanceSample + facet: entityName + eventId: entityGuid +cpuUtilization: + title: Average cpu utilization + query: + select: average(`instance.cpu.Utilization`) + from: GcpSpannerInstanceSample + facet: entityName + eventId: entityGuid +diskUsage: + title: Storage used in bytes + query: + select: average(`instance.storage.UsedBytes`) + from: GcpSpannerInstanceSample + facet: entityName + eventId: entityGuid +sessions: + title: Sessions in use + query: + select: latest(`instance.sessions`) + from: GcpSpannerInstanceSample + facet: entityName + eventId: entityGuide diff --git a/definitions/infra-gcpstoragebucket/definition.yml b/definitions/infra-gcpstoragebucket/definition.yml index 1453d3c19..9ab2ec7aa 100644 --- a/definitions/infra-gcpstoragebucket/definition.yml +++ b/definitions/infra-gcpstoragebucket/definition.yml @@ -1,5 +1,10 @@ domain: INFRA type: GCPSTORAGEBUCKET +goldenTags: + - gcp.projectId + - gcp.zone compositeMetrics: summaryMetrics: - summary_metrics.yml + goldenMetrics: + - golden_metrics.yml diff --git a/definitions/infra-gcpstoragebucket/golden_metrics.yml b/definitions/infra-gcpstoragebucket/golden_metrics.yml new file mode 100644 index 000000000..43f087398 --- /dev/null +++ b/definitions/infra-gcpstoragebucket/golden_metrics.yml @@ -0,0 +1,21 @@ +receivedBytes: + title: Received bytes + query: + select: sum(`network.ReceivedBytes`) + from: GcpStorageBucketSample + facet: entityName + eventId: entityGuid +sentBytes: + title: Sent bytes + query: + select: sum(`network.SentBytes`) + from: GcpStorageBucketSample + facet: entityName + eventId: entityGuid +apiRequests: + title: Number API calls + query: + select: sum(`api.Requests`) + from: GcpStorageBucketSample + facet: entityName + eventId: entityGuid diff --git a/definitions/infra-gcpstoragebucket/summary_metrics.yml b/definitions/infra-gcpstoragebucket/summary_metrics.yml index 2f0b826cc..c5ae50d48 100644 --- a/definitions/infra-gcpstoragebucket/summary_metrics.yml +++ b/definitions/infra-gcpstoragebucket/summary_metrics.yml @@ -17,3 +17,10 @@ sent: from: GcpStorageBucketSample unit: BYTES title: Sent +apicalls: + query: + eventId: entityGuid + select: sum(`api.Requests`) + from: GcpStorageBucketSample + unit: COUNT + title: apiCalls