From 437bcb1e0b514959648eed36ba3963aa4fbeffc8 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 29 Aug 2024 11:45:25 -0700 Subject: [PATCH] feat(storage): introduce gRPC client-side metrics (#10639) * feat: initial prototype testing client-side metrics * added tests and refactored to help test * use 1.21 and refactor test * fix go.sum go.work.sum * add option to disable client metrics * wip * add endpoint and project detection * revert unnecessary changes * update documentation * remove unnecessary code and update docs * address vet workflow feedback * update missed name changes * revert version downgrades * add exporterLogSuppressor; remove sdkmetric alias; log when metric failed to init * fix typo after rename * remove internalMetricsConfig, remove use of internal, and clean up docs/comments * remove smoke test * const metricPrefix * doc edit * add additional comments and remove metric descriptions; fix failing test * create ConnPool to access endpoint before GAPIC is initialized * double up on ConnPool :( * go work sync * clean up DialPool creation for metrics * add todo to remove workaround * revert line removals * make sure we emit RPC name and remove initializing second connpool * remove endpoint magic * edit docs --- go.work.sum | 16 +- storage/client.go | 2 + storage/doc.go | 33 +++++ storage/go.mod | 11 +- storage/go.sum | 20 +++ storage/grpc_client.go | 13 ++ storage/grpc_metrics.go | 275 +++++++++++++++++++++++++++++++++++ storage/grpc_metrics_test.go | 152 +++++++++++++++++++ storage/option.go | 34 ++++- storage/option_test.go | 34 +++-- 10 files changed, 566 insertions(+), 24 deletions(-) create mode 100644 storage/grpc_metrics.go create mode 100644 storage/grpc_metrics_test.go diff --git a/go.work.sum b/go.work.sum index fa19f95ec70a..bc606f61d47a 100644 --- a/go.work.sum +++ b/go.work.sum @@ -4,6 +4,7 @@ cloud.google.com/go/dataproc v1.12.0 h1:W47qHL3W4BPkAIbk4SWmIERwsWBaNnWm0P2sdx3Y cloud.google.com/go/gaming v1.9.0 h1:7vEhFnZmd931Mo7sZ6pJy7uQPDxF7m7v8xtBheG08tc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.18.0 h1:ugYJK/neZQtQeh2jc5xNoDFiMQojlAkoqJMRb7vTu1U= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.18.0/go.mod h1:Xx0VKh7GJ4si3rmElbh19Mejxz68ibWg/J30ZOMrqzU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.23.0/go.mod h1:p2puVVSKjQ84Qb1gzw2XHLs34WQyHTYFZLaVxypAFYs= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= @@ -26,13 +27,11 @@ github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0V github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= -github.com/envoyproxy/go-control-plane v0.12.1-0.20240621013728-1eb8caab5155/go.mod h1:5Wkq+JduFtdAXihLmeTJf+tRYIT4KBc2vPXDhwVo1pA= github.com/fullstorydev/grpcurl v1.8.7/go.mod h1:pVtM4qe3CMoLaIzYS8uvTuDj2jVYmXqMUkZeijnXp/E= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA= -github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/hoisie/redis v0.0.0-20160730154456-b5c6e81454e0/go.mod h1:pMYMxVaKJqCDC1JUg/XbPJ4/fSazB25zORpFzqsIGIc= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -45,19 +44,18 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lyft/protoc-gen-star/v2 v2.0.4-0.20230330145011-496ad1ac90a4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/miekg/dns v1.1.33/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= -github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +go.opentelemetry.io/contrib/detectors/gcp v1.27.0/go.mod h1:amd+4uZxqJAUx7zI1JvygUtAc2EVWtQeyz8D+3161SQ= go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA= go.opentelemetry.io/otel/bridge/opencensus v0.40.0 h1:pqDiayRhBgoqy1vwnscik+TizcImJ58l053NScJyZso= go.opentelemetry.io/otel/bridge/opencensus v0.40.0/go.mod h1:1NvVHb6tLTe5A9qCYz+eErW0t8iPn4ZfR6tDKcqlGTM= @@ -65,26 +63,24 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0/go.mod h1:qcTO4xHAxZLaLxPd60TdE88rxtItPHgHWqOhOGRr0as= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.44.0/go.mod h1:sTt30Evb7hJB/gEk27qLb1+l9n4Tb8HvHkR0Wx3S6CU= go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= +go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= google.golang.org/api v0.174.0/go.mod h1:aC7tB6j0HR1Nl0ni5ghpx6iLasmAX78Zkh/wgxAAjLg= google.golang.org/genproto v0.0.0-20230725213213-b022f6e96895/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= +google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= google.golang.org/genproto/googleapis/api v0.0.0-20230725213213-b022f6e96895/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/api v0.0.0-20240823204242-4ba0660f739c/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g= google.golang.org/genproto/googleapis/bytestream v0.0.0-20231120223509-83a465c0220f/go.mod h1:iIgEblxoG4klcXsG0d9cpoxJ4xndv6+1FkDROCHhPRI= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240102182953-50ed04b92917/go.mod h1:O9TvT7A9NLgdqqF0JJXJ+axpaoYiEb8txGmkvy+AvLc= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240513163218-0867130af1f8/go.mod h1:RCpt0+3mpEDPldc32vXBM8ADXlFL95T8Chxx0nv0/zE= google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/storage/client.go b/storage/client.go index 4bb10c9c4983..aebba2251757 100644 --- a/storage/client.go +++ b/storage/client.go @@ -132,6 +132,8 @@ type settings struct { // userProject is the user project that should be billed for the request. userProject string + + metricsContext *metricsContext } func initSettings(opts ...storageOption) *settings { diff --git a/storage/doc.go b/storage/doc.go index 48c98b0e4796..d9cde1a46ec1 100644 --- a/storage/doc.go +++ b/storage/doc.go @@ -360,6 +360,33 @@ applications depending on this package. If you are not using gRPC, you can use the build tag `disable_grpc_modules` to opt out of these dependencies and reduce the binary size. +The gRPC client emits metrics by default and will export the +gRPC telemetry discussed in [gRFC/66] and [gRFC/78] to +[Google Cloud Monitoring]. The metrics are accessible through Cloud Monitoring +API and you incur no additional cost for publishing the metrics. Google Cloud +Support can use this information to more quickly diagnose problems related to +GCS and gRPC. +Sending this data does not incur any billing charges, and requires minimal +CPU (a single RPC every minute) or memory (a few KiB to batch the +telemetry). + +To access the metrics you can view them through Cloud Monitoring +[metric explorer] with the prefix `storage.googleapis.com/client`. Metrics are emitted +every minute. + +You can disable metrics using the following example when creating a new gRPC +client using [WithDisabledClientMetrics]. + +The metrics exporter uses Cloud Monitoring API which determines +project ID and credentials doing the following: + +* Project ID is determined using OTel Resource Detector for the environment +otherwise it falls back to the project provided by [google.FindCredentials]. + +* Credentials are determined using [Application Default Credentials]. The +principal must have `roles/monitoring.metricWriter` role granted. If not a +logged warning will be emitted. Subsequent are silenced to prevent noisy logs. + # Storage Control API Certain control plane and long-running operations for Cloud Storage (including Folder @@ -367,6 +394,11 @@ and Managed Folder operations) are supported via the autogenerated Storage Contr client, which is available as a subpackage in this module. See package docs at [cloud.google.com/go/storage/control/apiv2] or reference the [Storage Control API] docs. +[Application Default Credentials]: https://cloud.google.com/docs/authentication/application-default-credentials +[google.FindCredentials]: https://pkg.go.dev/golang.org/x/oauth2/google#FindDefaultCredentials +[gRFC/66]: https://github.com/grpc/proposal/blob/master/A66-otel-stats.md +[gRFC/78]: https://github.com/grpc/proposal/blob/master/A78-grpc-metrics-wrr-pf-xds.md +[Google Cloud Monitoring]: https://cloud.google.com/monitoring/docs [Cloud Storage IAM docs]: https://cloud.google.com/storage/docs/access-control/iam [XML POST Object docs]: https://cloud.google.com/storage/docs/xml-api/post-object [Cloud Storage retry docs]: https://cloud.google.com/storage/docs/retry-strategy @@ -376,5 +408,6 @@ client, which is available as a subpackage in this module. See package docs at [IAM Service Account Credentials API]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview [custom audit logging]: https://cloud.google.com/storage/docs/audit-logging#add-custom-metadata [Storage Control API]: https://cloud.google.com/storage/docs/reference/rpc/google.storage.control.v2 +[metric explorer]: https://console.cloud.google.com/projectselector/monitoring/metrics-explorer */ package storage // import "cloud.google.com/go/storage" diff --git a/storage/go.mod b/storage/go.mod index a7e91eab7af5..0a9d4fb02966 100644 --- a/storage/go.mod +++ b/storage/go.mod @@ -9,14 +9,20 @@ require ( cloud.google.com/go/compute/metadata v0.5.0 cloud.google.com/go/iam v1.2.0 cloud.google.com/go/longrunning v0.6.0 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/googleapis/gax-go/v2 v2.13.0 + go.opentelemetry.io/contrib/detectors/gcp v1.29.0 + go.opentelemetry.io/otel v1.29.0 + go.opentelemetry.io/otel/sdk v1.29.0 + go.opentelemetry.io/otel/sdk/metric v1.29.0 golang.org/x/oauth2 v0.22.0 google.golang.org/api v0.194.0 google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed google.golang.org/grpc v1.66.0 + google.golang.org/grpc/stats/opentelemetry v0.0.0-20240815194846-86135c37f383 google.golang.org/protobuf v1.34.2 ) @@ -24,6 +30,9 @@ require ( cel.dev/expr v0.16.0 // indirect cloud.google.com/go/auth v0.9.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/monitoring v1.21.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20240822171458-6449f94b4d59 // indirect @@ -40,9 +49,7 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel v1.29.0 // indirect go.opentelemetry.io/otel/metric v1.29.0 // indirect - go.opentelemetry.io/otel/sdk v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/net v0.28.0 // indirect diff --git a/storage/go.sum b/storage/go.sum index 00977ee3cc0a..2d08f8c24b07 100644 --- a/storage/go.sum +++ b/storage/go.sum @@ -11,9 +11,23 @@ cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJ cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8= cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q= +cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= +cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= cloud.google.com/go/longrunning v0.6.0 h1:mM1ZmaNsQsnb+5n1DNPeL0KwQd9jQRqSqSDEkBZr+aI= cloud.google.com/go/longrunning v0.6.0/go.mod h1:uHzSZqW89h7/pasCWNYdUpwGz3PcVWhrWupreVPYLts= +cloud.google.com/go/monitoring v1.21.0 h1:EMc0tB+d3lUewT2NzKC/hr8cSR9WsUieVywzIHetGro= +cloud.google.com/go/monitoring v1.21.0/go.mod h1:tuJ+KNDdJbetSsbSGTqnaBvbauS5kr3Q/koy3Up6r+4= +cloud.google.com/go/trace v1.11.0 h1:UHX6cOJm45Zw/KIbqHe4kII8PupLt/V5tscZUkeiJVI= +cloud.google.com/go/trace v1.11.0/go.mod h1:Aiemdi52635dBR7o3zuc9lLjXo3BwGaChEjCa3tJNmM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 h1:pB2F2JKCj1Znmp2rwxxt1J0Fg0wezTMgWYk5Mpbi1kg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1 h1:oTX4vsorBZo/Zdum6OKPA4o7544hm6smoRv1QjpTwGo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= @@ -93,6 +107,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/detectors/gcp v1.29.0 h1:TiaiXB4DpGD3sdzNlYQxruQngn5Apwzi1X0DRhuGvDQ= +go.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= @@ -103,6 +119,8 @@ go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2 go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/sdk/metric v1.29.0 h1:K2CfmJohnRgvZ9UAj2/FhIf/okdWcNdBwe1m8xFXiSY= +go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -167,6 +185,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc/stats/opentelemetry v0.0.0-20240815194846-86135c37f383 h1:CNj4PPoKghvgFjLa98bpnWt2rn11Ip+vVsrFW812c3c= +google.golang.org/grpc/stats/opentelemetry v0.0.0-20240815194846-86135c37f383/go.mod h1:hb10rC63I9VqLKoQnzWeS/mUSZffTlSCYAa//wAheWQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/storage/grpc_client.go b/storage/grpc_client.go index 011a9e233786..44a001b035f1 100644 --- a/storage/grpc_client.go +++ b/storage/grpc_client.go @@ -21,6 +21,7 @@ import ( "fmt" "hash/crc32" "io" + "log" "net/url" "os" @@ -124,6 +125,15 @@ func newGRPCStorageClient(ctx context.Context, opts ...storageOption) (storageCl return nil, errors.New("storage: GRPC is incompatible with any option that specifies an API for reads") } + if !config.disableClientMetrics { + // Do not fail client creation if enabling metrics fails. + if metricsContext, err := enableClientMetrics(ctx, s); err == nil { + s.metricsContext = metricsContext + s.clientOption = append(s.clientOption, metricsContext.clientOpts...) + } else { + log.Printf("Failed to enable client metrics: %v", err) + } + } g, err := gapic.NewClient(ctx, s.clientOption...) if err != nil { return nil, err @@ -136,6 +146,9 @@ func newGRPCStorageClient(ctx context.Context, opts ...storageOption) (storageCl } func (c *grpcStorageClient) Close() error { + if c.settings.metricsContext != nil { + c.settings.metricsContext.close() + } return c.raw.Close() } diff --git a/storage/grpc_metrics.go b/storage/grpc_metrics.go new file mode 100644 index 000000000000..460a9d0a2b8f --- /dev/null +++ b/storage/grpc_metrics.go @@ -0,0 +1,275 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "fmt" + "log" + "strings" + "time" + + mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" + "github.com/google/uuid" + "go.opentelemetry.io/contrib/detectors/gcp" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/resource" + "google.golang.org/api/option" + "google.golang.org/api/transport" + "google.golang.org/grpc" + "google.golang.org/grpc/stats/opentelemetry" +) + +const ( + monitoredResourceName = "storage.googleapis.com/Client" + metricPrefix = "storage.googleapis.com/client/" +) + +func latencyHistogramBoundaries() []float64 { + boundaries := []float64{} + boundary := 0.0 + increment := 0.002 + // 2ms buckets for first 100ms, so we can have higher resolution for uploads and downloads in the 100 KiB range + for i := 0; i < 50; i++ { + boundaries = append(boundaries, boundary) + // increment by 2ms + boundary += increment + } + // For the remaining buckets do 10 10ms, 10 20ms, and so on, up until 5 minutes + for i := 0; i < 150 && boundary < 300; i++ { + boundaries = append(boundaries, boundary) + if i != 0 && i%10 == 0 { + increment *= 2 + } + boundary += increment + } + return boundaries +} + +func sizeHistogramBoundaries() []float64 { + kb := 1024.0 + mb := 1024.0 * kb + gb := 1024.0 * mb + boundaries := []float64{} + boundary := 0.0 + increment := 128 * kb + // 128 KiB increments up to 4MiB, then exponential growth + for len(boundaries) < 200 && boundary <= 16*gb { + boundaries = append(boundaries, boundary) + boundary += increment + if boundary >= 4*mb { + increment *= 2 + } + } + return boundaries +} + +func metricFormatter(m metricdata.Metrics) string { + return metricPrefix + strings.ReplaceAll(string(m.Name), ".", "/") +} + +func gcpAttributeExpectedDefaults() []attribute.KeyValue { + return []attribute.KeyValue{ + {Key: "location", Value: attribute.StringValue("global")}, + {Key: "cloud_platform", Value: attribute.StringValue("unknown")}, + {Key: "host_id", Value: attribute.StringValue("unknown")}} +} + +// Added to help with tests +type preparedResource struct { + projectToUse string + resource *resource.Resource +} + +func newPreparedResource(ctx context.Context, project string, resourceOptions []resource.Option) (*preparedResource, error) { + detectedAttrs, err := resource.New(ctx, resourceOptions...) + if err != nil { + return nil, err + } + preparedResource := &preparedResource{} + s := detectedAttrs.Set() + p, present := s.Value("cloud.account.id") + if present { + preparedResource.projectToUse = p.AsString() + } else { + preparedResource.projectToUse = project + } + updates := []attribute.KeyValue{} + for _, kv := range gcpAttributeExpectedDefaults() { + if val, present := s.Value(kv.Key); !present || val.AsString() == "" { + updates = append(updates, attribute.KeyValue{Key: kv.Key, Value: kv.Value}) + } + } + r, err := resource.New( + ctx, + resource.WithAttributes( + attribute.KeyValue{Key: "gcp.resource_type", Value: attribute.StringValue(monitoredResourceName)}, + attribute.KeyValue{Key: "instance_id", Value: attribute.StringValue(uuid.New().String())}, + attribute.KeyValue{Key: "project_id", Value: attribute.StringValue(project)}, + attribute.KeyValue{Key: "api", Value: attribute.StringValue("grpc")}, + ), + resource.WithAttributes(detectedAttrs.Attributes()...), + // Last duplicate key / value wins + resource.WithAttributes(updates...), + ) + if err != nil { + return nil, err + } + preparedResource.resource = r + return preparedResource, nil +} + +type metricsContext struct { + // project used by exporter + project string + // client options passed to gRPC channels + clientOpts []option.ClientOption + // instance of metric reader used by gRPC client-side metrics + provider *metric.MeterProvider + // clean func to call when closing gRPC client + close func() +} + +func createHistogramView(name string, boundaries []float64) metric.View { + return metric.NewView(metric.Instrument{ + Name: name, + Kind: metric.InstrumentKindHistogram, + }, metric.Stream{ + Name: name, + Aggregation: metric.AggregationExplicitBucketHistogram{Boundaries: boundaries}, + }) +} + +func newGRPCMetricContext(ctx context.Context, project string) (*metricsContext, error) { + preparedResource, err := newPreparedResource(ctx, project, []resource.Option{resource.WithDetectors(gcp.NewDetector())}) + if err != nil { + return nil, err + } + // Implementation requires a project, if one is not determined possibly user + // credentials. Then we will fail stating gRPC Metrics require a project-id. + if project == "" && preparedResource.projectToUse != "" { + return nil, fmt.Errorf("google cloud project is required to start client-side metrics") + } + // If projectTouse isn't the same as project provided to Storage client, then + // emit a log stating which project is being used to emit metrics to. + if project != preparedResource.projectToUse { + log.Printf("The Project ID configured for metrics is %s, but the Project ID of the storage client is %s. Make sure that the service account in use has the required metric writing role (roles/monitoring.metricWriter) in the project projectIdToUse or metrics will not be written.", preparedResource.projectToUse, project) + } + meOpts := []mexporter.Option{ + mexporter.WithProjectID(preparedResource.projectToUse), + mexporter.WithMetricDescriptorTypeFormatter(metricFormatter), + mexporter.WithCreateServiceTimeSeries(), + mexporter.WithMonitoredResourceDescription(monitoredResourceName, []string{"project_id", "location", "cloud_platform", "host_id", "instance_id", "api"})} + exporter, err := mexporter.New(meOpts...) + if err != nil { + return nil, err + } + // Metric views update histogram boundaries to be relevant to GCS + // otherwise default OTel histogram boundaries are used. + metricViews := []metric.View{ + createHistogramView("grpc.client.attempt.duration", latencyHistogramBoundaries()), + createHistogramView("grpc.client.attempt.rcvd_total_compressed_message_size", sizeHistogramBoundaries()), + createHistogramView("grpc.client.attempt.sent_total_compressed_message_size", sizeHistogramBoundaries()), + } + provider := metric.NewMeterProvider( + metric.WithReader(metric.NewPeriodicReader(&exporterLogSuppressor{exporter: exporter}, metric.WithInterval(time.Minute))), + metric.WithResource(preparedResource.resource), + metric.WithView(metricViews...), + ) + mo := opentelemetry.MetricsOptions{ + MeterProvider: provider, + Metrics: opentelemetry.DefaultMetrics().Add( + "grpc.lb.wrr.rr_fallback", + "grpc.lb.wrr.endpoint_weight_not_yet_usable", + "grpc.lb.wrr.endpoint_weight_stale", + "grpc.lb.wrr.endpoint_weights", + "grpc.lb.rls.cache_entries", + "grpc.lb.rls.cache_size", + "grpc.lb.rls.default_target_picks", + "grpc.lb.rls.target_picks", + "grpc.lb.rls.failed_picks"), + OptionalLabels: []string{"grpc.lb.locality"}, + } + opts := []option.ClientOption{ + option.WithGRPCDialOption(opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo})), + option.WithGRPCDialOption(grpc.WithDefaultCallOptions(grpc.StaticMethodCallOption{})), + } + context := &metricsContext{ + project: preparedResource.projectToUse, + clientOpts: opts, + provider: provider, + close: createShutdown(ctx, provider), + } + return context, nil +} + +func enableClientMetrics(ctx context.Context, s *settings) (*metricsContext, error) { + var project string + c, err := transport.Creds(ctx, s.clientOption...) + if err == nil { + project = c.ProjectID + } + // Enable client-side metrics for gRPC + metricsContext, err := newGRPCMetricContext(ctx, project) + if err != nil { + return nil, fmt.Errorf("gRPC Metrics: %w", err) + } + return metricsContext, nil +} + +func createShutdown(ctx context.Context, provider *metric.MeterProvider) func() { + return func() { + provider.Shutdown(ctx) + } +} + +// Silences permission errors after initial error is emitted to prevent +// chatty logs. +type exporterLogSuppressor struct { + exporter metric.Exporter + emittedFailure bool +} + +// Implements OTel SDK metric.Exporter interface to prevent noisy logs from +// lack of credentials after initial failure. +// https://pkg.go.dev/go.opentelemetry.io/otel/sdk/metric@v1.28.0#Exporter +func (e *exporterLogSuppressor) Export(ctx context.Context, rm *metricdata.ResourceMetrics) error { + if err := e.exporter.Export(ctx, rm); err != nil && !e.emittedFailure { + if strings.Contains(err.Error(), "PermissionDenied") { + e.emittedFailure = true + return fmt.Errorf("gRPC metrics failed due permission issue: %w", err) + } + return err + } + return nil +} + +func (e *exporterLogSuppressor) Temporality(k metric.InstrumentKind) metricdata.Temporality { + return e.exporter.Temporality(k) +} + +func (e *exporterLogSuppressor) Aggregation(k metric.InstrumentKind) metric.Aggregation { + return e.exporter.Aggregation(k) +} + +func (e *exporterLogSuppressor) ForceFlush(ctx context.Context) error { + return e.exporter.ForceFlush(ctx) +} + +func (e *exporterLogSuppressor) Shutdown(ctx context.Context) error { + return e.exporter.Shutdown(ctx) +} diff --git a/storage/grpc_metrics_test.go b/storage/grpc_metrics_test.go new file mode 100644 index 000000000000..23b3cf981e1c --- /dev/null +++ b/storage/grpc_metrics_test.go @@ -0,0 +1,152 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "fmt" + "testing" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/resource" +) + +func TestMetricFormatter(t *testing.T) { + want := "storage.googleapis.com/client/metric/name" + s := metricdata.Metrics{Name: "metric.name"} + got := metricFormatter(s) + if want != got { + t.Errorf("got: %v, want %v", got, want) + } +} + +func TestNewPreparedResource(t *testing.T) { + ctx := context.Background() + for _, test := range []struct { + desc string + detectedAttributes []attribute.KeyValue + wantAttributes attribute.Set + }{ + { + desc: "default values set when GCP attributes are not detected", + wantAttributes: attribute.NewSet(attribute.KeyValue{ + Key: "location", + Value: attribute.StringValue("global"), + }, attribute.KeyValue{ + Key: "cloud_platform", + Value: attribute.StringValue("unknown"), + }, attribute.KeyValue{ + Key: "host_id", + Value: attribute.StringValue("unknown"), + }), + }, + { + desc: "use detected values when GCP attributes are detected", + detectedAttributes: []attribute.KeyValue{ + {Key: "location", + Value: attribute.StringValue("us-central1")}, + {Key: "cloud_platform", + Value: attribute.StringValue("gcp")}, + {Key: "host_id", + Value: attribute.StringValue("gce-instance-id")}, + }, + wantAttributes: attribute.NewSet(attribute.KeyValue{ + Key: "location", + Value: attribute.StringValue("us-central1"), + }, attribute.KeyValue{ + Key: "cloud_platform", + Value: attribute.StringValue("gcp"), + }, attribute.KeyValue{ + Key: "host_id", + Value: attribute.StringValue("gce-instance-id"), + }), + }, { + desc: "use default when value is empty string", + detectedAttributes: []attribute.KeyValue{ + {Key: "location", + Value: attribute.StringValue("us-central1")}, + {Key: "cloud_platform", + Value: attribute.StringValue("")}, + {Key: "host_id", + Value: attribute.StringValue("")}, + }, + wantAttributes: attribute.NewSet(attribute.KeyValue{ + Key: "location", + Value: attribute.StringValue("us-central1"), + }, attribute.KeyValue{ + Key: "cloud_platform", + Value: attribute.StringValue("unknown"), + }, attribute.KeyValue{ + Key: "host_id", + Value: attribute.StringValue("unknown"), + }), + }, + } { + t.Run(test.desc, func(t *testing.T) { + resourceOptions := []resource.Option{resource.WithAttributes(test.detectedAttributes...)} + result, err := newPreparedResource(ctx, "project", resourceOptions) + if err != nil { + t.Errorf("newPreparedResource: %v", err) + } + resultSet := result.resource.Set() + for _, want := range test.wantAttributes.ToSlice() { + got, exists := resultSet.Value(want.Key) + if !exists { + t.Errorf("newPreparedResource: %v not set", want.Key) + continue + } + if got != want.Value { + t.Errorf("newPreparedResource: want[%v] = %v, got: %v", want.Key, want.Value, got) + continue + } + } + }) + } +} + +func TestNewExporterLogSuppressor(t *testing.T) { + ctx := context.Background() + s := &exporterLogSuppressor{exporter: &failingExporter{}} + if err := s.Export(ctx, nil); err == nil { + t.Errorf("exporterLogSuppressor: did not emit an error when one was expected") + } + if err := s.Export(ctx, nil); err != nil { + t.Errorf("exporterLogSuppressor: emitted an error when it should have suppressed") + } +} + +type failingExporter struct{} + +func (f *failingExporter) Export(ctx context.Context, rm *metricdata.ResourceMetrics) error { + return fmt.Errorf("PermissionDenied") +} + +func (f *failingExporter) Temporality(m metric.InstrumentKind) metricdata.Temporality { + return metricdata.CumulativeTemporality +} + +func (f *failingExporter) Aggregation(ik metric.InstrumentKind) metric.Aggregation { + return metric.AggregationDefault{} +} + +func (f *failingExporter) ForceFlush(ctx context.Context) error { + return nil +} + +func (f *failingExporter) Shutdown(ctx context.Context) error { + return nil +} diff --git a/storage/option.go b/storage/option.go index debdb0f52d51..0fc82ed59064 100644 --- a/storage/option.go +++ b/storage/option.go @@ -22,8 +22,9 @@ import ( // storageConfig contains the Storage client option configuration that can be // set through storageClientOptions. type storageConfig struct { - useJSONforReads bool - readAPIWasSet bool + useJSONforReads bool + readAPIWasSet bool + disableClientMetrics bool } // newStorageConfig generates a new storageConfig with all the given @@ -78,3 +79,32 @@ func (w *withReadAPI) ApplyStorageOpt(c *storageConfig) { c.useJSONforReads = w.useJSON c.readAPIWasSet = true } + +type withDisabledClientMetrics struct { + internaloption.EmbeddableAdapter + disabledClientMetrics bool +} + +// WithDisabledClientMetrics is an option that may be passed to [NewClient]. +// gRPC metrics are enabled by default in the GCS client and will export the +// gRPC telemetry discussed in [gRFC/66] and [gRFC/78] to +// [Google Cloud Monitoring]. The option is used to disable metrics. +// Google Cloud Support can use this information to more quickly diagnose +// problems related to GCS and gRPC. +// Sending this data does not incur any billing charges, and requires minimal +// CPU (a single RPC every few minutes) or memory (a few KiB to batch the +// telemetry). +// +// The default is to enable client metrics. To opt-out of metrics collected use +// this option. +// +// [gRFC/66]: https://github.com/grpc/proposal/blob/master/A66-otel-stats.md +// [gRFC/78]: https://github.com/grpc/proposal/blob/master/A78-grpc-metrics-wrr-pf-xds.md +// [Google Cloud Monitoring]: https://cloud.google.com/monitoring/docs +func WithDisabledClientMetrics() option.ClientOption { + return &withDisabledClientMetrics{disabledClientMetrics: true} +} + +func (w *withDisabledClientMetrics) ApplyStorageOpt(c *storageConfig) { + c.disableClientMetrics = w.disabledClientMetrics +} diff --git a/storage/option_test.go b/storage/option_test.go index 08659073e24d..c28aaba2441b 100644 --- a/storage/option_test.go +++ b/storage/option_test.go @@ -31,40 +31,54 @@ func TestApplyStorageOpt(t *testing.T) { desc: "set JSON option", opts: []option.ClientOption{WithJSONReads()}, want: storageConfig{ - useJSONforReads: true, - readAPIWasSet: true, + useJSONforReads: true, + readAPIWasSet: true, + disableClientMetrics: false, }, }, { desc: "set XML option", opts: []option.ClientOption{WithXMLReads()}, want: storageConfig{ - useJSONforReads: false, - readAPIWasSet: true, + useJSONforReads: false, + readAPIWasSet: true, + disableClientMetrics: false, }, }, { desc: "set conflicting options, last option set takes precedence", opts: []option.ClientOption{WithJSONReads(), WithXMLReads()}, want: storageConfig{ - useJSONforReads: false, - readAPIWasSet: true, + useJSONforReads: false, + readAPIWasSet: true, + disableClientMetrics: false, }, }, { desc: "empty options", opts: []option.ClientOption{}, want: storageConfig{ - useJSONforReads: false, - readAPIWasSet: false, + useJSONforReads: false, + readAPIWasSet: false, + disableClientMetrics: false, }, }, { desc: "set Google API option", opts: []option.ClientOption{option.WithEndpoint("")}, want: storageConfig{ - useJSONforReads: false, - readAPIWasSet: false, + useJSONforReads: false, + readAPIWasSet: false, + disableClientMetrics: false, + }, + }, + { + desc: "disable metrics option", + opts: []option.ClientOption{WithDisabledClientMetrics()}, + want: storageConfig{ + useJSONforReads: false, + readAPIWasSet: false, + disableClientMetrics: true, }, }, } {