Add OpenTelemetry Tracing capabilities to Cloud Controller

Gemfile

@@ -68,6 +68,19 @@ gem 'fog-openstack'
 gem 'cf-uaa-lib', '~> 4.0.4'
 gem 'vcap-concurrency', git: 'https://github.com/cloudfoundry/vcap-concurrency.git', ref: '2a5b0179'
 
+gem 'opentelemetry-exporter-otlp', '~> 0.26.1'
+gem 'opentelemetry-instrumentation-delayed_job', '~> 0.22.1'
+gem 'opentelemetry-instrumentation-http_client', '~> 0.22.3'
+gem 'opentelemetry-instrumentation-mysql2', '~> 0.27.0'
+gem 'opentelemetry-instrumentation-net_http', '~> 0.22.4'
+gem 'opentelemetry-instrumentation-pg', '~> 0.27.1'
+gem 'opentelemetry-instrumentation-rake', '~> 0.2.1'
+gem 'opentelemetry-instrumentation-redis', '~> 0.25.3'
+gem 'opentelemetry-propagator-b3'
+gem 'opentelemetry-propagator-jaeger', '~> 0.21.0'
+gem 'opentelemetry-propagator-xray', '~> 0.22.1'
+gem 'opentelemetry-sdk', '~> 1.3'
+
 group :db do
   gem 'mysql2', '~> 0.5.6'
   gem 'pg'

Gemfile.lock

@@ -351,6 +351,66 @@ GEM
     oj (3.16.3)
       bigdecimal (>= 3.0)
     openssl (3.2.0)
+    opentelemetry-api (1.2.5)
+    opentelemetry-common (0.20.1)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-otlp (0.26.3)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos-types (~> 1.3)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-helpers-mysql (0.1.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20)
+    opentelemetry-helpers-sql-obfuscation (0.1.0)
+      opentelemetry-common (~> 0.20)
+    opentelemetry-instrumentation-base (0.22.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-registry (~> 0.1)
+    opentelemetry-instrumentation-delayed_job (0.22.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-http_client (0.22.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-mysql2 (0.27.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-helpers-mysql
+      opentelemetry-helpers-sql-obfuscation
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-net_http (0.22.4)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-pg (0.27.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-helpers-sql-obfuscation
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-rake (0.2.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-redis (0.25.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-propagator-b3 (0.21.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-propagator-jaeger (0.21.1)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-propagator-xray (0.22.1)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-registry (0.3.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-sdk (1.4.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.10.0)
+      opentelemetry-api (~> 1.0)
     os (1.1.4)
     palm_civet (1.1.0)
     parallel (1.24.0)
@@ -635,6 +695,18 @@ DEPENDENCIES
   nokogiri (>= 1.10.5)
   oj
   openssl (>= 3.2)
+  opentelemetry-exporter-otlp (~> 0.26.1)
+  opentelemetry-instrumentation-delayed_job (~> 0.22.1)
+  opentelemetry-instrumentation-http_client (~> 0.22.3)
+  opentelemetry-instrumentation-mysql2 (~> 0.27.0)
+  opentelemetry-instrumentation-net_http (~> 0.22.4)
+  opentelemetry-instrumentation-pg (~> 0.27.1)
+  opentelemetry-instrumentation-rake (~> 0.2.1)
+  opentelemetry-instrumentation-redis (~> 0.25.3)
+  opentelemetry-propagator-b3
+  opentelemetry-propagator-jaeger (~> 0.21.0)
+  opentelemetry-propagator-xray (~> 0.22.1)
+  opentelemetry-sdk (~> 1.3)
   palm_civet
   parallel_tests
   pg

app/jobs/cc_job.rb

@@ -1,6 +1,8 @@
 module VCAP::CloudController
   module Jobs
     class CCJob
+      attr_accessor :otel_tracing_carrier
+
       def reschedule_at(time, attempts)
         time + (attempts**4) + 5
       end

app/jobs/enqueuer.rb

@@ -5,6 +5,7 @@
 require 'jobs/logging_context_job'
 require 'jobs/timeout_job'
 require 'securerandom'
+require 'opentelemetry/sdk'
 
 module VCAP::CloudController
   module Jobs
@@ -26,8 +27,12 @@ def enqueue_pollable(existing_guid: nil)
 
         wrapped_job = yield wrapped_job if block_given?
 
-        delayed_job = enqueue_job(wrapped_job)
-        PollableJobModel.find_by_delayed_job(delayed_job)
+        tracer = OpenTelemetry.tracer_provider.tracer(self.class.name, '1.0.0')
+        job_handler = wrapped_job.handler.class.name.respond_to?(:split) ? wrapped_job.handler.class.name.split('::').last : wrapped_job.handler.class.name
+        tracer.in_span("enqueue-job: #{job_handler}", kind: :producer) do
+          delayed_job = enqueue_job(wrapped_job)
+          PollableJobModel.find_by_delayed_job(delayed_job)
+        end
       end
 
       def run_inline
@@ -42,7 +47,10 @@ def enqueue_job(job)
         @opts['guid'] = SecureRandom.uuid
         request_id = ::VCAP::Request.current_id
         timeout_job = TimeoutJob.new(job, job_timeout)
-        logging_context_job = LoggingContextJob.new(timeout_job, request_id)
+        logging_context_job = LoggingContextJob.new(
+          timeout_job,
+          request_id
+        )
         @opts[:priority] = job_priority unless @opts[:priority] || job_priority.nil?
         Delayed::Job.enqueue(logging_context_job, @opts)
       end

app/jobs/logging_context_job.rb

@@ -1,5 +1,6 @@
 require 'jobs/wrapping_job'
 require 'presenters/error_presenter'
+require 'opentelemetry/sdk'
 
 module VCAP::CloudController
   module Jobs

app/jobs/wrapping_job.rb

@@ -1,9 +1,11 @@
 require 'jobs/cc_job'
+require 'opentelemetry-sdk'
 
 module VCAP::CloudController
   module Jobs
     class WrappingJob < VCAP::CloudController::Jobs::CCJob
       attr_reader :handler
+      attr_accessor :otel_api_trace_carrier, :otel_job_trace_carrier
 
       def initialize(handler)
         @handler = handler

config/cloud_controller.yml

@@ -402,3 +402,16 @@ custom_metric_tag_prefix_list: ["metric.tag.cloudfoundry.org"]
 
 max_manifest_service_binding_poll_duration_in_seconds: 60
 update_metric_tags_on_rename: true
+
+otel:
+  tracing:
+    enabled: false
+    api_url: ''
+    api_token: ''
+    sampling_ratio: 1.0
+    redact:
+      db_statements: true
+    propagation:
+      accept_sampling_instruction: true
+      extractors: ['tracecontext', 'xray', 'jaeger', 'baggage', 'b3', 'b3multi']
+      injectors: ['tracecontext', 'xray', 'jaeger', 'baggage', 'b3', 'b3multi']

config/initializers/cloudfront_signer.rb

@@ -1,7 +1,7 @@
 require 'cloudfront-signer'
 
 module CCInitializers
-  def self.cloudfront_signer(cc_config)
+  def self.cloudfront_signer(cc_config, _)
     return if cc_config[:droplets].blank?
 
     cdn_config = cc_config[:droplets][:cdn]

config/initializers/delayed_job.rb

@@ -1,5 +1,5 @@
 module CCInitializers
-  def self.delayed_job(_)
+  def self.delayed_job(_, _)
     ::Delayed::Worker.backend = :sequel
   end
 end

config/initializers/honeycomb.rb

@@ -1,7 +1,7 @@
 require 'honeycomb-beeline'
 
 module CCInitializers
-  def self.honeycomb(cc_config)
+  def self.honeycomb(cc_config, _)
     return unless cc_config[:honeycomb]
 
     Honeycomb.configure do |hc|

config/initializers/inflections.rb

@@ -1,5 +1,5 @@
 module CCInitializers
-  def self.inflections(_)
+  def self.inflections(_, _)
     ActiveSupport::Inflector.inflections(:en) do |inflect|
       inflect.irregular 'quota', 'quotas'
     end

config/initializers/json.rb

@@ -1,7 +1,7 @@
 require 'active_support/json/encoding'
 
 module CCInitializers
-  def self.json(_cc_config)
+  def self.json(_, _)
     MultiJson.use(:oj)
     Oj::Rails.optimize # Use optimized encoders instead of as_json() methods for available classes.
     Oj.default_options = {

config/initializers/opentelemetry.rb

@@ -0,0 +1,94 @@
+require 'opentelemetry/sdk'
+require 'opentelemetry/exporter/otlp'
+require 'opentelemetry/instrumentation/http_client'
+require 'opentelemetry/instrumentation/net/http'
+require 'opentelemetry/instrumentation/mysql2'
+require 'opentelemetry/instrumentation/redis'
+require 'opentelemetry/instrumentation/rake'
+require 'opentelemetry/instrumentation/pg'
+require 'opentelemetry/propagator/jaeger'
+require 'opentelemetry/propagator/xray'
+require 'delayed_job/opentelemetry/instrumentation'
+
+module CCInitializers
+  def self.opentelemetry(cc_config, context)
+    return unless cc_config.dig(:otel, :tracing, :enabled)
+
+    OpenTelemetry.logger = Steno.logger(context == :api ? 'cc.api.opentelemetry' : 'cc.background.opentelemetry')
+
+    trace_api_url = cc_config[:otel][:tracing][:api_url]
+    trace_api_token = cc_config[:otel][:tracing][:api_token]
+    sampler = OpenTelemetry::SDK::Trace::Samplers.trace_id_ratio_based(cc_config[:otel][:tracing][:sampling_ratio])
+    sampler = OpenTelemetry::SDK::Trace::Samplers.parent_based(root: sampler) if cc_config.dig(:otel, :tracing, :propagation, :accept_sampling_instruction)
+
+    OpenTelemetry::SDK.configure do |c|
+      c.add_span_processor(
+        OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+          OpenTelemetry::Exporter::OTLP::Exporter.new(
+            endpoint: trace_api_url,
+            headers: {
+              Authorization: trace_api_token
+            }
+          )
+        )
+      )
+      version = {
+        V3: VCAP::CloudController::Constants::API_VERSION_V3.to_s,
+        V2: VCAP::CloudController::Constants::API_VERSION.to_s,
+        OSBAPI: VCAP::CloudController::Constants::OSBAPI_VERSION.to_s
+      }
+      # Set the service name to cloud_controller_ng_api if it is the webserver process and to cloud_controller_ng_worker if it is the worker process
+      resource = OpenTelemetry::SDK::Resources::Resource
+      conventions = OpenTelemetry::SemanticConventions::Resource
+      c.resource = resource.create({
+                                     conventions::SERVICE_NAMESPACE => 'cloud_controller_ng',
+                                     conventions::SERVICE_NAME => "cloud_controller_ng-#{context}",
+                                     conventions::SERVICE_VERSION => version.to_json,
+                                     conventions::SERVICE_INSTANCE_ID => Socket.gethostname + ':' + Process.pid.to_s,
+                                     conventions::HOST_NAME => Socket.gethostname
+                                   })
+
+      c.use 'OpenTelemetry::Instrumentation::HttpClient'
+      c.use 'OpenTelemetry::Instrumentation::Net::HTTP'
+      c.use 'OpenTelemetry::Instrumentation::Redis'
+      c.use 'OpenTelemetry::Instrumentation::Rake'
+      c.use 'OpenTelemetry::Instrumentation::CCDelayedJob'
+      unless defined?(::PG).nil?
+        c.use 'OpenTelemetry::Instrumentation::PG', {
+          db_statement: (cc_config[:otel][:tracing][:redact][:db_statement] ? :obfuscate : :include)
+        }
+      end
+      unless defined?(::Mysql2).nil?
+        c.use 'OpenTelemetry::Instrumentation::Mysql2', {
+          db_statement: (cc_config[:otel][:tracing][:redact][:db_statement] ? :obfuscate : :include)
+        }
+      end
+    end
+
+    # Configuration of sampling
+    OpenTelemetry.tracer_provider.sampler = sampler if trace_api_url && !trace_api_url.empty? && !trace_api_token.empty?
+
+    extractors = define_propagators(cc_config.dig(:otel, :tracing, :propagation, :extractors))
+    injectors = define_propagators(cc_config.dig(:otel, :tracing, :propagation, :injectors))
+
+    OpenTelemetry.propagation = OpenTelemetry::Context::Propagation::CompositeTextMapPropagator.compose(injectors:, extractors:)
+  end
+
+  def self.define_propagators(config_propagators)
+    config_propagators = ['none'] if config_propagators.nil?
+    config_propagators.uniq.collect do |propagator|
+      case propagator
+      when 'tracecontext' then OpenTelemetry::Trace::Propagation::TraceContext.text_map_propagator
+      when 'baggage' then OpenTelemetry::Baggage::Propagation.text_map_propagator
+      when 'b3' then OpenTelemetry::Propagator::B3::Single.text_map_propagator
+      when 'b3multi' then OpenTelemetry::Propagator::B3::Multi.text_map_propagator
+      when 'jaeger' then OpenTelemetry::Propagator::Jaeger.text_map_propagator
+      when 'xray' then OpenTelemetry::Propagator::XRay.text_map_propagator
+      when 'none' then OpenTelemetry::SDK::Configurator::NoopTextMapPropagator.new
+      else
+        OpenTelemetry.logger.warn "The #{propagator} propagator is unknown and cannot be configured"
+        OpenTelemetry::SDK::Configurator::NoopTextMapPropagator.new
+      end
+    end
+  end
+end

config/initializers/wrap_parameters.rb

@@ -3,7 +3,7 @@
 # This file contains settings for ActionController::ParamsWrapper which
 # is enabled by default.
 module CCInitializers
-  def self.wrap_parameters(_)
+  def self.wrap_parameters(_, _)
     # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
     ActiveSupport.on_load(:action_controller) do
       wrap_parameters format: []

config/newrelic/initializers/new_relic_app_observer_instrumentation.rb

@@ -2,7 +2,7 @@
 require 'cloud_controller/process_observer'
 
 module CCInitializers
-  def self.new_relic_app_observer_instrumentation(_)
+  def self.new_relic_app_observer_instrumentation(_, _)
     VCAP::CloudController::ProcessObserver.class_eval do
       include ::NewRelic::Agent::MethodTracer
 

config/newrelic/initializers/new_relic_enable_gc_profiler.rb

@@ -1,7 +1,7 @@
 require 'newrelic_rpm'
 
 module CCInitializers
-  def self.new_relic_enable_gc_profiler(_)
+  def self.new_relic_enable_gc_profiler(_, _)
     # NewRelic agent's CoreGCProfiler will clear GC stats
     GC::Profiler.enable
   end

config/newrelic/initializers/new_relic_loggregator_instrumentation.rb

@@ -2,7 +2,7 @@
 require 'app_log_emitter'
 
 module CCInitializers
-  def self.new_relic_loggregator_instrumentation(_)
+  def self.new_relic_loggregator_instrumentation(_, _)
     VCAP::AppLogEmitter.class_eval do
       include ::NewRelic::Agent::MethodTracer
 

lib/cloud_controller.rb

@@ -34,9 +34,10 @@ module VCAP::CloudController; end
 require 'cloud_controller/errors/buildpack_error'
 require 'cloud_controller/errors/v3/api_error'
 require 'cloud_controller/errors/v3/details'
-require 'delayed_job_plugins/deserialization_retry'
-require 'delayed_job_plugins/before_enqueue_hook'
-require 'delayed_job_plugins/after_enqueue_hook'
+require 'delayed_job/plugins/deserialization_retry'
+require 'delayed_job/plugins/before_enqueue_hook'
+require 'delayed_job/plugins/after_enqueue_hook'
+
 require 'sequel_plugins/sequel_plugins'
 require 'vcap/sequel_add_association_dependencies_monkeypatch'
 require 'access/access'

lib/cloud_controller/config.rb

@@ -99,10 +99,11 @@ def valid_in_userinfo?(value)
       end
     end
 
-    attr_reader :config_hash
+    attr_reader :config_hash, :context
 
     def initialize(config_hash, context: :api)
       @config_hash = config_hash
+      @context = context
       @schema_class = self.class.schema_class_for_context(context, config_hash)
     end
 
@@ -181,7 +182,7 @@ def run_initializers_in_directory(path)
       Dir.glob(File.expand_path(path, __FILE__)).each do |file|
         require file
         method = File.basename(file).sub('.rb', '').tr('-', '_')
-        CCInitializers.send(method, @config_hash)
+        CCInitializers.send(method, @config_hash, @context)
       end
     end
   end