diff --git a/CHANGELOG.md b/CHANGELOG.md index 54ef6f5..f19d426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## 4.1.0 + - Added support for custom headers [#187](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/187) + +## 4.0.0 + - SSL settings that were marked deprecated in version `3.15.0` are now marked obsolete, and will prevent the plugin from starting. + - These settings are: + - `ca_file`, which should be replaced by `ssl_certificate_authorities` + - `keystore`, which should be replaced by `ssl_keystore_path` + - `keystore_password`, which should be replaced by `ssl_keystore_password` + - `keystore_type`, which should be replaced by `ssl_keystore_password` + - `ssl`, which should be replaced by `ssl_enabled` + - [#183](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/183) + ## 3.16.2 - Add `x-elastic-product-origin` header to Elasticsearch requests [#185](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/185) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 00f5b16..a40925b 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -110,7 +110,7 @@ Authentication to a secure Elasticsearch cluster is possible using _one_ of the * <> AND <> * <> * <> -* <> and/or <> +* <> and/or <> [id="plugins-{type}s-{plugin}-autz"] ==== Authorization @@ -121,7 +121,10 @@ The `monitoring` permission at cluster level is necessary to perform periodic co [id="plugins-{type}s-{plugin}-options"] ==== Elasticsearch Filter Configuration Options -This plugin supports the following configuration options plus the <> and the <> described later. +This plugin supports the following configuration options plus the <> described later. + +NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the +<> for more details. [cols="<,<,<",options="header",] |======================================================================= @@ -132,6 +135,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -144,7 +148,6 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No -| <> |<>|__Deprecated__ | <> |<>|No | <> |list of <>|No | <> |list of <>|No @@ -228,6 +231,15 @@ Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. For more info, check out the {logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. +[id="plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is <> + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to Elasticsearch. +These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + [id="plugins-{type}s-{plugin}-docinfo_fields"] ===== `docinfo_fields` @@ -247,6 +259,15 @@ Example: } } +[id="plugins-{type}s-{plugin}-custom_headers"] +===== `custom_headers` + + * Value type is <> + * Default value is empty + +Pass a set of key value pairs as the headers sent in each request to an elasticsearch node. +The values of these custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers. + [id="plugins-{type}s-{plugin}-enable_sort"] ===== `enable_sort` @@ -519,57 +540,21 @@ Tags the event on failure to look up previous log event information. This can be Basic Auth - username -[id="plugins-{type}s-{plugin}-deprecated-options"] -==== Elasticsearch Filter Deprecated Configuration Options - -This plugin supports the following deprecated configurations. +[id="plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Filter Obsolete Configuration Options -WARNING: Deprecated options are subject to removal in future releases. +WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. [cols="<,<,<",options="header",] |======================================================================= -|Setting|Input type|Replaced by -| <> |a valid filesystem path|<> -| <> |a valid filesystem path|<> -| <> |<>|<> +|Setting|Replaced by +| ca_file |<> +| keystore |<> +| keystore_password |<> +| ssl |<> |======================================================================= -[id="plugins-{type}s-{plugin}-ca_file"] -===== `ca_file` -deprecated[3.15.0, Replaced by <>] - -* Value type is <> -* There is no default value for this setting. - -SSL Certificate Authority file - -[id="plugins-{type}s-{plugin}-ssl"] -===== `ssl` -deprecated[3.15.0, Replaced by <>] - -* Value type is <> -* Default value is `false` - -SSL - -[id="plugins-{type}s-{plugin}-keystore"] -===== `keystore` -deprecated[3.15.0, Replaced by <>] - -* Value type is <> -* There is no default value for this setting. - -The keystore used to present a certificate to the server. It can be either .jks or .p12 - -[id="plugins-{type}s-{plugin}-keystore_password"] -===== `keystore_password` -deprecated[3.15.0, Replaced by <>] - -* Value type is <> -* There is no default value for this setting. - -Set the keystore password - [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] diff --git a/lib/logstash/filters/elasticsearch.rb b/lib/logstash/filters/elasticsearch.rb index 1b0fd0a..abd3329 100644 --- a/lib/logstash/filters/elasticsearch.rb +++ b/lib/logstash/filters/elasticsearch.rb @@ -3,7 +3,6 @@ require "logstash/namespace" require "logstash/json" require 'logstash/plugin_mixins/ca_trusted_fingerprint_support' -require "logstash/plugin_mixins/normalize_config_support" require "monitor" require_relative "elasticsearch/client" @@ -36,6 +35,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base # Hash of docinfo fields to copy from old event (found via elasticsearch) into new event config :docinfo_fields, :validate => :hash, :default => {} + # Custom headers for Elasticsearch requests + config :custom_headers, :validate => :hash, :default => {} + # Hash of aggregation names to copy from elasticsearch response into Logstash event fields config :aggregation_fields, :validate => :hash, :default => {} @@ -62,18 +64,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base # Set the address of a forward HTTP proxy. config :proxy, :validate => :uri_or_empty - # SSL - config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead." - - # SSL Certificate Authority file - config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." - - # The keystore used to present a certificate to the server. - # It can be either .jks or .p12 - config :keystore, :validate => :path, :deprecated => "Use 'ssl_keystore_path' instead." - - # Set the keystore password - config :keystore_password, :validate => :password, :deprecated => "Use 'ssl_keystore_password' instead." # OpenSSL-style X.509 certificate certificate to authenticate the client config :ssl_certificate, :validate => :path @@ -135,11 +125,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base # What status codes to retry on? config :retry_on_status, :validate => :number, :list => true, :default => [500, 502, 503, 504] + + config :ssl, :obsolete => "Set 'ssl_enabled' instead." + config :ca_file, :obsolete => "Set 'ssl_certificate_authorities' instead." + config :keystore, :obsolete => "Set 'ssl_keystore_path' instead." + config :keystore_password, :validate => :password, :obsolete => "Set 'ssl_keystore_password' instead." + # config :ca_trusted_fingerprint, :validate => :sha_256_hex include LogStash::PluginMixins::CATrustedFingerprintSupport - include LogStash::PluginMixins::NormalizeConfigSupport - include MonitorMixin attr_reader :shared_client @@ -269,7 +263,9 @@ def client_options :ssl => client_ssl_options, :retry_on_failure => @retry_on_failure, :retry_on_status => @retry_on_status, - :user_agent => prepare_user_agent + :user_agent => prepare_user_agent, + :custom_headers => @custom_headers + } end @@ -488,46 +484,9 @@ def setup_serverless end def setup_ssl_params! - @ssl_enabled = normalize_config(:ssl_enabled) do |normalize| - normalize.with_deprecated_alias(:ssl) - end - - # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set - infer_ssl_enabled_from_hosts - - @ssl_keystore_path = normalize_config(:ssl_keystore_path) do |normalize| - normalize.with_deprecated_alias(:keystore) - end - - @ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize| - normalize.with_deprecated_alias(:keystore_password) - end - - @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize| - normalize.with_deprecated_mapping(:ca_file) do |ca_file| - [ca_file] - end - end - - params['ssl_enabled'] = @ssl_enabled - params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil? - params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil? - params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil? - end - - def infer_ssl_enabled_from_hosts - return if original_params.include?('ssl') || original_params.include?('ssl_enabled') - - @ssl_enabled = params['ssl_enabled'] = effectively_ssl? - end - - def effectively_ssl? - return true if @ssl_enabled - - hosts = Array(@hosts) - return false if hosts.nil? || hosts.empty? - - hosts.all? { |host| host && host.to_s.start_with?("https") } + # Infer the value if neither `ssl_enabled` was not set + return if original_params.include?('ssl_enabled') + params['ssl_enabled'] = @ssl_enabled ||= Array(@hosts).all? { |host| host && host.to_s.start_with?("https") } end end #class LogStash::Filters::Elasticsearch diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index c6223e3..4bc68da 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -20,6 +20,7 @@ def initialize(logger, hosts, options = {}) api_key = options.fetch(:api_key, nil) proxy = options.fetch(:proxy, nil) user_agent = options[:user_agent] + custom_headers = options[:custom_headers] transport_options = { } transport_options[:headers] = options.fetch(:serverless, false) ? DEFAULT_EAV_HEADER.dup : {} @@ -27,7 +28,7 @@ def initialize(logger, hosts, options = {}) transport_options[:headers].merge!(setup_api_key(api_key)) transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" }) transport_options[:headers].merge!(INTERNAL_ORIGIN_HEADER) - + transport_options[:headers].merge!(custom_headers) unless custom_headers.empty? transport_options[:pool_max] = 1000 transport_options[:pool_max_per_route] = 100 diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec index 7802533..a05f9c3 100644 --- a/logstash-filter-elasticsearch.gemspec +++ b/logstash-filter-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-elasticsearch' - s.version = '3.16.2' + s.version = '4.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Copies fields from previous log events in Elasticsearch to current events " s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -24,7 +24,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5 s.add_runtime_dependency 'manticore', ">= 0.7.1" s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0' - s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0' s.add_development_dependency 'cabin', ['~> 0.6'] s.add_development_dependency 'webrick' s.add_development_dependency 'logstash-devutils' diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb index 30f3ddc..ad752b7 100644 --- a/spec/filters/elasticsearch_spec.rb +++ b/spec/filters/elasticsearch_spec.rb @@ -333,6 +333,30 @@ end end + context "with custom headers" do + let(:config) do + { + "query" => "*", + "custom_headers" => { "Custom-Header-1" => "Custom Value 1", "Custom-Header-2" => "Custom Value 2" } + } + end + + let(:plugin) { LogStash::Filters::Elasticsearch.new(config) } + let(:client_double) { double("client") } + let(:transport_double) { double("transport", options: { transport_options: { headers: config["custom_headers"] } }) } + + before do + allow(plugin).to receive(:get_client).and_return(client_double) + allow(client_double).to receive(:client).and_return(transport_double) + end + + it "sets custom headers" do + plugin.register + client = plugin.send(:get_client).client + expect(client.options[:transport_options][:headers]).to match(hash_including(config["custom_headers"])) + end + end + context "if query is on nested field" do let(:config) do { diff --git a/spec/filters/elasticsearch_ssl_spec.rb b/spec/filters/elasticsearch_ssl_spec.rb index 6d505d8..1cdc80f 100644 --- a/spec/filters/elasticsearch_ssl_spec.rb +++ b/spec/filters/elasticsearch_ssl_spec.rb @@ -24,6 +24,23 @@ subject.close end + describe "obsolete settings" do + [{:name => 'ca_file', :canonical_name => 'ssl_certificate_authorities'}, + {:name => "keystore", :canonical_name => 'ssl_keystore_path'}, + {:name => "keystore_password", :canonical_name => "ssl_keystore_password"}, + {:name => "ssl", :canonical_name => "ssl_enabled"} + ].each do |config_settings| + context "with option #{config_settings[:name]}" do + let(:obsolete_config) { settings.merge(config_settings[:name] => 'test_value') } + it "emits an error about the setting `#{config_settings[:name]}` now being obsolete and provides guidance to use `#{config_settings[:canonical_name]}`" do + error_text = /The setting `#{config_settings[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. Set '#{config_settings[:canonical_name]}' instead/i + expect { LogStash::Filters::Elasticsearch.new(obsolete_config) }.to raise_error LogStash::ConfigurationError, error_text + end + + end + end + end + context "when ssl_enabled is" do context "true and there is no https hosts" do let(:hosts) { %w[http://es01 http://es01] }