diff --git a/elastic/logs/README.md b/elastic/logs/README.md index 2c9bc7d2..7cd1f561 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -259,6 +259,7 @@ The following parameters are available: * `index_mode` (default: unset): What index mode to use. Accepted values: `standard` and `logs`. * `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. Only supported in `logging-querying` track. * `include_non_serverless_index_settings` (default: true for non-serverless clusters, false for serverless clusters): Whether to include non-serverless index settings. +* `codec` (default: unset): Configured the `index.codec` index setting, which controls how stored fields get stored / compressed. ### Querying parameters diff --git a/elastic/logs/challenges/logging-indexing-with-ccr.json b/elastic/logs/challenges/logging-indexing-with-ccr.json new file mode 100644 index 00000000..89eb2248 --- /dev/null +++ b/elastic/logs/challenges/logging-indexing-with-ccr.json @@ -0,0 +1,47 @@ +{% import "rally.helpers" as rally %} +{% set p_follow_index_pattern = "copy-{{leader_index}}" %} +{ + "name": "logging-indexing-with-ccr-enabled", + "description": "Configures CCR (Cross Cluster Replication) using an auto follow pattern that automatically auto follows logs-* data streams into the local cluster using a different name. After the log data set get indexed.", + "default": false, + "schedule": [ + {# non-serverless-index-statistics-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%} + { + "name": "setup-remote-for-local", + "operation": { + "operation-type": "setup-local-remote" + } + }, + { + "name": "put_auto_follow_pattern", + "operation": { + "operation-type": "raw-request", + "path": "/_ccr/auto_follow/my_pattern", + "method": "PUT", + "body": { + "remote_cluster": "local", + "leader_index_patterns": ["logs-*"], + "follow_index_pattern": "{{ p_follow_index_pattern }}" + } + } + }, + {% include "tasks/index-setup.json" %}, + {%- endif -%}{# non-serverless-index-statistics-marker-end #} + { + "name": "bulk-index", + "operation": { + "operation-type": "raw-bulk", + "param-source": "processed-source", + "time-format": "milliseconds", + "profile": "fixed_interval", + "bulk-size": {{ p_bulk_size }}, + "detailed-results": true + }, + "clients": {{ p_bulk_indexing_clients }}{% if p_throttle_indexing %}, + "ignore-response-error-level": "{{error_level | default('non-fatal')}}", + "schedule": "timestamp-throttler", + "max-delay-secs": 1 + {% endif %} + } + ] +} diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json index 41326b5d..c533134f 100644 --- a/elastic/logs/templates/component/track-shared-logsdb-mode.json +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -4,6 +4,9 @@ {% if index_mode %} "index": { "mode": {{ index_mode | tojson }} + {% if codec %} + ,"codec": {{codec | tojson}} + {% endif %} {% if source_mode %} ,"mapping.source.mode": {{ source_mode | tojson }} {% endif %} diff --git a/elastic/logs/templates/composable/logs-k8-application.log.json b/elastic/logs/templates/composable/logs-k8-application.log.json index 3a35be45..4d358075 100644 --- a/elastic/logs/templates/composable/logs-k8-application.log.json +++ b/elastic/logs/templates/composable/logs-k8-application.log.json @@ -5,7 +5,6 @@ "template": { "settings": { "index": { - "codec": "best_compression", "mapping": { "total_fields": { "limit": "10000" diff --git a/elastic/logs/track.py b/elastic/logs/track.py index 390ed6a0..04729918 100644 --- a/elastic/logs/track.py +++ b/elastic/logs/track.py @@ -46,6 +46,16 @@ from shared.track_processors.track_id_generator import TrackIdGenerator +async def setup_local_remote(es, params): + response = await es.cluster.state() + master_node = response["master_node"] + response = await es.nodes.info() + ip = response["nodes"][master_node]["transport_address"] + p_settings = {"cluster.remote.local.seeds": ip} + response = await es.cluster.put_settings(persistent=p_settings) + return {"weight": 1, "unit": "ops"} + + def register(registry): registry.register_param_source("initial-indices-source", InitialIndicesParamSource) registry.register_param_source("add-track-path", parameter_sources.add_track_path) @@ -88,3 +98,5 @@ def register(registry): registry.register_runner("configure-remote-clusters", ConfigureRemoteClusters(), async_runner=True) registry.register_runner("configure-ccr", ConfigureCrossClusterReplication(), async_runner=True) registry.register_runner("multi-cluster-wrapper", MultiClusterWrapper(), async_runner=True) + + registry.register_runner("setup-local-remote", setup_local_remote, async_runner=True)