Add auto follow pattern challenge (#734)

Add a new challenge that uses auto follow pattern to auto follow data streams with logs-* pattern into the local cluster. This allows to benchmark CCR in a semi realistic way while the data set gets indexed, without running multiple clusters.

elastic/logs/README.md

@@ -259,6 +259,7 @@ The following parameters are available:
 * `index_mode` (default: unset): What index mode to use. Accepted values: `standard` and `logs`. 
 * `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. Only supported in `logging-querying` track.
 * `include_non_serverless_index_settings` (default: true for non-serverless clusters, false for serverless clusters): Whether to include non-serverless index settings.
+* `codec` (default: unset): Configured the `index.codec` index setting, which controls how stored fields get stored / compressed.
 
 ### Querying parameters
 

elastic/logs/challenges/logging-indexing-with-ccr.json

@@ -0,0 +1,47 @@
+{% import "rally.helpers" as rally %}
+{% set p_follow_index_pattern = "copy-{{leader_index}}" %}
+{
+  "name": "logging-indexing-with-ccr-enabled",
+  "description": "Configures CCR (Cross Cluster Replication) using an auto follow pattern that automatically auto follows logs-* data streams into the local cluster using a different name. After the log data set get indexed.",
+  "default": false,
+  "schedule": [
+    {# non-serverless-index-statistics-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}
+    {
+        "name": "setup-remote-for-local",
+        "operation": {
+          "operation-type": "setup-local-remote"
+        }
+    },
+    {
+        "name": "put_auto_follow_pattern",
+        "operation": {
+          "operation-type": "raw-request",
+          "path": "/_ccr/auto_follow/my_pattern",
+          "method": "PUT",
+          "body": {
+            "remote_cluster": "local",
+            "leader_index_patterns": ["logs-*"],
+            "follow_index_pattern": "{{ p_follow_index_pattern }}"
+          }
+        }
+    },
+    {% include "tasks/index-setup.json" %},
+    {%- endif -%}{# non-serverless-index-statistics-marker-end #}
+    {
+      "name": "bulk-index",
+      "operation": {
+        "operation-type": "raw-bulk",
+        "param-source": "processed-source",
+        "time-format": "milliseconds",
+        "profile": "fixed_interval",
+        "bulk-size": {{ p_bulk_size }},
+        "detailed-results": true
+      },
+      "clients": {{ p_bulk_indexing_clients }}{% if p_throttle_indexing %},
+      "ignore-response-error-level": "{{error_level | default('non-fatal')}}",
+      "schedule": "timestamp-throttler",
+      "max-delay-secs": 1
+      {% endif %}
+    }
+  ]
+}

elastic/logs/templates/component/track-shared-logsdb-mode.json

@@ -4,6 +4,9 @@
         {% if index_mode %}
         "index": {
             "mode": {{ index_mode | tojson }}
+            {% if codec %}
+            ,"codec": {{codec | tojson}}
+            {% endif %}
             {% if source_mode %}
             ,"mapping.source.mode": {{ source_mode | tojson }}
             {% endif %}

elastic/logs/templates/composable/logs-k8-application.log.json

@@ -5,7 +5,6 @@
   "template": {
     "settings": {
       "index": {
-        "codec": "best_compression",
         "mapping": {
           "total_fields": {
             "limit": "10000"

elastic/logs/track.py

@@ -46,6 +46,16 @@
 from shared.track_processors.track_id_generator import TrackIdGenerator
 
 
+async def setup_local_remote(es, params):
+    response = await es.cluster.state()
+    master_node = response["master_node"]
+    response = await es.nodes.info()
+    ip = response["nodes"][master_node]["transport_address"]
+    p_settings = {"cluster.remote.local.seeds": ip}
+    response = await es.cluster.put_settings(persistent=p_settings)
+    return {"weight": 1, "unit": "ops"}
+
+
 def register(registry):
     registry.register_param_source("initial-indices-source", InitialIndicesParamSource)
     registry.register_param_source("add-track-path", parameter_sources.add_track_path)
@@ -88,3 +98,5 @@ def register(registry):
     registry.register_runner("configure-remote-clusters", ConfigureRemoteClusters(), async_runner=True)
     registry.register_runner("configure-ccr", ConfigureCrossClusterReplication(), async_runner=True)
     registry.register_runner("multi-cluster-wrapper", MultiClusterWrapper(), async_runner=True)
+
+    registry.register_runner("setup-local-remote", setup_local_remote, async_runner=True)