Conditionally enable logsdb by default (#121049)

Enable logsdb by default if logsdb.prior_logs_usage has not been set to true. Meaning that if no data streams were created matching with the logs-- pattern in 8.x, then logsdb will be enabled by default for data streams matching with logs-*-* pattern. Also removes LogsPatternUsageService as with version 9.0 and beyond, this component is no longer necessary. Followup from #120708 Closes #106489
elastic · Jan 29, 2025 · 952bf22 · 952bf22
1 parent 2b66916
commit 952bf22
Show file tree

Hide file tree

Showing 14 changed files with 178 additions and 554 deletions.
diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json
@@ -291,6 +291,7 @@
         "JVM option",
         "Java API",
         "Logging",
+        "Logs",
         "Mapping",
         "Packaging",
         "Painless",

diff --git a/docs/changelog/121049.yaml b/docs/changelog/121049.yaml
@@ -0,0 +1,19 @@
+pr: 121049
+summary: Conditionally enable logsdb by default for data streams matching with logs-*-*
+  pattern.
+area: Logs
+type: breaking
+issues:
+ - 106489
+breaking:
+  title: Conditionally enable logsdb by default
+  area: Logs
+  details: |-
+    Logsdb will be enabled by default for data streams matching with logs-*-* pattern.
+    If upgrading from 8.x to 9.x and data streams matching with log-*-* do exist,
+    then Logsdb will not be enabled by default.
+  impact: |-
+    Logsdb reduce storage footprint in Elasticsearch for logs, but there are side effects
+    to be taken into account that are described in the Logsdb docs:
+    https://www.elastic.co/guide/en/elasticsearch/reference/current/logs-data-stream.html#upgrade-to-logsdb-notes
+  notable: true
diff --git a/docs/reference/data-streams/logs.asciidoc b/docs/reference/data-streams/logs.asciidoc
@@ -237,3 +237,9 @@ The `logsdb` index mode uses the following settings:
 * **`index.mapping.ignore_above`**: `8191`
 
 * **`index.mapping.total_fields.ignore_dynamic_beyond_limit`**: `true`
+
+[discrete]
+[[upgrade-to-logsdb-notes]]
+=== Notes about upgrading to Logsdb
+
+TODO: add notes.
diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc
@@ -177,7 +177,7 @@ Example response:
       },
       "logsdb": {
         "available": true,
-        "enabled": false
+        "enabled": true
       }
    },
    "tagline" : "You know, for X"

diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc
@@ -514,7 +514,7 @@ GET /_xpack/usage
   },
   "logsdb": {
     "available": true,
-    "enabled": false,
+    "enabled": true,
     "indices_count": 0,
     "indices_with_synthetic_source": 0,
     "num_docs": 0,

diff --git a/...t/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java b/...t/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java
@@ -120,6 +120,10 @@ protected ElasticsearchCluster getUpgradeCluster() {
         }""";
 
     public void testLogsIndexing() throws IOException {
+        assumeTrue(
+            "otherwise first backing index of logs-apache-production will be in logsdb mode",
+            getOldClusterTestVersion().before("9.0.0")
+        );
         if (isRunningAgainstOldCluster()) {
             assertOK(client().performRequest(putTemplate(client(), "logs-template", STANDARD_TEMPLATE)));
             assertOK(client().performRequest(createDataStream("logs-apache-production")));

diff --git a/...g-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsUsageRollingUpgradeIT.java b/...g-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsUsageRollingUpgradeIT.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT.bulkIndex;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+
+public class LogsUsageRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
+
+    public LogsUsageRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    public void testUsage() throws Exception {
+        assumeTrue("logsdb.prior_logs_usage only gets set in 8.x", getOldClusterTestVersion().before("9.0.0"));
+        String dataStreamName = "logs-mysql-error";
+        if (isOldCluster()) {
+            bulkIndex(dataStreamName, 4, 256, Instant.now());
+            ensureGreen(dataStreamName);
+            assertBusy(() -> {
+                var getClusterSettingsResponse = getClusterSettings();
+                Map<?, ?> persistentSettings = (Map<?, ?>) getClusterSettingsResponse.get("persistent");
+                assertThat(persistentSettings, hasEntry("logsdb.prior_logs_usage", "true"));
+            }, 2, TimeUnit.MINUTES);
+        } else {
+            String newIndex = rolloverDataStream(dataStreamName);
+            bulkIndex(dataStreamName, 4, 256, Instant.now());
+            Map<?, ?> indexResponse = (Map<?, ?>) getIndexSettings(newIndex, true).get(newIndex);
+            Map<?, ?> settings = (Map<?, ?>) indexResponse.get("settings");
+            Map<?, ?> defaults = (Map<?, ?>) indexResponse.get("defaults");
+            assertThat(settings, not(hasKey("index.mode")));
+            assertThat(defaults, hasEntry("index.mode", "standard"));
+        }
+    }
+
+    static Map<String, Object> getClusterSettings() throws IOException {
+        var request = new Request("GET", "/_cluster/settings");
+        request.addParameter("flat_settings", "true");
+        request.addParameter("include_defaults", "true");
+        var response = client().performRequest(request);
+        assertOK(response);
+        return entityAsMap(response);
+    }
+
+    static String rolloverDataStream(String dataStreamName) throws IOException {
+        var request = new Request("POST", "/" + dataStreamName + "/_rollover");
+        var response = client().performRequest(request);
+        assertOK(response);
+        var responseBody = entityAsMap(response);
+        return (String) responseBody.get("new_index");
+    }
+
+}
diff --git a/...rade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java b/...rade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsdbIndexingRollingUpgradeIT.java
@@ -22,13 +22,18 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.time.Instant;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.enableLogsdbByDefault;
 import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex;
 import static org.elasticsearch.upgrades.TsdbIT.formatInstant;
-import static org.hamcrest.Matchers.*;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class LogsdbIndexingRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
 
@@ -122,7 +127,8 @@ static void createTemplate(String dataStreamName, String id, String template) th
         assertOK(client().performRequest(putIndexTemplateRequest));
     }
 
-    static void bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception {
+    static String bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception {
+        String firstIndex = null;
         for (int i = 0; i < numRequest; i++) {
             var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk");
             StringBuilder requestBody = new StringBuilder();
@@ -155,7 +161,11 @@ static void bulkIndex(String dataStreamName, int numRequest, int numDocs, Instan
             assertOK(response);
             var responseBody = entityAsMap(response);
             assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false));
+            if (firstIndex == null) {
+                firstIndex = (String) ((Map<?, ?>) ((Map<?, ?>) ((List<?>) responseBody.get("items")).get(0)).get("create")).get("_index");
+            }
         }
+        return firstIndex;
     }
 
     void search(String dataStreamName) throws Exception {

diff --git a/...upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/NoLogsUsageRollingUpgradeIT.java b/...upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/NoLogsUsageRollingUpgradeIT.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import java.time.Instant;
+import java.util.Map;
+
+import static org.elasticsearch.upgrades.LogsUsageRollingUpgradeIT.getClusterSettings;
+import static org.elasticsearch.upgrades.LogsdbIndexingRollingUpgradeIT.bulkIndex;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+
+public class NoLogsUsageRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
+
+    public NoLogsUsageRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    public void testUsage() throws Exception {
+        String dataStreamName = "logs-mysql-error";
+        if (isOldCluster()) {
+            dataStreamName = dataStreamName.replace("logs-", "log-");
+            bulkIndex(dataStreamName, 4, 256, Instant.now());
+            ensureGreen(dataStreamName);
+        } else if (isUpgradedCluster()) {
+            String newIndex = bulkIndex(dataStreamName, 4, 256, Instant.now());
+            ensureGreen(dataStreamName);
+            Map<?, ?> indexResponse = (Map<?, ?>) getIndexSettings(newIndex, true).get(newIndex);
+            Map<?, ?> settings = (Map<?, ?>) indexResponse.get("settings");
+            assertThat(settings, hasEntry("index.mode", "logsdb"));
+            var getClusterSettingsResponse = getClusterSettings();
+            Map<?, ?> defaults = (Map<?, ?>) getClusterSettingsResponse.get("defaults");
+            Map<?, ?> persistentSettings = (Map<?, ?>) getClusterSettingsResponse.get("persistent");
+            assertThat(persistentSettings, not(hasKey("logsdb.prior_logs_usage")));
+            assertThat(defaults, hasEntry("cluster.logsdb.enabled", "true"));
+        }
+    }
+
+}
diff --git a/...src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeDisabledRestTestIT.java b/...src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeDisabledRestTestIT.java
@@ -63,14 +63,14 @@ public void setup() throws Exception {
 
     private RestClient client;
 
-    public void testLogsSettingsIndexModeDisabled() throws IOException {
+    public void testLogsSettingsIndexModeEnabledByDefault() throws IOException {
         assertOK(createDataStream(client, "logs-custom-dev"));
         final String indexMode = (String) getSetting(
             client,
             getDataStreamBackingIndex(client, "logs-custom-dev", 0),
             IndexSettings.MODE.getKey()
         );
-        assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
+        assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName()));
     }
 
     public void testTogglingLogsdb() throws IOException {
@@ -81,29 +81,21 @@ public void testTogglingLogsdb() throws IOException {
             getDataStreamBackingIndex(client, "logs-custom-dev", 0),
             IndexSettings.MODE.getKey()
         );
-        assertThat(indexModeBefore, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
-        assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "true"));
+        assertThat(indexModeBefore, equalTo(IndexMode.LOGSDB.getName()));
+        assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "false"));
         final String indexModeAfter = (String) getSetting(
             client,
             getDataStreamBackingIndex(client, "logs-custom-dev", 0),
             IndexSettings.MODE.getKey()
         );
-        assertThat(indexModeAfter, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
+        assertThat(indexModeAfter, equalTo(IndexMode.LOGSDB.getName()));
         assertOK(rolloverDataStream(client, "logs-custom-dev"));
         final String indexModeLater = (String) getSetting(
             client,
             getDataStreamBackingIndex(client, "logs-custom-dev", 1),
             IndexSettings.MODE.getKey()
         );
-        assertThat(indexModeLater, equalTo(IndexMode.LOGSDB.getName()));
-        assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "false"));
-        assertOK(rolloverDataStream(client, "logs-custom-dev"));
-        final String indexModeFinal = (String) getSetting(
-            client,
-            getDataStreamBackingIndex(client, "logs-custom-dev", 2),
-            IndexSettings.MODE.getKey()
-        );
-        assertThat(indexModeFinal, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
+        assertThat(indexModeLater, Matchers.not(equalTo(IndexMode.LOGSDB.getName())));
 
     }
 

diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
@@ -11,10 +11,8 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -42,17 +40,21 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.xpack.logsdb.LogsPatternUsageService.LOGSDB_PRIOR_LOGS_USAGE;
-import static org.elasticsearch.xpack.logsdb.LogsPatternUsageService.USAGE_CHECK_MAX_PERIOD;
 import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING;
 
 public class LogsDBPlugin extends Plugin implements ActionPlugin {
 
     private final Settings settings;
     private final SyntheticSourceLicenseService licenseService;
+    private static final Setting<Boolean> LOGSDB_PRIOR_LOGS_USAGE = Setting.boolSetting(
+        "logsdb.prior_logs_usage",
+        false,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
     public static final Setting<Boolean> CLUSTER_LOGSDB_ENABLED = Setting.boolSetting(
         "cluster.logsdb.enabled",
-        false,
+        settings -> Boolean.toString(LOGSDB_PRIOR_LOGS_USAGE.get(settings) == false),
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     );
@@ -81,18 +83,6 @@ public Collection<?> createComponents(PluginServices services) {
             logsdbIndexModeSettingsProvider::updateClusterIndexModeLogsdbEnabled
         );
 
-        var clusterService = services.clusterService();
-        Supplier<Metadata> metadataSupplier = () -> clusterService.state().metadata();
-        var historicLogsUsageService = new LogsPatternUsageService(services.client(), settings, services.threadPool(), metadataSupplier);
-        clusterService.addLocalNodeMasterListener(historicLogsUsageService);
-        clusterService.addLifecycleListener(new LifecycleListener() {
-
-            @Override
-            public void beforeStop() {
-                historicLogsUsageService.offMaster();
-            }
-        });
-
         // Nothing to share here:
         return super.createComponents(services);
     }
@@ -112,7 +102,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
 
     @Override
     public List<Setting<?>> getSettings() {
-        return List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED, USAGE_CHECK_MAX_PERIOD, LOGSDB_PRIOR_LOGS_USAGE);
+        return List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED, LOGSDB_PRIOR_LOGS_USAGE);
     }
 
     @Override