elastic#104233 Allow Watcher Node Allocation Settings (elastic#115251)

* Update settings endpoint modified

Now accepts index.routing.allocation.* settings but denies changing
the allocation setting that keeps watches on data nodes

* Get settings endpoint modified

Now returns index.routing.allocation.* settings explicitly filters out
the `index.routing.allocation.include._tier_preference` setting

* Tests for modified endpoints

* Update docs

docs/reference/watcher/how-watcher-works.asciidoc

@@ -146,15 +146,18 @@ add, the more distributed the watches can be executed. If you add or remove
 replicas, all watches need to be reloaded. If a shard is relocated, the
 primary and all replicas of this particular shard will reload.
 
-Because the watches are executed on the node, where the watch shards are, you can create
-dedicated watcher nodes by using shard allocation filtering.
+Because the watches are executed on the node, where the watch shards are, you
+can create dedicated watcher nodes by using shard allocation filtering. To do this
+, configure nodes with a dedicated `node.attr.role: watcher` property.
 
-You could configure nodes with a dedicated `node.attr.role: watcher` property and
-then configure the `.watches` index like this:
+As the `.watches` index is a system index, you can't use the normal `.watcher/_settings`
+endpoint to modify its routing allocation. Instead, you can use the following dedicated
+endpoint to adjust the allocation of the `.watches` shards to the nodes with the
+`watcher` role attribute:
 
 [source,console]
 ------------------------
-PUT .watches/_settings
+PUT _watcher/settings
 {
   "index.routing.allocation.include.role": "watcher"
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/UpdateWatcherSettingsAction.java

@@ -16,13 +16,13 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.UpdateForV9;
 
 import java.io.IOException;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 public class UpdateWatcherSettingsAction extends ActionType<AcknowledgedResponse> {
 
@@ -34,6 +34,16 @@ public class UpdateWatcherSettingsAction extends ActionType<AcknowledgedResponse
         IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS
     );
 
+    public static final Set<String> ALLOWED_SETTINGS_PREFIXES = Set.of(
+        IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX,
+        IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX,
+        IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX
+    );
+
+    public static final Set<String> EXPLICITLY_DENIED_SETTINGS = Set.of(
+        IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX + "._tier_preference"
+    );
+
     public UpdateWatcherSettingsAction() {
         super(NAME);
     }
@@ -79,13 +89,25 @@ public Map<String, Object> settings() {
 
         @Override
         public ActionRequestValidationException validate() {
-            Set<String> forbiddenSettings = Sets.difference(settings.keySet(), ALLOWED_SETTING_KEYS);
-            if (forbiddenSettings.size() > 0) {
+            Set<String> forbiddenSettings = settings.keySet()
+                .stream()
+                .filter(
+                    setting -> (ALLOWED_SETTING_KEYS.contains(setting) == false
+                        && ALLOWED_SETTINGS_PREFIXES.stream().noneMatch(prefix -> setting.startsWith(prefix + ".")))
+                        || EXPLICITLY_DENIED_SETTINGS.contains(setting)
+                )
+                .collect(Collectors.toSet());
+
+            if (forbiddenSettings.isEmpty() == false) {
                 return ValidateActions.addValidationError(
                     "illegal settings: "
                         + forbiddenSettings
                         + ", these settings may not be configured. Only the following settings may be configured: "
-                        + ALLOWED_SETTING_KEYS,
+                        + ALLOWED_SETTING_KEYS
+                        + ", "
+                        + ALLOWED_SETTINGS_PREFIXES.stream().map(s -> s + ".*").collect(Collectors.toSet())
+                        + " excluding the following explicitly denied settings: "
+                        + EXPLICITLY_DENIED_SETTINGS,
                     null
                 );
             }

x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/settings_endpoints/10_watcher_settings.yml

@@ -0,0 +1,104 @@
+---
+setup:
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+  - do:
+      watcher.put_watch:
+        id: "my_watch"
+        body: >
+          {
+            "trigger": {
+              "schedule": {
+                "hourly": {
+                  "minute": [ 0, 5 ]
+                  }
+                }
+            },
+            "input": {
+              "simple": {
+                "payload": {
+                  "send": "yes"
+                }
+              }
+            },
+            "condition": {
+              "always": {}
+            },
+            "actions": {
+              "test_index": {
+                "index": {
+                  "index": "test"
+                }
+              }
+            }
+          }
+
+---
+"Test update and get watch settings api":
+  - do:
+      watcher.get_settings: { }
+
+  - match: { index.auto_expand_replicas: "0-1" }
+  - match: { index.number_of_replicas: "0" }
+
+  - do:
+      watcher.update_settings:
+        body:
+          index.auto_expand_replicas: "0-all"
+
+  - do:
+      watcher.get_settings: { }
+
+  - match: { index.auto_expand_replicas: "0-all" }
+  - is_false: index.routing.allocation.include._tier_preference
+
+  - do:
+      watcher.update_settings:
+        body:
+          index.auto_expand_replicas: null
+          index.number_of_replicas: 1
+
+  - do:
+      watcher.get_settings: { }
+
+  - match: { index.number_of_replicas: "1" }
+---
+"Test disallowed setting name throws error":
+  - requires:
+      test_runner_features: regex
+  - do:
+      watcher.update_settings:
+        body:
+          index.disallowed_setting: "some_invalid_value"
+      catch: bad_request
+  - match:
+      error:
+        type: "action_request_validation_exception"
+        reason: '/illegal settings\: \[index.disallowed_setting\].*/'
+---
+"Test allowed prefix setting name":
+  - do:
+      watcher.update_settings:
+        body:
+          index.routing.allocation.include.role: "watcher"
+          index.routing.allocation.exclude.role: "noWatcher"
+          index.routing.allocation.require.role: "mustWatcher"
+  - do:
+      watcher.get_settings: { }
+  - match: { index.routing.allocation.include.role: "watcher" }
+  - match: { index.routing.allocation.exclude.role: "noWatcher" }
+  - match: { index.routing.allocation.require.role: "mustWatcher" }
+---
+"Test explicitly disallowed prefix setting name throws error":
+  - requires:
+      test_runner_features: regex
+  - do:
+      watcher.update_settings:
+        body:
+          index.routing.allocation.include.disallowed_prefix: "some_invalid_value"
+      catch: bad_request
+  - match:
+      error:
+        type: "action_request_validation_exception"
+        reason: '/illegal settings\: \[index.routing.allocation.include.disallowed_prefix\].*/'

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportGetWatcherSettingsAction.java

@@ -23,8 +23,10 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.GetWatcherSettingsAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction;
 
+import static org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction.ALLOWED_SETTINGS_PREFIXES;
+import static org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction.ALLOWED_SETTING_KEYS;
+import static org.elasticsearch.xpack.core.watcher.transport.actions.put.UpdateWatcherSettingsAction.EXPLICITLY_DENIED_SETTINGS;
 import static org.elasticsearch.xpack.watcher.transport.actions.TransportUpdateWatcherSettingsAction.WATCHER_INDEX_NAME;
 import static org.elasticsearch.xpack.watcher.transport.actions.TransportUpdateWatcherSettingsAction.WATCHER_INDEX_REQUEST;
 
@@ -73,11 +75,14 @@ protected void masterOperation(
      */
     private static Settings filterSettableSettings(Settings settings) {
         Settings.Builder builder = Settings.builder();
-        for (String settingName : UpdateWatcherSettingsAction.ALLOWED_SETTING_KEYS) {
-            if (settings.hasValue(settingName)) {
-                builder.put(settingName, settings.get(settingName));
-            }
-        }
+        settings.keySet()
+            .stream()
+            .filter(
+                setting -> (ALLOWED_SETTING_KEYS.contains(setting)
+                    || ALLOWED_SETTINGS_PREFIXES.stream().anyMatch(prefix -> setting.startsWith(prefix + ".")))
+                    && EXPLICITLY_DENIED_SETTINGS.contains(setting) == false
+            )
+            .forEach(setting -> builder.put(setting, settings.get(setting)));
         return builder.build();
     }