[reconfigurator] clickhouse-admin endpoints to generate ClickHouse co…

…nfig files (#6476)

## Overview

This commit introduces two new endpoints to the `clickhouse-admin` API.
One to generate a ClickHouse server XML configuration file and the other
to generate a ClickHouse keeper XML configuration file.

## Purpose

"Reconfigurator" will need to call these endpoints to generate the
necessary files when bringing up new `clickhouse_server` and
`clickhouse_keeper` zones. They will also be necessary to update
existing zones with the keeper and server information from new zones.

## API structure

Moving forward the API endpoints will follow the following structure:

- `/keeper/` : For endpoints that will perform actions on keeper nodes
solely.
- `/server/` : For endpoints that will perform actions on server nodes
solely.

## Usage

To generate a server XML config file:

```console
$ curl -X put http://example/server/config \ 
-H "Content-Type: application/json" \
-d '{"generation": {GENERATION_NUM}, settings: {"id": {ID}, "keepers": [{"ipv6|ipv4|domain": "{ADDRESS}"}], "remote_servers": [{"ipv6|ipv4|domain": "{ADDRESS}"}], "config_dir": "{CONFIG_PATH}", "datastore_path": "{DATA_PATH}", "listen_addr": "{LISTEN_ADDRESS}" }}'
```

To generate a keeper XML config file:

```console
$ curl -X put http://example/keeper/config \ 
-H "Content-Type: application/json" \
-d '{"generation": {GENERATION_NUM}, settings: {"node_id": {ID}, "raft_servers": [{"id": {NODE_ID}, "host": {"ipv6|ipv4|domain": "{ADDRESS}"}}], "config_dir": "{CONFIG_PATH}", "datastore_path": "{DATA_PATH}", "listen_addr": "{LISTEN_ADDRESS}"}}'
```

## Caveats

For the time being the generation number isn't used for anything. In a
follow up PR we will use it to keep track of the configuration version
to make sure we're not generating an incorrect XML configuration file.

## Testing

Dropshot server:

```console
$ cargo run --bin=clickhouse-admin -- run -c ./smf/clickhouse-admin/config.toml -a [::1]:8888
   Compiling omicron-clickhouse-admin v0.1.0 (/Users/karcar/src/omicron/clickhouse-admin)
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 2.52s
     Running `target/debug/clickhouse-admin run -c ./smf/clickhouse-admin/config.toml -a '[::1]:8888'`
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:32:00.106783Z","hostname":"ixchel","pid":11318,"local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"}
{"msg":"accepted connection","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:32:17.626437Z","hostname":"ixchel","pid":11318,"local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[::1]:51896"}
Path { inner: GenerationNum { generation: Generation(123) } }
{"msg":"request completed","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:32:17.628323Z","hostname":"ixchel","pid":11318,"uri":"/node/server/generate-config/123","method":"post","req_id":"0b623f6b-441c-4418-9189-99504fbea5bf","remote_addr":"[::1]:51896","local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1044,"response_code":"201"}
{"msg":"accepted connection","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:34:07.829199Z","hostname":"ixchel","pid":11318,"local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[::1]:52117"}
Path { inner: GenerationNum { generation: Generation(123) } }
{"msg":"request completed","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:34:07.830355Z","hostname":"ixchel","pid":11318,"uri":"/node/keeper/generate-config/123","method":"post","req_id":"1c729e0c-04bf-4e75-af0a-b23eef40f20f","remote_addr":"[::1]:52117","local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":985,"response_code":"201"}

```

Generate server XML file:

```console
$ curl -X put http://[::1]:8888/server/config -H "Content-Type: application/json" -d '{"generation": 3, "settings": {"id": 45, "keepers": [{"ipv6": "ff::01"}, {"ipv4": "127.0.0.1"}, {"domain_name": "hi.there"}], "remote_servers": [{"ipv6": "ff::08"}, {"ipv4": "127.0.0.2"}], "config_dir": "./", "datastore_path": "./", "listen_addr": "::1" }}' | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   858  100   602  100   256   210k  91789 --:--:-- --:--:-- --:--:--  418k
{
  "logger": {
    "level": "trace",
    "log": "./log/clickhouse.log",
    "errorlog": "./log/clickhouse.err.log",
    "size": 100,
    "count": 1
  },
  "macros": {
    "shard": 1,
    "replica": 45,
    "cluster": "oximeter_cluster"
  },
  "listen_host": "::1",
  "http_port": 8123,
  "tcp_port": 9000,
  "interserver_http_port": 9009,
  "remote_servers": {
    "cluster": "oximeter_cluster",
    "secret": "some-unique-value",
    "replicas": [
      {
        "host": {
          "ipv6": "ff::8"
        },
        "port": 9000
      },
      {
        "host": {
          "ipv4": "127.0.0.2"
        },
        "port": 9000
      }
    ]
  },
  "keepers": {
    "nodes": [
      {
        "host": {
          "ipv6": "ff::1"
        },
        "port": 9181
      },
      {
        "host": {
          "ipv4": "127.0.0.1"
        },
        "port": 9181
      },
      {
        "host": {
          "domain_name": "hi.there"
        },
        "port": 9181
      }
    ]
  },
  "data_path": "./data"
}
```

Generate keeper XML file:

```console
$ curl -X put http://[::1]:8888/keeper/config -H "Content-Type: application/json" -d '{"generation": 3, "settings": {"id": 1, "raft_servers": [{"id": 1, "host": {"ipv6": "ff::01"}}, {"id": 2, "host":{"ipv4": "127.0.0.1"}}, {"id": 3, "host":{"domain_name": "hi.there"}}], "config_dir": "./", "datastore_path": "./", "listen_addr": "::1" }}' | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   820  100   568  100   252   301k   133k --:--:-- --:--:-- --:--:--  800k
{
  "logger": {
    "level": "trace",
    "log": "./log/clickhouse-keeper.log",
    "errorlog": "./log/clickhouse-keeper.err.log",
    "size": 100,
    "count": 1
  },
  "listen_host": "::1",
  "tcp_port": 9181,
  "server_id": 1,
  "log_storage_path": "./coordination/log",
  "snapshot_storage_path": "./coordination/snapshots",
  "coordination_settings": {
    "operation_timeout_ms": 10000,
    "session_timeout_ms": 30000,
    "raft_logs_level": "trace"
  },
  "raft_config": {
    "servers": [
      {
        "id": 1,
        "hostname": {
          "ipv6": "ff::1"
        },
        "port": 9234
      },
      {
        "id": 2,
        "hostname": {
          "ipv4": "127.0.0.1"
        },
        "port": 9234
      },
      {
        "id": 3,
        "hostname": {
          "domain_name": "hi.there"
        },
        "port": 9234
      }
    ]
  }
}
```


Related: #5999 , #3824

clickhouse-admin/api/Cargo.toml

@@ -8,6 +8,7 @@ license = "MPL-2.0"
 workspace = true
 
 [dependencies]
+clickhouse-admin-types.workspace = true
 dropshot.workspace = true
 omicron-common.workspace = true
 omicron-uuid-kinds.workspace = true

clickhouse-admin/api/src/lib.rs

@@ -2,27 +2,52 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use dropshot::{HttpError, HttpResponseOk, RequestContext};
+use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
+use clickhouse_admin_types::{KeeperSettings, ServerSettings};
+use dropshot::{HttpError, HttpResponseCreated, RequestContext, TypedBody};
+use omicron_common::api::external::Generation;
 use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-use std::net::SocketAddrV6;
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize, JsonSchema)]
+pub struct ServerConfigurableSettings {
+    /// A unique identifier for the configuration generation.
+    pub generation: Generation,
+    /// Configurable settings for a ClickHouse replica server node.
+    pub settings: ServerSettings,
+}
+
+#[derive(Debug, Deserialize, JsonSchema)]
+pub struct KeeperConfigurableSettings {
+    /// A unique identifier for the configuration generation.
+    pub generation: Generation,
+    /// Configurable settings for a ClickHouse keeper node.
+    pub settings: KeeperSettings,
+}
 
 #[dropshot::api_description]
 pub trait ClickhouseAdminApi {
     type Context;
 
-    /// Retrieve the address the ClickHouse server or keeper node is listening on
+    /// Generate a ClickHouse configuration file for a server node on a specified
+    /// directory.
     #[endpoint {
-        method = GET,
-        path = "/node/address",
+        method = PUT,
+        path = "/server/config",
     }]
-    async fn clickhouse_address(
+    async fn generate_server_config(
         rqctx: RequestContext<Self::Context>,
-    ) -> Result<HttpResponseOk<ClickhouseAddress>, HttpError>;
-}
+        body: TypedBody<ServerConfigurableSettings>,
+    ) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError>;
 
-#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
-#[serde(rename_all = "snake_case")]
-pub struct ClickhouseAddress {
-    pub clickhouse_address: SocketAddrV6,
+    /// Generate a ClickHouse configuration file for a keeper node on a specified
+    /// directory.
+    #[endpoint {
+        method = PUT,
+        path = "/keeper/config",
+    }]
+    async fn generate_keeper_config(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<KeeperConfigurableSettings>,
+    ) -> Result<HttpResponseCreated<KeeperConfig>, HttpError>;
 }

clickhouse-admin/src/bin/clickhouse-admin.rs

@@ -20,14 +20,8 @@ use std::net::{SocketAddr, SocketAddrV6};
 enum Args {
     /// Start the ClickHouse admin server
     Run {
-        // TODO: This address is solely for testing now. We should remove it
-        // once we have more endpoints up and running.
-        /// Socket address for a running clickhouse server or keeper instance
-        #[clap(long, short = 'a', action)]
-        clickhouse_address: SocketAddrV6,
-
         /// Address on which this server should run
-        #[clap(long, short = 'H', action)]
+        #[clap(long, short = 'a', action)]
         http_address: SocketAddrV6,
 
         /// Path to the server configuration file
@@ -47,12 +41,12 @@ async fn main_impl() -> Result<(), CmdError> {
     let args = Args::parse();
 
     match args {
-        Args::Run { clickhouse_address, http_address, config } => {
+        Args::Run { http_address, config } => {
             let mut config = Config::from_file(&config)
                 .map_err(|err| CmdError::Failure(anyhow!(err)))?;
             config.dropshot.bind_address = SocketAddr::V6(http_address);
 
-            let clickward = Clickward::new(clickhouse_address);
+            let clickward = Clickward::new();
 
             let server =
                 omicron_clickhouse_admin::start_server(clickward, config)

clickhouse-admin/src/clickward.rs

@@ -2,18 +2,17 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use clickhouse_admin_api::ClickhouseAddress;
+use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
+use clickhouse_admin_types::{KeeperSettings, ServerSettings};
 use dropshot::HttpError;
 use slog_error_chain::{InlineErrorChain, SlogInlineError};
-use std::io;
-use std::net::SocketAddrV6;
 
 #[derive(Debug, thiserror::Error, SlogInlineError)]
 pub enum ClickwardError {
-    #[error("clickward failure")]
+    #[error("clickward XML generation failure")]
     Failure {
         #[source]
-        err: io::Error,
+        err: anyhow::Error,
     },
 }
 
@@ -34,18 +33,32 @@ impl From<ClickwardError> for HttpError {
 }
 
 #[derive(Debug)]
-pub struct Clickward {
-    clickhouse_address: SocketAddrV6,
-}
+pub struct Clickward {}
 
 impl Clickward {
-    pub fn new(clickhouse_address: SocketAddrV6) -> Self {
-        Self { clickhouse_address }
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    pub fn generate_server_config(
+        &self,
+        settings: ServerSettings,
+    ) -> Result<ReplicaConfig, ClickwardError> {
+        let replica_config = settings
+            .generate_xml_file()
+            .map_err(|e| ClickwardError::Failure { err: e })?;
+
+        Ok(replica_config)
     }
 
-    pub fn clickhouse_address(
+    pub fn generate_keeper_config(
         &self,
-    ) -> Result<ClickhouseAddress, ClickwardError> {
-        Ok(ClickhouseAddress { clickhouse_address: self.clickhouse_address })
+        settings: KeeperSettings,
+    ) -> Result<KeeperConfig, ClickwardError> {
+        let keeper_config = settings
+            .generate_xml_file()
+            .map_err(|e| ClickwardError::Failure { err: e })?;
+
+        Ok(keeper_config)
     }
 }

clickhouse-admin/src/http_entrypoints.rs

@@ -4,9 +4,8 @@
 
 use crate::context::ServerContext;
 use clickhouse_admin_api::*;
-use dropshot::HttpError;
-use dropshot::HttpResponseOk;
-use dropshot::RequestContext;
+use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
+use dropshot::{HttpError, HttpResponseCreated, RequestContext, TypedBody};
 use std::sync::Arc;
 
 type ClickhouseApiDescription = dropshot::ApiDescription<Arc<ServerContext>>;
@@ -21,11 +20,28 @@ enum ClickhouseAdminImpl {}
 impl ClickhouseAdminApi for ClickhouseAdminImpl {
     type Context = Arc<ServerContext>;
 
-    async fn clickhouse_address(
+    async fn generate_server_config(
         rqctx: RequestContext<Self::Context>,
-    ) -> Result<HttpResponseOk<ClickhouseAddress>, HttpError> {
+        body: TypedBody<ServerConfigurableSettings>,
+    ) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError> {
         let ctx = rqctx.context();
-        let output = ctx.clickward().clickhouse_address()?;
-        Ok(HttpResponseOk(output))
+        let replica_server = body.into_inner();
+        // TODO(https://github.com/oxidecomputer/omicron/issues/5999): Do something
+        // with the generation number `replica_server.generation`
+        let output =
+            ctx.clickward().generate_server_config(replica_server.settings)?;
+        Ok(HttpResponseCreated(output))
+    }
+
+    async fn generate_keeper_config(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<KeeperConfigurableSettings>,
+    ) -> Result<HttpResponseCreated<KeeperConfig>, HttpError> {
+        let ctx = rqctx.context();
+        let keeper = body.into_inner();
+        // TODO(https://github.com/oxidecomputer/omicron/issues/5999): Do something
+        // with the generation number `keeper.generation`
+        let output = ctx.clickward().generate_keeper_config(keeper.settings)?;
+        Ok(HttpResponseCreated(output))
     }
 }