refactor: update config struct fields and add `common_config.go' (#118)

* chore: update frontend config fields

* chore: update datanode config fields and add 'common_config.go'

* chore: update metasrv config fields

pkg/dbconfig/common_config.go

@@ -0,0 +1,64 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dbconfig
+
+type HTTPOptions struct {
+	Addr             string `toml:"addr,omitempty"`
+	Timeout          string `toml:"timeout,omitempty"`
+	DisableDashboard *bool  `toml:"disable_dashboard,omitempty"`
+	BodyLimit        string `toml:"body_limit,omitempty"`
+}
+
+type HeartbeatOptions struct {
+	Interval      string `toml:"interval,omitempty"`
+	RetryInterval string `toml:"retry_interval,omitempty"`
+}
+
+type MetaClientOptions struct {
+	// Metasrv address list.
+	MetaSrvAddrs []string `toml:"metasrv_addrs,omitempty"`
+
+	// Operation timeout in milliseconds.
+	Timeout string `toml:"timeout,omitempty"`
+
+	// Connect server timeout in milliseconds.
+	ConnectTimeout string `toml:"connect_timeout,omitempty"`
+
+	// DDL operation timeout.
+	DDLTimeout string `toml:"ddl_timeout,omitempty"`
+
+	// `TCP_NODELAY` option for accepted connections.
+	TCPNoDelay *bool `toml:"tcp_nodelay,omitempty"`
+}
+
+type LoggingOptions struct {
+	Dir                 string `toml:"dir,omitempty"`
+	Level               string `toml:"level,omitempty"`
+	EnableJaegerTracing bool   `toml:"enable_jaeger_tracing,omitempty"`
+}
+
+type ProcedureConfig struct {
+	// Max retry times of procedure.
+	MaxRetryTimes int32 `toml:"max_retry_times,omitempty"`
+
+	// Initial retry delay of procedures, increases exponentially.
+	RetryDelay string `toml:"retry_delay,omitempty"`
+}
+
+type DatanodeClientOptions struct {
+	Timeout        string `toml:"timeout,omitempty"`
+	ConnectTimeout string `toml:"connect_timeout,omitempty"`
+	TcpNoDelay     bool   `toml:"tcp_nodelay,omitempty"`
+}

pkg/dbconfig/datanode_config.go

@@ -31,11 +31,13 @@ type (
 		// Node running mode.
 		Mode string `toml:"mode,omitempty"`
 
+		// The datanode identifier, should be unique.
+		NodeID *uint64 `toml:"node_id,omitempty"`
+
 		// Whether to use in-memory catalog.
 		EnableMemoryCatalog *bool `toml:"enable_memory_catalog,omitempty"`
 
-		// The datanode identifier, should be unique.
-		NodeID *uint64 `toml:"node_id,omitempty"`
+		RequireLeaseBeforeStartup *bool `toml:"require_lease_before_startup,omitempty"`
 
 		// gRPC server address.
 		RPCAddr string `toml:"rpc_addr,omitempty"`
@@ -46,19 +48,17 @@ type (
 		// The number of gRPC server worker threads.
 		RPCRuntimeSize int32 `toml:"rpc_runtime_size,omitempty"`
 
-		MetaClientOptions struct {
-			// Metasrv address list.
-			MetaSrvAddrs []string `toml:"metasrv_addrs,omitempty"`
+		// Max gRPC receiving(decoding) message size.
+		RPCMaxRecvMessageSize string `toml:"rpc_max_recv_message_size,omitempty"`
+
+		// Max gRPC sending(encoding) message size.
+		RPCMaxSendMessageSize string `toml:"rpc_max_send_message_size,omitempty"`
 
-			// Operation timeout in milliseconds.
-			TimeoutMillis int32 `toml:"timeout_millis,omitempty"`
+		HeartbeatOptions HeartbeatOptions `toml:"heartbeat,omitempty"`
 
-			// Connect server timeout in milliseconds.
-			ConnectTimeoutMillis int32 `toml:"connect_timeout_millis,omitempty"`
+		HTTPOptions HTTPOptions `toml:"http,omitempty"`
 
-			// `TCP_NODELAY` option for accepted connections.
-			TCPNoDelay *bool `toml:"tcp_nodelay,omitempty"`
-		} `toml:"meta_client_options,omitempty"`
+		MetaClientOptions MetaClientOptions `toml:"meta_client,omitempty"`
 
 		Wal struct {
 			Dir            string `toml:"dir,omitempty"`
@@ -70,9 +70,11 @@ type (
 		} `toml:"wal,omitempty"`
 
 		Storage struct {
+			DataHome  string `toml:"data_home,omitempty"`
+			GlobalTTL string `toml:"global_ttl,omitempty"`
+
 			// Storage options.
 			Type            string `toml:"type,omitempty"`
-			DataHome        string `toml:"data_home,omitempty"`
 			Bucket          string `toml:"bucket,omitempty"`
 			Root            string `toml:"root,omitempty"`
 			AccessKeyID     string `toml:"access_key_id,omitempty"`
@@ -83,6 +85,20 @@ type (
 			CachePath       string `toml:"cache_path,omitempty"`
 			CacheCapacity   string `toml:"cache_capacity,omitempty"`
 
+			Compaction struct {
+				// Max task number that can concurrently run.
+				MaxInflightTasks int32 `toml:"max_inflight_tasks,omitempty"`
+
+				// Max files in level 0 to trigger compaction.
+				MaxFilesInLevel0 int32 `toml:"max_files_in_level0,omitempty"`
+
+				// Max task number for SST purge task after compaction.
+				MaxPurgeTasks int32 `toml:"max_purge_tasks,omitempty"`
+
+				// Buffer threshold while writing SST files.
+				SSTWriteBufferSize string `toml:"sst_write_buffer_size,omitempty"`
+			} `toml:"compaction,omitempty"`
+
 			// Storage manifest options.
 			Manifest struct {
 				// Region checkpoint actions margin.
@@ -92,7 +108,7 @@ type (
 				GCDuration string `toml:"gc_duration,omitempty"`
 
 				// Whether to try creating a manifest checkpoint on region opening.
-				CheckpointOnStartup *bool `toml:"checkpoint_on_startup,omitempty"`
+				Compress *bool `toml:"compress,omitempty"`
 			} `toml:"manifest,omitempty"`
 
 			// Storage flush options.
@@ -114,15 +130,25 @@ type (
 			} `toml:"flush,omitempty"`
 		} `toml:"storage,omitempty"`
 
-		Procedure struct {
-			MaxRetryTimes int32  `toml:"max_retry_times,omitempty"`
-			RetryDelay    string `toml:"retry_delay,omitempty"`
-		} `toml:"procedure,omitempty"`
-
-		Logging struct {
-			Dir   string `toml:"dir,omitempty"`
-			Level string `toml:"level,omitempty"`
-		} `toml:"logging,omitempty"`
+		RegionEngine []struct {
+			MitoConfig struct {
+				NumWorkers                  int    `toml:"num_workers"`
+				WorkerChannelSize           int    `toml:"worker_channel_size"`
+				WorkerRequestBatchSize      int    `toml:"worker_request_batch_size"`
+				ManifestCheckpointDistance  int    `toml:"manifest_checkpoint_distance"`
+				ManifestCompressType        string `toml:"manifest_compress_type"`
+				MaxBackgroundJobs           int    `toml:"max_background_jobs"`
+				AutoFlushInterval           string `toml:"auto_flush_interval"`
+				GlobalWriteBufferSize       string `toml:"global_write_buffer_size"`
+				GlobalWriteBufferRejectSize string `toml:"global_write_buffer_reject_size"`
+				SstMetaCacheSize            string `toml:"sst_meta_cache_size"`
+				VectorCacheSize             string `toml:"vector_cache_size"`
+			} `toml:"mito,omitempty"`
+		} `toml:"region_engine,omitempty"`
+
+		LoggingOptions LoggingOptions `toml:"logging,omitempty"`
+
+		EnableTelemetry *bool `toml:"enable_telemetry,omitempty"`
 	}
 )
 

pkg/dbconfig/dbconfig_test.go

@@ -135,12 +135,12 @@ level = 'error'
 
 	frontendCfg := cfg.(*FrontendConfig)
 	frontendExtraCfg := extraCfg.(*FrontendConfig)
-	if !reflect.DeepEqual(frontendCfg.Logging.Level, frontendExtraCfg.Logging.Level) {
-		t.Errorf("logging.level is not equal: want %s, got %s", frontendExtraCfg.Logging.Level, frontendCfg.Logging.Level)
+	if !reflect.DeepEqual(frontendCfg.LoggingOptions.Level, frontendExtraCfg.LoggingOptions.Level) {
+		t.Errorf("logging.level is not equal: want %s, got %s", frontendExtraCfg.LoggingOptions.Level, frontendCfg.LoggingOptions.Level)
 	}
 
-	if !reflect.DeepEqual(frontendCfg.Logging.Dir, frontendExtraCfg.Logging.Dir) {
-		t.Errorf("logging.dir is not equal: want %s, got %s", frontendExtraCfg.Logging.Dir, frontendCfg.Logging.Dir)
+	if !reflect.DeepEqual(frontendCfg.LoggingOptions.Dir, frontendExtraCfg.LoggingOptions.Dir) {
+		t.Errorf("logging.dir is not equal: want %s, got %s", frontendExtraCfg.LoggingOptions.Dir, frontendCfg.LoggingOptions.Dir)
 	}
 }
 

pkg/dbconfig/frontend_config.go

@@ -26,15 +26,16 @@ type (
 		// Node running mode.
 		Mode string `toml:"mode,omitempty"`
 
-		HTTPOptions struct {
-			Addr    string `toml:"addr,omitempty"`
-			Timeout string `toml:"timeout,omitempty"`
-		} `toml:"http_options,omitempty"`
+		NodeID string `toml:"node_id,omitempty"`
+
+		HeartbeatOptions HeartbeatOptions `toml:"heartbeat,omitempty"`
+
+		HTTPOptions HTTPOptions `toml:"http,omitempty"`
 
 		GRPCOptions struct {
 			Addr        string `toml:"addr,omitempty"`
 			RuntimeSize int32  `toml:"runtime_size,omitempty"`
-		} `toml:"grpc_options,omitempty"`
+		} `toml:"grpc,omitempty"`
 
 		// MySQL server options.
 		MySQLOptions struct {
@@ -47,7 +48,7 @@ type (
 				CertPath string `toml:"cert_path,omitempty"`
 				KeyPath  string `toml:"key_path,omitempty"`
 			} `toml:"tls,omitempty"`
-		} `toml:"mysql_options,omitempty"`
+		} `toml:"mysql,omitempty"`
 
 		// Postgres server options.
 		PostgresOptions struct {
@@ -60,43 +61,34 @@ type (
 				CertPath string `toml:"cert_path,omitempty"`
 				KeyPath  string `toml:"key_path,omitempty"`
 			} `toml:"tls,omitempty"`
-		} `toml:"postgres_options,omitempty"`
+		} `toml:"postgres,omitempty"`
 
 		OpenTSDBOptions struct {
 			Addr        string `toml:"addr,omitempty"`
 			RuntimeSize int32  `toml:"runtime_size,omitempty"`
-		} `toml:"opentsdb_options,omitempty"`
+		} `toml:"opentsdb,omitempty"`
 
 		InfluxDBOptions struct {
 			Enable *bool `toml:"enable,omitempty"`
-		} `toml:"influxdb_options,omitempty"`
+		} `toml:"influxdb,omitempty"`
 
 		PromStoreOptions struct {
 			Enable *bool `toml:"enable,omitempty"`
-		} `toml:"prom_store_options,omitempty"`
+		} `toml:"prom_store,omitempty"`
 
-		PrometheusOptions struct {
-			Addr string `toml:"addr,omitempty"`
-		} `toml:"prometheus_options,omitempty"`
-
-		MetaClientOptions struct {
-			// Metasrv address list.
-			MetaSrvAddrs []string `toml:"metasrv_addrs,omitempty"`
+		OLTPOptions struct {
+			Enable *bool `toml:"enable,omitempty"`
+		} `toml:"oltp,omitempty"`
 
-			// Operation timeout in milliseconds.
-			TimeoutMillis int32 `toml:"timeout_millis,omitempty"`
+		MetaClientOptions MetaClientOptions `toml:"meta_client,omitempty"`
 
-			// Connect server timeout in milliseconds.
-			ConnectTimeoutMillis int32 `toml:"connect_timeout_millis,omitempty"`
+		LoggingOptions LoggingOptions `toml:"logging,omitempty"`
 
-			// `TCP_NODELAY` option for accepted connections.
-			TCPNoDelay *bool `toml:"tcp_nodelay,omitempty"`
-		} `toml:"meta_client_options,omitempty"`
+		DatanodeOptions struct {
+			DatanodeClientOptions DatanodeClientOptions `toml:"client,omitempty"`
+		} `toml:"datanode,omitempty"`
 
-		Logging struct {
-			Dir   string `toml:"dir,omitempty"`
-			Level string `toml:"level,omitempty"`
-		} `toml:"logging,omitempty"`
+		UserProvider string `toml:"user_provider,omitempty"`
 	}
 )
 

pkg/dbconfig/metasrv_config.go

@@ -31,9 +31,6 @@ type MetasrvConfig struct {
 	// Etcd server address.
 	StoreAddr string `toml:"store_addr,omitempty"`
 
-	// Datanode lease in seconds.
-	DatanodeLeaseSec int32 `toml:"datanode_lease_sec,omitempty"`
-
 	// Datanode selector type, can be "LeaseBased" or "LoadBased".
 	Selector string `toml:"selector,omitempty"`
 
@@ -43,10 +40,19 @@ type MetasrvConfig struct {
 	// Enable region failover.
 	EnableRegionFailover *bool `toml:"enable_region_failover"`
 
-	Logging struct {
-		Dir   string `toml:"dir,omitempty"`
-		Level string `toml:"level,omitempty"`
-	} `toml:"logging,omitempty"`
+	HTTPOptions HTTPOptions `toml:"http,omitempty"`
+
+	LoggingOptions LoggingOptions `toml:"logging,omitempty"`
+
+	ProcedureConfig ProcedureConfig `toml:"procedure,omitempty"`
+
+	DatanodeOptions struct {
+		DatanodeClientOptions DatanodeClientOptions `toml:"client,omitempty"`
+	} `toml:"datanode,omitempty"`
+
+	EnableTelemetry *bool `toml:"enable_telemetry,omitempty"`
+
+	DataHome string `toml:"data_home,omitempty"`
 }
 
 // ConfigureByCluster configures the metasrv config by the given cluster.

pkg/dbconfig/testdata/datanode.toml

@@ -1,31 +1,38 @@
 mode = 'distributed'
-enable_memory_catalog = false
 node_id = 42
+require_lease_before_startup = false
 rpc_addr = '127.0.0.1:3001'
 rpc_hostname = '127.0.0.1'
 rpc_runtime_size = 8
 
-[meta_client_options]
+[heartbeat]
+interval = '3s'
+
+[meta_client]
 metasrv_addrs = ['127.0.0.1:3002']
-timeout_millis = 3000
-connect_timeout_millis = 5000
+timeout = '3s'
+connect_timeout = '1s'
 tcp_nodelay = true
 
 [wal]
-file_size = '1GB'
-purge_threshold = '50GB'
+file_size = '256MB'
+purge_threshold = '4GB'
 purge_interval = '10m'
 read_batch_size = 128
 sync_write = false
 
 [storage]
-type = 'File'
 data_home = '/tmp/greptimedb/'
+type = 'File'
+
+[storage.compaction]
+max_inflight_tasks = 4
+max_files_in_level0 = 8
+max_purge_tasks = 32
 
 [storage.manifest]
 checkpoint_margin = 10
-gc_duration = '30s'
-checkpoint_on_startup = false
+gc_duration = '10m'
 
 [storage.flush]
 max_flush_tasks = 8
@@ -34,10 +41,16 @@ picker_schedule_interval = '5m'
 auto_flush_interval = '1h'
 global_write_buffer_size = '1GB'
 
-[procedure]
-max_retry_times = 3
-retry_delay = '500ms'
-
-[logging]
-dir = '/tmp/greptimedb/logs'
-level = 'info'
+[[region_engine]]
+[region_engine.mito]
+num_workers = 8
+worker_channel_size = 128
+worker_request_batch_size = 64
+manifest_checkpoint_distance = 10
+manifest_compress_type = 'Uncompressed'
+max_background_jobs = 4
+auto_flush_interval = '1h'
+global_write_buffer_size = '1GB'
+global_write_buffer_reject_size = '2GB'
+sst_meta_cache_size = '128MB'
+vector_cache_size = '512MB'