From f7fdc75b76b3c9d64b0d6919664910ca77aab9ea Mon Sep 17 00:00:00 2001 From: xiangyu5632 Date: Tue, 19 Dec 2023 10:48:02 +0800 Subject: [PATCH] chore: support migrate data to other database Signed-off-by: xiangyu5632 --- README.md | 33 +++++++++++++++++---------------- cmd/root.go | 3 ++- src/dataMigrate.go | 6 +++++- src/migrator.go | 2 +- src/options.go | 1 + 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 292d8b0..15bc0f7 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ drwx------ 4 root root 128B 12 8 09:01 db1 We migrate `internal` db ```bash -> ./dataMigrate run --from /var/lib/influxdb/data --to ip:port --database _internal +> ./dataMigrate run --from /var/lib/influxdb/data --to ip:port --database _internal --dest_database _internal 2023/12/08 14:17:48 Data migrate tool starting 2023/12/08 14:17:48 Debug mode is enabled @@ -77,7 +77,7 @@ We migrate `internal` db ### example 2: Migrate the specified database ```bash -> ./dataMigrate run --from /var/lib/influxdb/data --to ip:port --database db0 +> ./dataMigrate run --from /var/lib/influxdb/data --to ip:port --database db0 --dest_database db3 2023/12/08 14:31:47 Data migrate tool starting 2023/12/08 14:31:47 Debug mode is enabled @@ -91,7 +91,7 @@ We migrate `internal` db ### example 3: Migrate the specified database with auth and https ```bash -> ./dataMigrate run --from /var/lib/influxdb/data --to ip:port --database db0 \ +> ./dataMigrate run --from /var/lib/influxdb/data --to ip:port --database db0 --dest_database db0\ --ssl --unsafeSsl --username rwusr --password This@123 2023/12/08 14:31:47 Data migrate tool starting @@ -113,19 +113,20 @@ Usage: run [flags] Flags: - --batch int Optional: specify batch size for inserting lines (default 1000) - --database string Optional: the database to read - --debug Optional: whether to enable debug log or not - --end string Optional: the end time to read (RFC3339 format) - -f, --from string Influxdb Data storage path. See your influxdb config item: data.dir (default "/var/lib/influxdb/data") - -h, --help help for run - -p, --password string Optional: The password to connect to the openGemini cluster. - --retention string Optional: the retention policy to read (required -database) - --ssl Optional: Use https for requests. - --start string Optional: the start time to read (RFC3339 format) - -t, --to string Destination host to write data to (default "127.0.0.1:8086") - --unsafeSsl Optional: Set this when connecting to the cluster using https and not use SSL verification. - -u, --username string Optional: The username to connect to the openGemini cluster. + --batch int Optional: specify batch size for inserting lines (default 1000) + --database string The Source database to read + --dest_database string Optional: the destination database to write, default use --database + --debug Optional: whether to enable debug log or not + --end string Optional: the end time to read (RFC3339 format) + -f, --from string Influxdb Data storage path. See your influxdb config item: data.dir (default "/var/lib/influxdb/data") + -h, --help help for run + -p, --password string Optional: The password to connect to the openGemini cluster. + --retention string Optional: the retention policy to read (required -database) + --ssl Optional: Use https for requests. + --start string Optional: the start time to read (RFC3339 format) + -t, --to string Destination host to write data to (default "127.0.0.1:8086") + --unsafeSsl Optional: Set this when connecting to the cluster using https and not use SSL verification. + -u, --username string Optional: The username to connect to the openGemini cluster. ``` **Welcome to add more features.** diff --git a/cmd/root.go b/cmd/root.go index cbfc8bc..34ece57 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -29,7 +29,8 @@ func init() { RootCmd.Flags().StringVarP(&opt.Password, "password", "p", "", "Optional: The password to connect to the openGemini cluster.") RootCmd.Flags().StringVarP(&opt.DataDir, "from", "f", "/var/lib/influxdb/data", "Influxdb Data storage path. See your influxdb config item: data.dir") RootCmd.Flags().StringVarP(&opt.Out, "to", "t", "127.0.0.1:8086", "Destination host to write data to") - RootCmd.Flags().StringVarP(&opt.Database, "database", "", "", "Optional: the database to read") + RootCmd.Flags().StringVarP(&opt.Database, "database", "", "", "the database to read") + RootCmd.Flags().StringVarP(&opt.DestDatabase, "dest_database", "", "", "Optional: the database to write") RootCmd.Flags().StringVarP(&opt.RetentionPolicy, "retention", "", "", "Optional: the retention policy to read (required -database)") RootCmd.Flags().StringVarP(&opt.Start, "start", "", "", "Optional: the start time to read (RFC3339 format)") RootCmd.Flags().StringVarP(&opt.End, "end", "", "", "Optional: the end time to read (RFC3339 format)") diff --git a/src/dataMigrate.go b/src/dataMigrate.go index a781703..4d6911e 100644 --- a/src/dataMigrate.go +++ b/src/dataMigrate.go @@ -138,13 +138,14 @@ func (cmd *DataMigrateCommand) Run() error { logger.LogString("Got param \"from\": "+cmd.opt.DataDir, TOLOGFILE, LEVEL_INFO) logger.LogString("Got param \"to\": "+cmd.opt.Out, TOLOGFILE, LEVEL_INFO) logger.LogString("Got param \"database\": "+cmd.opt.Database, TOLOGFILE, LEVEL_INFO) + logger.LogString("Got param \"dest_database\": "+cmd.opt.DestDatabase, TOLOGFILE, LEVEL_INFO) logger.LogString("Got param \"retention\": "+cmd.opt.RetentionPolicy, TOLOGFILE, LEVEL_INFO) logger.LogString("Got param \"start\": "+cmd.opt.Start, TOLOGFILE, LEVEL_INFO) logger.LogString("Got param \"end\": "+cmd.opt.End, TOLOGFILE, LEVEL_INFO) logger.LogString("Got param \"batch\": "+strconv.Itoa(cmd.opt.BatchSize), TOLOGFILE, LEVEL_INFO) gs := NewGeminiService(cmd) - shardGroupDuration, err := gs.GetShardGroupDuration(cmd.opt.Database, "autogen") + shardGroupDuration, err := gs.GetShardGroupDuration(cmd.opt.DestDatabase, "autogen") if err != nil { return err } @@ -175,6 +176,9 @@ func (cmd *DataMigrateCommand) validate() error { if cmd.opt.RetentionPolicy != "" && cmd.opt.Database == "" { return fmt.Errorf("dataMigrate: must specify a db") } + if cmd.opt.DestDatabase == "" { + cmd.opt.DestDatabase = cmd.opt.Database + } if cmd.opt.StartTime != 0 && cmd.opt.EndTime != 0 && cmd.opt.EndTime < cmd.opt.StartTime { return fmt.Errorf("dataMigrate: end time before start time") } diff --git a/src/migrator.go b/src/migrator.go index 60f8378..03e30c9 100644 --- a/src/migrator.go +++ b/src/migrator.go @@ -107,7 +107,7 @@ func (m *migrator) getStat() *statInfo { func NewMigrator(cmd *DataMigrateCommand, info *shardGroupInfo) *migrator { mig := &migrator{ out: cmd.opt.Out, - database: info.db, + database: cmd.opt.DestDatabase, retentionPolicy: info.rp, startTime: cmd.opt.StartTime, endTime: cmd.opt.EndTime, diff --git a/src/options.go b/src/options.go index 920ed25..f94daca 100644 --- a/src/options.go +++ b/src/options.go @@ -6,6 +6,7 @@ type DataMigrateOptions struct { Username string Password string Database string + DestDatabase string RetentionPolicy string Start string // rfc3339 format End string // rfc3339 format