commit.

.gitignore

@@ -0,0 +1,37 @@
+# Reference https://github.com/github/gitignore/blob/master/Go.gitignore
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+vendor/
+
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+
+# OS General
+Thumbs.db
+.DS_Store
+
+# project
+*.cert
+*.key
+*.log
+bin/
+
+# Develop tools
+.vscode/
+.idea/
+*.swp
+/config.yaml
+/config.json

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 AdamTyn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,44 @@
+GOPATH:=$(shell go env GOPATH)
+VERSION=$(shell git describe --tags --always)
+CONF_PROTO_FILES=$(shell find internal/conf -name *.proto)
+
+.PHONY: init
+# get default config.json
+init:
+	awk 'BEGIN { cmd="cp -ri ./config.json.example ./config.json"; print "n" |cmd; }'
+
+.PHONY: config
+# generate config proto
+config:
+	protoc --proto_path=./internal/conf \
+ 	       --go_out=paths=source_relative:./internal/conf \
+	       $(CONF_PROTO_FILES)
+
+.PHONY: run
+# go run main.go
+run:
+	cd ./cmd && go run .
+
+.PHONY: build
+# go build main.go
+build:
+	cd ./cmd && go build -o ../bin/at-migrator-tool .
+
+# show help
+help:
+	@echo ''
+	@echo 'Usage:'
+	@echo ' make [target]'
+	@echo ''
+	@echo 'Targets:'
+	@awk '/^[a-zA-Z0-9_-]+:/ { \
+	helpMessage = match(lastLine, /^# (.*)/); \
+		if (helpMessage) { \
+			helpCommand = substr($$1, 0, index($$1, ":")-1); \
+			helpMessage = substr(lastLine, RSTART + 2, RLENGTH); \
+			printf "\033[36m%-22s\033[0m %s\n", helpCommand,helpMessage; \
+		} \
+	} \
+	{ lastLine = $$0 }' $(MAKEFILE_LIST)
+
+.DEFAULT_GOAL := help

README.md

@@ -0,0 +1,44 @@
+## at-migrator-tool
+
+1. 文件结构梳理
+
+    ```txt
+    cmd/main.go # 程序入口文件
+    doc/... # 存放子文档
+    internal/collector/... # 任务进程会用加载的采集器
+    internal/conf/... # 解析配置的proto文件
+    internal/contract/... # 契约(interface)
+    internal/entity/... # 数据库实体
+    internal/pkg/... # 其他
+    internal/process/... # 加载到app容器的任务进程
+    internal/application.go # app容器
+    config.json # 配置文件
+    config.json.example # 配置文件模板
+    Makefile # 构建指令
+    ......
+    ```
+
+2. 构建指令详解
+   - 首次生成 *config.json* 配置文件
+
+     ```bash
+     make init
+     ```
+   - 编译 *internal/conf/conf.proto* 文件
+     
+     ```bashgit
+     make config
+     ```
+   - 只运行不编译
+     
+     ```bash
+     make run
+     ```
+   - 编译打包
+     
+     ```bash
+     make build
+     ```
+
+3. 子文档
+   - [operate-record-migrator](docs/operate_record_migrator.md)

cmd/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"at-migrator-tool/internal"
+	col "at-migrator-tool/internal/collector"
+	"at-migrator-tool/internal/conf"
+	"at-migrator-tool/internal/pkg"
+	"at-migrator-tool/internal/process"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+)
+
+func main() {
+	appC := getAppConfig()
+	app := internal.NewApp(appC)
+
+	http := process.NewHttp(app)
+	app.Add(http)
+
+	pg1 := pkg.DB(appC.Data.Source) // 迁移前的数据源
+	pg2 := pkg.DB(appC.Data.Target) // 迁移后的数据库
+	rd := pkg.NewRedis(appC.Data.Redis)
+	defer func() {
+		_ = pg1.Close()
+		_ = pg2.Close()
+		_ = rd.Close()
+	}()
+
+	operateRecordMigrator := process.NewOperateRecordMigrator(app, pg1, rd)
+	operateRecordMigrator.
+		Add(col.NewCompanyOLogCollector(1000, pg2, app.Logger)).
+		Add(col.NewDeliveryOLogCollector(5000, pg2, app.Logger)).
+		Add(col.NewInternOLogCollector(5000, pg2, app.Logger)).
+		Add(col.NewBadDataCollector(500, rd, app.Logger)).
+		Add(col.NewFsRobotCollector(3, appC.Webhook, app.Logger))
+	app.Add(operateRecordMigrator).
+		Go()
+}
+
+func getAppConfig() *conf.App {
+	fb, err := os.Open("../config.json")
+	if err != nil {
+		panic(err)
+	}
+	defer fb.Close()
+	var bytes []byte
+	bytes, err = ioutil.ReadAll(fb)
+	if err != nil {
+		panic(err)
+	}
+	var appC conf.App
+	err = json.Unmarshal(bytes, &appC)
+	if err != nil {
+		panic(err)
+	}
+	return &appC
+}

cmd/main_test.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"at-migrator-tool/internal"
+	col "at-migrator-tool/internal/collector"
+	"at-migrator-tool/internal/process"
+	"testing"
+)
+
+func TestGetAppConfig(t *testing.T) {
+	appC := getAppConfig()
+	t.Log(appC)
+	t.Log(appC.Migrator.OperateRecord.FetchStep)
+	t.Log(appC.Migrator.OperateRecord.MaxEmptyFetchNum)
+	t.Log(appC.Webhook.FsRobot)
+}
+
+func TestNoProcess(t *testing.T) {
+	appC := getAppConfig()
+	app := internal.NewApp(appC)
+	app.Go()
+}
+
+func TestRunHttp(t *testing.T) {
+	appC := getAppConfig()
+	app := internal.NewApp(appC)
+	http := process.NewHttp(app)
+	app.Add(http).
+		Go()
+}
+
+func TestFsRobotCollector(t *testing.T) {
+	appC := getAppConfig()
+	app := internal.NewApp(appC)
+	c := col.NewFsRobotCollector(3, appC.Webhook, app.Logger)
+	go c.Listen()
+	m := make(map[string]interface{})
+	m["msg_type"] = "text"
+	m["content"] = map[string]string{
+		"text": "[at-migrator-tool] func TestFsRobotCollector(t *testing.T)",
+	}
+	err := c.Put(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for {
+		select {}
+	}
+}

config.json.example

@@ -0,0 +1,33 @@
+{
+  "name": "at-migrator-tool",
+  "server": {
+    "http": {
+      "endpoint": "0.0.0.0:9000",
+      "timeout": 10
+    }
+  },
+  "data": {
+    "source": {
+      "driver": "postgres",
+      "dsn": "dbname=test user=test password=test host=test.com port=1234 sslmode=disable options='-c statement_timeout=5000'"
+    },
+    "target": {
+      "driver": "postgres",
+      "dsn": "dbname=test user=test password=test host=test.com port=1234 sslmode=disable options='-c statement_timeout=5000'"
+    },
+    "redis": {
+      "addr": "test.cn:1234",
+      "password": "123456",
+      "db": 1
+    }
+  },
+  "migrator": {
+    "operate_record": {
+      "fetch_step": 10000,
+      "max_empty_fetch_num": 100
+    }
+  },
+  "webhook": {
+    "fs_robot": "https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxxxxxxx"
+  }
+}

docs/operate_record_migrator.md

@@ -0,0 +1,19 @@
+## operate-record-migrator
+
+```go
+- version v1.0
+- latest_modify 2022.06.30
+```
+
+1. 链路分析
+
+    ```bash
+    [▼] cmd/main.go
+    [▼] internal/conf/conf.pb.go # 解析配置
+    [▼] internal/collector/*.go
+    [▼] internal/process/operate_record_migrator.go
+    [▼] internal/application.go
+    ```
+
+2. 流程图
+![流程图](operate_record_migrator.流程图.png)

go.mod

@@ -0,0 +1,15 @@
+module at-migrator-tool
+
+go 1.17
+
+require (
+	github.com/go-redis/redis v6.15.9+incompatible
+	github.com/go-sql-driver/mysql v1.6.0
+	github.com/lib/pq v1.10.6
+	google.golang.org/protobuf v1.26.0
+)
+
+require (
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.19.0 // indirect
+)