diff --git a/.github/ISSUE_TEMPLATE/issue.md b/.github/ISSUE_TEMPLATE/issue.md index 93ba91cf..649d22ff 100644 --- a/.github/ISSUE_TEMPLATE/issue.md +++ b/.github/ISSUE_TEMPLATE/issue.md @@ -7,7 +7,7 @@ assignees: '' --- -- [ ] 请确保已经看过 wiki:https://github.com/alibaba/RedisShake/wiki +- [ ] 请确保已经看过 wiki:https://RedisShake/wiki - [ ] 请确保已经学习过 Markdown 语法,良好的排版有助于维护人员了解你的问题 - [ ] 请在此提供足够的信息供社区维护人员排查问题 - [ ] 请在提交 issue 前删除此模板中多余的文字,包括这几句话 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aa44ff4..4ba9c3e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,13 @@ name: CI -on: [ pull_request ] +on: [ push, pull_request ] jobs: black-box-test: runs-on: ubuntu-latest strategy: matrix: - redis-version: [ 5, 6, 7 ] + redis-version: [ "2.8", "3.0", "4.0", "5.0", "6.0", "7.0" ] steps: - name: Git checkout uses: actions/checkout@v2 @@ -22,7 +22,7 @@ jobs: sudo apt-get install git git clone https://github.com/redis/redis cd redis - git checkout ${{ matrix.redis-version }}.0 + git checkout ${{ matrix.redis-version }} make -j mkdir bin cp src/redis-server bin/redis-server @@ -31,7 +31,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: make redis-shake run: | @@ -39,6 +39,5 @@ jobs: - name: test run: | - cd test - pip3 install -r requirements.txt - python3 main.py \ No newline at end of file + pip3 install -r tests/requirements.txt + sh test.sh \ No newline at end of file diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 00000000..9cfcdaaf --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,33 @@ +name: Pages +on: + workflow_dispatch: { } + push: + branches: + - main +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: 16 + cache: npm + - run: npm ci + - name: Build + run: npm run docs:build + - uses: actions/configure-pages@v2 + - uses: actions/upload-pages-artifact@v1 + with: + path: docs/.vitepress/dist + - name: Deploy + id: deployment + uses: actions/deploy-pages@v1 \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e8f933fd..a0a37ed5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,3 +66,23 @@ jobs: asset_content_type: application/gzip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "Upload release windows-amd64" + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ steps.release.outputs.upload_url }} + asset_path: ./bin/redis-shake-windows-amd64.tar.gz + asset_name: redis-shake-windows-amd64.tar.gz + asset_content_type: application/gzip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "Upload release windows-arm64" + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ steps.release.outputs.upload_url }} + asset_path: ./bin/redis-shake-windows-arm64.tar.gz + asset_name: redis-shake-windows-arm64.tar.gz + asset_content_type: application/gzip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 61d3c65e..b83fe407 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ -.idea -data -__pycache__ -bin -.DS_Store +# system +.idea/ +__pycache__/ +.DS_Store/ + +# compiled output or test output +bin/ +dist/ +tmp/ *.log *.rdb *.aof diff --git a/README.md b/README.md index ee713ad4..23fa1048 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # redis-shake -[![CI](https://github.com/alibaba/RedisShake/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/alibaba/RedisShake/actions/workflows/ci.yml) +[![CI](https://RedisShake/actions/workflows/ci.yml/badge.svg?branch=v3)](https://RedisShake/actions/workflows/ci.yml) -- [中文文档](https://github.com/alibaba/RedisShake/wiki) +- [中文文档](https://RedisShake/wiki) redis-shake is a tool for Redis data migration and data filtering. @@ -16,7 +16,7 @@ redis-shake is a tool for Redis data migration and data filtering. * ☁️ Support Aliyun Redis and ElastiCache For older versions of redis-shake (support codis, twemproxy) please -visit [here](https://github.com/alibaba/RedisShake/tree/develop). +visit [here](https://RedisShake/tree/develop). ![redis-shake2.PNG](https://s2.loli.net/2022/07/10/OZrSGutknlI8XNp.png) @@ -28,14 +28,14 @@ visit [here](https://github.com/alibaba/RedisShake/tree/develop). ### Binary package -Download from Release: [https://github.com/alibaba/RedisShake/releases](https://github.com/alibaba/RedisShake/releases) +Download from Release: [https://RedisShake/releases](https://RedisShake/releases) ### Compile from source After downloading the source code, run the `sh build.sh` command to compile. ```shell -git clone https://github.com/alibaba/RedisShake +git clone https://RedisShake cd RedisShake sh build.sh ``` diff --git a/build.sh b/build.sh index c4e1378e..d4132c12 100755 --- a/build.sh +++ b/build.sh @@ -7,11 +7,7 @@ BIN_DIR=$(pwd)/bin/ rm -rf "$BIN_DIR" mkdir -p "$BIN_DIR" -cp sync.toml "$BIN_DIR" -cp scan.toml "$BIN_DIR" -cp restore.toml "$BIN_DIR" -cp -r filters "$BIN_DIR" -cp -r scripts/cluster_helper "$BIN_DIR" +cp -r configs/* "$BIN_DIR" dist() { echo "try build GOOS=$1 GOARCH=$2" @@ -30,7 +26,7 @@ dist() { if [ "$1" == "dist" ]; then echo "[ DIST ]" - for g in "linux" "darwin"; do + for g in "linux" "darwin" "windows"; do for a in "amd64" "arm64"; do dist "$g" "$a" done diff --git a/cmd/redis-shake/main.go b/cmd/redis-shake/main.go index 1f16b5a1..b39d1c1c 100644 --- a/cmd/redis-shake/main.go +++ b/cmd/redis-shake/main.go @@ -1,120 +1,124 @@ package main import ( - "fmt" - "github.com/alibaba/RedisShake/internal/commands" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/filter" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/reader" - "github.com/alibaba/RedisShake/internal/statistics" - "github.com/alibaba/RedisShake/internal/writer" - "net/http" + "RedisShake/internal/commands" + "RedisShake/internal/config" + "RedisShake/internal/log" + "RedisShake/internal/reader" + "RedisShake/internal/status" + "RedisShake/internal/transform" + "RedisShake/internal/utils" + "RedisShake/internal/writer" + "github.com/mcuadros/go-defaults" _ "net/http/pprof" - "os" - "runtime" ) func main() { - if len(os.Args) < 2 || len(os.Args) > 3 { - fmt.Println("Usage: redis-shake ") - fmt.Println("Example: redis-shake config.toml filter.lua") - os.Exit(1) - } - - // load filter file - if len(os.Args) == 3 { - luaFile := os.Args[2] - filter.LoadFromFile(luaFile) - } - - // load config - configFile := os.Args[1] - config.LoadFromFile(configFile) + v := config.LoadConfig() - log.Init() - log.Infof("GOOS: %s, GOARCH: %s", runtime.GOOS, runtime.GOARCH) - log.Infof("Ncpu: %d, GOMAXPROCS: %d", config.Config.Advanced.Ncpu, runtime.GOMAXPROCS(0)) - log.Infof("pid: %d", os.Getpid()) - log.Infof("pprof_port: %d", config.Config.Advanced.PprofPort) - if len(os.Args) == 2 { - log.Infof("No lua file specified, will not filter any cmd.") - } - - // start pprof - if config.Config.Advanced.PprofPort != 0 { - go func() { - err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Config.Advanced.PprofPort), nil) - if err != nil { - log.PanicError(err) - } - }() - } + log.Init(config.Opt.Advanced.LogLevel, config.Opt.Advanced.LogFile) + utils.ChdirAndAcquireFileLock() + utils.SetNcpu() + utils.SetPprofPort() + transform.Init() - // start statistics - if config.Config.Advanced.MetricsPort != 0 { - statistics.Metrics.Address = config.Config.Source.Address - go func() { - log.Infof("metrics url: http://localhost:%d", config.Config.Advanced.MetricsPort) - mux := http.NewServeMux() - mux.HandleFunc("/", statistics.Handler) - err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Config.Advanced.MetricsPort), mux) - if err != nil { - log.PanicError(err) - } - }() + // create reader + var theReader reader.Reader + if v.IsSet("SyncStandaloneReader") { + opts := new(reader.SyncStandaloneReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("SyncStandaloneReader", opts) + if err != nil { + log.Panicf("failed to read the SyncReader config entry. err: %v", err) + } + theReader = reader.NewSyncStandaloneReader(opts) + log.Infof("create SyncStandaloneReader: %v", opts.Address) + } else if v.IsSet("SyncClusterReader") { + opts := new(reader.SyncClusterReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("SyncClusterReader", opts) + if err != nil { + log.Panicf("failed to read the SyncReader config entry. err: %v", err) + } + theReader = reader.NewSyncClusterReader(opts) + log.Infof("create SyncClusterReader: %v", opts.Address) + } else if v.IsSet("ScanStandaloneReader") { + opts := new(reader.ScanStandaloneReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("ScanStandaloneReader", opts) + if err != nil { + log.Panicf("failed to read the ScanReader config entry. err: %v", err) + } + theReader = reader.NewScanStandaloneReader(opts) + log.Infof("create ScanStandaloneReader: %v", opts.Address) + } else if v.IsSet("ScanClusterReader") { + opts := new(reader.ScanClusterReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("ScanClusterReader", opts) + if err != nil { + log.Panicf("failed to read the ScanReader config entry. err: %v", err) + } + theReader = reader.NewScanClusterReader(opts) + log.Infof("create ScanClusterReader: %v", opts.Address) + } else if v.IsSet("RdbReader") { + opts := new(reader.RdbReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("RdbReader", opts) + if err != nil { + log.Panicf("failed to read the RdbReader config entry. err: %v", err) + } + theReader = reader.NewRDBReader(opts) + log.Infof("create RdbReader: %v", opts.Filepath) + } else { + log.Panicf("no reader config entry found") } // create writer var theWriter writer.Writer - target := &config.Config.Target - switch config.Config.Target.Type { - case "standalone": - theWriter = writer.NewRedisWriter(target.Address, target.Username, target.Password, target.IsTLS) - case "cluster": - theWriter = writer.NewRedisClusterWriter(target.Address, target.Username, target.Password, target.IsTLS) - default: - log.Panicf("unknown target type: %s", target.Type) - } - - // create reader - source := &config.Config.Source - var theReader reader.Reader - if config.Config.Type == "sync" { - theReader = reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS, source.ElastiCachePSync) - } else if config.Config.Type == "restore" { - theReader = reader.NewRDBReader(source.RDBFilePath) - } else if config.Config.Type == "scan" { - theReader = reader.NewScanReader(source.Address, source.Username, source.Password, source.IsTLS) + if v.IsSet("RedisStandaloneWriter") { + opts := new(writer.RedisStandaloneWriterOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("RedisStandaloneWriter", opts) + if err != nil { + log.Panicf("failed to read the RedisStandaloneWriter config entry. err: %v", err) + } + theWriter = writer.NewRedisStandaloneWriter(opts) + log.Infof("create RedisStandaloneWriter: %v", opts.Address) + } else if v.IsSet("RedisClusterWriter") { + opts := new(writer.RedisClusterWriterOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("RedisClusterWriter", opts) + if err != nil { + log.Panicf("failed to read the RedisClusterWriter config entry. err: %v", err) + } + theWriter = writer.NewRedisClusterWriter(opts) + log.Infof("create RedisClusterWriter: %v", opts.Address) } else { - log.Panicf("unknown source type: %s", config.Config.Type) + log.Panicf("no writer config entry found") } - ch := theReader.StartRead() - // start sync - statistics.Init() - id := uint64(0) + // create status + status.Init(theReader, theWriter) + + ch := theReader.StartRead() for e := range ch { - statistics.UpdateInQueueEntriesCount(uint64(len(ch))) // calc arguments - e.Id = id - id++ e.CmdName, e.Group, e.Keys = commands.CalcKeys(e.Argv) e.Slots = commands.CalcSlots(e.Keys) // filter - code := filter.Filter(e) - statistics.UpdateEntryId(e.Id) - if code == filter.Allow { + code := transform.Transform(e) + if code == transform.Allow { theWriter.Write(e) - statistics.AddAllowEntriesCount() - } else if code == filter.Disallow { - // do something - statistics.AddDisallowEntriesCount() + status.AddEntryCount(e.CmdName, true) + } else if code == transform.Disallow { + status.AddEntryCount(e.CmdName, false) } else { - log.Panicf("error when run lua filter. entry: %s", e.ToString()) + log.Panicf("error when run lua filter. entry: %s", e.String()) } } - theWriter.Close() - log.Infof("finished.") + + theWriter.Close() // Wait for all writing operations to complete + utils.ReleaseFileLock() // Release file lock + log.Infof("all done") } diff --git a/sync.toml b/configs/all.toml similarity index 60% rename from sync.toml rename to configs/all.toml index 71629e40..d94224c0 100644 --- a/sync.toml +++ b/configs/all.toml @@ -1,37 +1,46 @@ -type = "sync" +#transform = "" -[source] -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... +#[RdbReader] +#filepath = "/data/dump.rdb" + + +#[SyncStandaloneReader] +#address = "127.0.0.1:6379" +#username = "" # keep empty if not using ACL +#password = "" # keep empty if no authentication is required +#tls = false + + +[SyncClusterReader] address = "127.0.0.1:6379" username = "" # keep empty if not using ACL password = "" # keep empty if no authentication is required tls = false -elasticache_psync = "" # using when source is ElastiCache. ref: https://github.com/alibaba/RedisShake/issues/373 -[target] -type = "standalone" # "standalone" or "cluster" -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -# When the target is a cluster, write the address of one of the nodes. -# redis-shake will obtain other nodes through the `cluster nodes` command. + +[RedisStandaloneWriter] address = "127.0.0.1:6380" username = "" # keep empty if not using ACL password = "" # keep empty if no authentication is required tls = false -[advanced] -dir = "data" -# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores -ncpu = 4 +#[RedisClusterWriter] +#address = "127.0.0.1:6380" +#username = "" # keep empty if not using ACL +#password = "" # keep empty if no authentication is required +#tls = false -# pprof port, 0 means disable -pprof_port = 0 -# metric port, 0 means disable -metrics_port = 0 +[advanced] +dir = "data" +ncpu = 3 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores + +pprof_port = 0 # pprof port, 0 means disable +status_port = 0 # status port, 0 means disable # log -log_file = "redis-shake.log" +log_file = "shake.log" log_level = "info" # debug, info or warn log_interval = 5 # in seconds @@ -44,7 +53,8 @@ log_interval = 5 # in seconds # ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip -# pipeline +# redis-shake uses pipeline to improve sending performance. +# This item limits the maximum number of commands in a pipeline. pipeline_count_limit = 1024 # Client query buffers accumulate new commands. They are limited to a fixed @@ -53,4 +63,7 @@ target_redis_client_max_querybuf_len = 1024_000_000 # In the Redis protocol, bulk requests, that are, elements representing single # strings, are normally limited to 512 mb. -target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file +target_redis_proto_max_bulk_len = 512_000_000 + +# If the source is Elasticache or MemoryDB, you can set this item. +aws_psync = "" \ No newline at end of file diff --git a/filters/aliyun.lua b/configs/transform/aliyun.lua similarity index 66% rename from filters/aliyun.lua rename to configs/transform/aliyun.lua index 8ab684b6..2cab44ce 100644 --- a/filters/aliyun.lua +++ b/configs/transform/aliyun.lua @@ -1,5 +1,5 @@ -- Aliyun Redis 4.0: skip OPINFO command -function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) +function transform(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) if cmd_name == "OPINFO" then return 1, db_id -- disallow else diff --git a/filters/aws.lua b/configs/transform/aws.lua similarity index 66% rename from filters/aws.lua rename to configs/transform/aws.lua index 23101c39..afeee287 100644 --- a/filters/aws.lua +++ b/configs/transform/aws.lua @@ -1,5 +1,5 @@ -- ElastiCache: skip REPLCONF command -function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) +function transform(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) if cmd_name == "REPLCONF" then return 1, db_id -- disallow else diff --git a/filters/key_prefix.lua b/configs/transform/key_prefix.lua similarity index 100% rename from filters/key_prefix.lua rename to configs/transform/key_prefix.lua diff --git a/filters/print.lua b/configs/transform/print.lua similarity index 100% rename from filters/print.lua rename to configs/transform/print.lua diff --git a/filters/skip_scripts.lua b/configs/transform/skip_scripts.lua similarity index 100% rename from filters/skip_scripts.lua rename to configs/transform/skip_scripts.lua diff --git a/filters/swap_db.lua b/configs/transform/swap_db.lua similarity index 100% rename from filters/swap_db.lua rename to configs/transform/swap_db.lua diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..ff359c63 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +cache/ \ No newline at end of file diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts new file mode 100644 index 00000000..9b1bba3d --- /dev/null +++ b/docs/.vitepress/config.ts @@ -0,0 +1,29 @@ +import { defineConfig } from 'vitepress' + +// https://vitepress.dev/reference/site-config +export default defineConfig({ + title: "RedisShake", + description: "RedisShake is a tool for processing and migrating Redis data.", + + themeConfig: { + // https://vitepress.dev/reference/default-theme-config + nav: [ + { text: 'Home', link: '/' }, + { text: 'Examples', link: '/markdown-examples' } + ], + + sidebar: [ + { + text: 'Examples', + items: [ + { text: 'Markdown Examples', link: '/markdown-examples' }, + { text: 'Runtime API Examples', link: '/api-examples' } + ] + } + ], + + socialLinks: [ + { icon: 'github', link: 'https://github.com/tair-opensource/RedisShake' } + ] + } +}) diff --git a/docs/api-examples.md b/docs/api-examples.md new file mode 100644 index 00000000..6bd8bb5c --- /dev/null +++ b/docs/api-examples.md @@ -0,0 +1,49 @@ +--- +outline: deep +--- + +# Runtime API Examples + +This page demonstrates usage of some of the runtime APIs provided by VitePress. + +The main `useData()` API can be used to access site, theme, and page data for the current page. It works in both `.md` and `.vue` files: + +```md + + +## Results + +### Theme Data +
{{ theme }}
+ +### Page Data +
{{ page }}
+ +### Page Frontmatter +
{{ frontmatter }}
+``` + + + +## Results + +### Theme Data +
{{ theme }}
+ +### Page Data +
{{ page }}
+ +### Page Frontmatter +
{{ frontmatter }}
+ +## More + +Check out the documentation for the [full list of runtime APIs](https://vitepress.dev/reference/runtime-api#usedata). diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..9f731cf8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,25 @@ +--- +# https://vitepress.dev/reference/default-theme-home-page +layout: home + +hero: + name: "RedisShake" + text: "Data Transform And Data Migration For Redis-like Database" + tagline: RedisShake is a tool for transform and migrating Redis data. + actions: + - theme: brand + text: Markdown Examples + link: /markdown-examples + - theme: alt + text: API Examples + link: /api-examples + +features: + - title: Feature A + details: Lorem ipsum dolor sit amet, consectetur adipiscing elit + - title: Feature B + details: Lorem ipsum dolor sit amet, consectetur adipiscing elit + - title: Feature C + details: Lorem ipsum dolor sit amet, consectetur adipiscing elit +--- + diff --git a/docs/markdown-examples.md b/docs/markdown-examples.md new file mode 100644 index 00000000..8e55eb8a --- /dev/null +++ b/docs/markdown-examples.md @@ -0,0 +1,85 @@ +# Markdown Extension Examples + +This page demonstrates some of the built-in markdown extensions provided by VitePress. + +## Syntax Highlighting + +VitePress provides Syntax Highlighting powered by [Shiki](https://github.com/shikijs/shiki), with additional features like line-highlighting: + +**Input** + +```` +```js{4} +export default { + data () { + return { + msg: 'Highlighted!' + } + } +} +``` +```` + +**Output** + +```js{4} +export default { + data () { + return { + msg: 'Highlighted!' + } + } +} +``` + +## Custom Containers + +**Input** + +```md +::: info +This is an info box. +::: + +::: tip +This is a tip. +::: + +::: warning +This is a warning. +::: + +::: danger +This is a dangerous warning. +::: + +::: details +This is a details block. +::: +``` + +**Output** + +::: info +This is an info box. +::: + +::: tip +This is a tip. +::: + +::: warning +This is a warning. +::: + +::: danger +This is a dangerous warning. +::: + +::: details +This is a details block. +::: + +## More + +Check out the documentation for the [full list of markdown extensions](https://vitepress.dev/guide/markdown). diff --git a/docs/package-lock.json b/docs/package-lock.json new file mode 100644 index 00000000..03785fe5 --- /dev/null +++ b/docs/package-lock.json @@ -0,0 +1,1129 @@ +{ + "name": "docs", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "devDependencies": { + "vitepress": "^1.0.0-alpha.75" + } + }, + "node_modules/@algolia/autocomplete-core": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.8.2.tgz", + "integrity": "sha512-mTeshsyFhAqw/ebqNsQpMtbnjr+qVOSKXArEj4K0d7sqc8It1XD0gkASwecm9mF/jlOQ4Z9RNg1HbdA8JPdRwQ==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-shared": "1.8.2" + } + }, + "node_modules/@algolia/autocomplete-preset-algolia": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.8.2.tgz", + "integrity": "sha512-J0oTx4me6ZM9kIKPuL3lyU3aB8DEvpVvR6xWmHVROx5rOYJGQcZsdG4ozxwcOyiiu3qxMkIbzntnV1S1VWD8yA==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-shared": "1.8.2" + }, + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/autocomplete-shared": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.8.2.tgz", + "integrity": "sha512-b6Z/X4MczChMcfhk6kfRmBzPgjoPzuS9KGR4AFsiLulLNRAAqhP+xZTKtMnZGhLuc61I20d5WqlId02AZvcO6g==", + "dev": true + }, + "node_modules/@algolia/cache-browser-local-storage": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.17.0.tgz", + "integrity": "sha512-myRSRZDIMYB8uCkO+lb40YKiYHi0fjpWRtJpR/dgkaiBlSD0plRyB6lLOh1XIfmMcSeBOqDE7y9m8xZMrXYfyQ==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.17.0" + } + }, + "node_modules/@algolia/cache-common": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.17.0.tgz", + "integrity": "sha512-g8mXzkrcUBIPZaulAuqE7xyHhLAYAcF2xSch7d9dABheybaU3U91LjBX6eJTEB7XVhEsgK4Smi27vWtAJRhIKQ==", + "dev": true + }, + "node_modules/@algolia/cache-in-memory": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.17.0.tgz", + "integrity": "sha512-PT32ciC/xI8z919d0oknWVu3kMfTlhQn3MKxDln3pkn+yA7F7xrxSALysxquv+MhFfNAcrtQ/oVvQVBAQSHtdw==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.17.0" + } + }, + "node_modules/@algolia/client-account": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.17.0.tgz", + "integrity": "sha512-sSEHx9GA6m7wrlsSMNBGfyzlIfDT2fkz2u7jqfCCd6JEEwmxt8emGmxAU/0qBfbhRSuGvzojoLJlr83BSZAKjA==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.17.0", + "@algolia/client-search": "4.17.0", + "@algolia/transporter": "4.17.0" + } + }, + "node_modules/@algolia/client-analytics": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.17.0.tgz", + "integrity": "sha512-84ooP8QA3mQ958hQ9wozk7hFUbAO+81CX1CjAuerxBqjKIInh1fOhXKTaku05O/GHBvcfExpPLIQuSuLYziBXQ==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.17.0", + "@algolia/client-search": "4.17.0", + "@algolia/requester-common": "4.17.0", + "@algolia/transporter": "4.17.0" + } + }, + "node_modules/@algolia/client-common": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.17.0.tgz", + "integrity": "sha512-jHMks0ZFicf8nRDn6ma8DNNsdwGgP/NKiAAL9z6rS7CymJ7L0+QqTJl3rYxRW7TmBhsUH40wqzmrG6aMIN/DrQ==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.17.0", + "@algolia/transporter": "4.17.0" + } + }, + "node_modules/@algolia/client-personalization": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.17.0.tgz", + "integrity": "sha512-RMzN4dZLIta1YuwT7QC9o+OeGz2cU6eTOlGNE/6RcUBLOU3l9tkCOdln5dPE2jp8GZXPl2yk54b2nSs1+pAjqw==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.17.0", + "@algolia/requester-common": "4.17.0", + "@algolia/transporter": "4.17.0" + } + }, + "node_modules/@algolia/client-search": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.17.0.tgz", + "integrity": "sha512-x4P2wKrrRIXszT8gb7eWsMHNNHAJs0wE7/uqbufm4tZenAp+hwU/hq5KVsY50v+PfwM0LcDwwn/1DroujsTFoA==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.17.0", + "@algolia/requester-common": "4.17.0", + "@algolia/transporter": "4.17.0" + } + }, + "node_modules/@algolia/logger-common": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.17.0.tgz", + "integrity": "sha512-DGuoZqpTmIKJFDeyAJ7M8E/LOenIjWiOsg1XJ1OqAU/eofp49JfqXxbfgctlVZVmDABIyOz8LqEoJ6ZP4DTyvw==", + "dev": true + }, + "node_modules/@algolia/logger-console": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.17.0.tgz", + "integrity": "sha512-zMPvugQV/gbXUvWBCzihw6m7oxIKp48w37QBIUu/XqQQfxhjoOE9xyfJr1KldUt5FrYOKZJVsJaEjTsu+bIgQg==", + "dev": true, + "dependencies": { + "@algolia/logger-common": "4.17.0" + } + }, + "node_modules/@algolia/requester-browser-xhr": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.17.0.tgz", + "integrity": "sha512-aSOX/smauyTkP21Pf52pJ1O2LmNFJ5iHRIzEeTh0mwBeADO4GdG94cAWDILFA9rNblq/nK3EDh3+UyHHjplZ1A==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.17.0" + } + }, + "node_modules/@algolia/requester-common": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.17.0.tgz", + "integrity": "sha512-XJjmWFEUlHu0ijvcHBoixuXfEoiRUdyzQM6YwTuB8usJNIgShua8ouFlRWF8iCeag0vZZiUm4S2WCVBPkdxFgg==", + "dev": true + }, + "node_modules/@algolia/requester-node-http": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.17.0.tgz", + "integrity": "sha512-bpb/wDA1aC6WxxM8v7TsFspB7yBN3nqCGs2H1OADolQR/hiAIjAxusbuMxVbRFOdaUvAIqioIIkWvZdpYNIn8w==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.17.0" + } + }, + "node_modules/@algolia/transporter": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.17.0.tgz", + "integrity": "sha512-6xL6H6fe+Fi0AEP3ziSgC+G04RK37iRb4uUUqVAH9WPYFI8g+LYFq6iv5HS8Cbuc5TTut+Bwj6G+dh/asdb9uA==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.17.0", + "@algolia/logger-common": "4.17.0", + "@algolia/requester-common": "4.17.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.21.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.21.8.tgz", + "integrity": "sha512-6zavDGdzG3gUqAdWvlLFfk+36RilI+Pwyuuh7HItyeScCWP3k6i8vKclAQ0bM/0y/Kz/xiwvxhMv9MgTJP5gmA==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@docsearch/css": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-3.3.4.tgz", + "integrity": "sha512-vDwCDoVXDgopw/hvr0zEADew2wWaGP8Qq0Bxhgii1Ewz2t4fQeyJwIRN/mWADeLFYPVkpz8TpEbxya/i6Tm0WA==", + "dev": true + }, + "node_modules/@docsearch/js": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@docsearch/js/-/js-3.3.4.tgz", + "integrity": "sha512-Xd2saBziXJ1UuVpcDz94zAFEFAM6ap993agh0za2e3LDZLhaW993b1f9gyUL4e1CZLsR076tztG2un2gVncvpA==", + "dev": true, + "dependencies": { + "@docsearch/react": "3.3.4", + "preact": "^10.0.0" + } + }, + "node_modules/@docsearch/react": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@docsearch/react/-/react-3.3.4.tgz", + "integrity": "sha512-aeOf1WC5zMzBEi2SI6WWznOmIo9rnpN4p7a3zHXxowVciqlI4HsZGtOR9nFOufLeolv7HibwLlaM0oyUqJxasw==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-core": "1.8.2", + "@algolia/autocomplete-preset-algolia": "1.8.2", + "@docsearch/css": "3.3.4", + "algoliasearch": "^4.0.0" + }, + "peerDependencies": { + "@types/react": ">= 16.8.0 < 19.0.0", + "react": ">= 16.8.0 < 19.0.0", + "react-dom": ">= 16.8.0 < 19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.18.tgz", + "integrity": "sha512-EmwL+vUBZJ7mhFCs5lA4ZimpUH3WMAoqvOIYhVQwdIgSpHC8ImHdsRyhHAVxpDYUSm0lWvd63z0XH1IlImS2Qw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.18.tgz", + "integrity": "sha512-/iq0aK0eeHgSC3z55ucMAHO05OIqmQehiGay8eP5l/5l+iEr4EIbh4/MI8xD9qRFjqzgkc0JkX0LculNC9mXBw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.18.tgz", + "integrity": "sha512-x+0efYNBF3NPW2Xc5bFOSFW7tTXdAcpfEg2nXmxegm4mJuVeS+i109m/7HMiOQ6M12aVGGFlqJX3RhNdYM2lWg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.18.tgz", + "integrity": "sha512-6tY+djEAdF48M1ONWnQb1C+6LiXrKjmqjzPNPWXhu/GzOHTHX2nh8Mo2ZAmBFg0kIodHhciEgUBtcYCAIjGbjQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.18.tgz", + "integrity": "sha512-Qq84ykvLvya3dO49wVC9FFCNUfSrQJLbxhoQk/TE1r6MjHo3sFF2tlJCwMjhkBVq3/ahUisj7+EpRSz0/+8+9A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.18.tgz", + "integrity": "sha512-fw/ZfxfAzuHfaQeMDhbzxp9mc+mHn1Y94VDHFHjGvt2Uxl10mT4CDavHm+/L9KG441t1QdABqkVYwakMUeyLRA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.18.tgz", + "integrity": "sha512-FQFbRtTaEi8ZBi/A6kxOC0V0E9B/97vPdYjY9NdawyLd4Qk5VD5g2pbWN2VR1c0xhzcJm74HWpObPszWC+qTew==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.18.tgz", + "integrity": "sha512-jW+UCM40LzHcouIaqv3e/oRs0JM76JfhHjCavPxMUti7VAPh8CaGSlS7cmyrdpzSk7A+8f0hiedHqr/LMnfijg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.18.tgz", + "integrity": "sha512-R7pZvQZFOY2sxUG8P6A21eq6q+eBv7JPQYIybHVf1XkQYC+lT7nDBdC7wWKTrbvMXKRaGudp/dzZCwL/863mZQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.18.tgz", + "integrity": "sha512-ygIMc3I7wxgXIxk6j3V00VlABIjq260i967Cp9BNAk5pOOpIXmd1RFQJQX9Io7KRsthDrQYrtcx7QCof4o3ZoQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.18.tgz", + "integrity": "sha512-bvPG+MyFs5ZlwYclCG1D744oHk1Pv7j8psF5TfYx7otCVmcJsEXgFEhQkbhNW8otDHL1a2KDINW20cfCgnzgMQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.18.tgz", + "integrity": "sha512-oVqckATOAGuiUOa6wr8TXaVPSa+6IwVJrGidmNZS1cZVx0HqkTMkqFGD2HIx9H1RvOwFeWYdaYbdY6B89KUMxA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.18.tgz", + "integrity": "sha512-3dLlQO+b/LnQNxgH4l9rqa2/IwRJVN9u/bK63FhOPB4xqiRqlQAU0qDU3JJuf0BmaH0yytTBdoSBHrb2jqc5qQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.18.tgz", + "integrity": "sha512-/x7leOyDPjZV3TcsdfrSI107zItVnsX1q2nho7hbbQoKnmoeUWjs+08rKKt4AUXju7+3aRZSsKrJtaRmsdL1xA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.18.tgz", + "integrity": "sha512-cX0I8Q9xQkL/6F5zWdYmVf5JSQt+ZfZD2bJudZrWD+4mnUvoZ3TDDXtDX2mUaq6upMFv9FlfIh4Gfun0tbGzuw==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.18.tgz", + "integrity": "sha512-66RmRsPlYy4jFl0vG80GcNRdirx4nVWAzJmXkevgphP1qf4dsLQCpSKGM3DUQCojwU1hnepI63gNZdrr02wHUA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.18.tgz", + "integrity": "sha512-95IRY7mI2yrkLlTLb1gpDxdC5WLC5mZDi+kA9dmM5XAGxCME0F8i4bYH4jZreaJ6lIZ0B8hTrweqG1fUyW7jbg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.18.tgz", + "integrity": "sha512-WevVOgcng+8hSZ4Q3BKL3n1xTv5H6Nb53cBrtzzEjDbbnOmucEVcZeGCsCOi9bAOcDYEeBZbD2SJNBxlfP3qiA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.18.tgz", + "integrity": "sha512-Rzf4QfQagnwhQXVBS3BYUlxmEbcV7MY+BH5vfDZekU5eYpcffHSyjU8T0xucKVuOcdCsMo+Ur5wmgQJH2GfNrg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.18.tgz", + "integrity": "sha512-Kb3Ko/KKaWhjeAm2YoT/cNZaHaD1Yk/pa3FTsmqo9uFh1D1Rfco7BBLIPdDOozrObj2sahslFuAQGvWbgWldAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.18.tgz", + "integrity": "sha512-0/xUMIdkVHwkvxfbd5+lfG7mHOf2FRrxNbPiKWg9C4fFrB8H0guClmaM3BFiRUYrznVoyxTIyC/Ou2B7QQSwmw==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.18.tgz", + "integrity": "sha512-qU25Ma1I3NqTSHJUOKi9sAH1/Mzuvlke0ioMJRthLXKm7JiSKVwFghlGbDLOO2sARECGhja4xYfRAZNPAkooYg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@types/web-bluetooth": { + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.17.tgz", + "integrity": "sha512-4p9vcSmxAayx72yn70joFoL44c9MO/0+iVEBIQXe3v2h2SiAsEIo/G5v6ObFWvNKRFjbrVadNf9LqEEZeQPzdA==", + "dev": true + }, + "node_modules/@vitejs/plugin-vue": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-4.2.2.tgz", + "integrity": "sha512-kNH4wMAqs13UiZe/2If1ioO0Mjz71rr2oALTl2c5ajBIox9Vz/UGW/wGkr7GA3SC6Eb29c1HtzAtxdGfbXAkfQ==", + "dev": true, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.0.0", + "vue": "^3.2.25" + } + }, + "node_modules/@vue/compiler-core": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.3.1.tgz", + "integrity": "sha512-5le1qYSBgLWg2jdLrbydlhnPJkkzMw46UrRUvTnOKlfg6pThtm9ohhqBhNPHbr0RcM1MCbK5WZe/3Ghz0SZjpQ==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.21.3", + "@vue/shared": "3.3.1", + "estree-walker": "^2.0.2", + "source-map-js": "^1.0.2" + } + }, + "node_modules/@vue/compiler-dom": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.3.1.tgz", + "integrity": "sha512-VmgIsoLivCft3+oNc5KM7b9wd0nZxP/g2qilMwi1hJyGA624KWnNKHn4hzBQs4FpzydUVpNy+TWVT8KiRCh3MQ==", + "dev": true, + "dependencies": { + "@vue/compiler-core": "3.3.1", + "@vue/shared": "3.3.1" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.3.1.tgz", + "integrity": "sha512-G+FPwBbXSLaA4+Ry5/bdD9Oda+sRslQcE9o6JSZaougRiT4OjVL0vtkbQHPrGRTULZV28OcrAjRfSZOSB0OTXQ==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.15", + "@vue/compiler-core": "3.3.1", + "@vue/compiler-dom": "3.3.1", + "@vue/compiler-ssr": "3.3.1", + "@vue/reactivity-transform": "3.3.1", + "@vue/shared": "3.3.1", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.0", + "postcss": "^8.1.10", + "source-map-js": "^1.0.2" + } + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.3.1.tgz", + "integrity": "sha512-QOQWGNCWuSeyKx4KvWSJlnIMGg+/2oCHgkFUYo7aJ+9Uaaz45yRgKQ+FNigy50NYBQIhpXn2e4OSR8GXh4knrQ==", + "dev": true, + "dependencies": { + "@vue/compiler-dom": "3.3.1", + "@vue/shared": "3.3.1" + } + }, + "node_modules/@vue/devtools-api": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.5.0.tgz", + "integrity": "sha512-o9KfBeaBmCKl10usN4crU53fYtC1r7jJwdGKjPT24t348rHxgfpZ0xL3Xm/gLUYnc0oTp8LAmrxOeLyu6tbk2Q==", + "dev": true + }, + "node_modules/@vue/reactivity": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.3.1.tgz", + "integrity": "sha512-zCfmazOtyUdC1NS/EPiSYJ4RqojqmTAviJyBbyVvY8zAv5NhK44Yfw0E1tt+m5vz0ZO1ptI9jDKBr3MWIEkpgw==", + "dev": true, + "dependencies": { + "@vue/shared": "3.3.1" + } + }, + "node_modules/@vue/reactivity-transform": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/reactivity-transform/-/reactivity-transform-3.3.1.tgz", + "integrity": "sha512-MkOrJauAGH4MNdxGW/PmrDegMyOGX0wGIdKUZJRBXOTpotDONg7/TPJe2QeGeBCow/5v9iOqZOWCfvmOWIaDMg==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.15", + "@vue/compiler-core": "3.3.1", + "@vue/shared": "3.3.1", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.0" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.3.1.tgz", + "integrity": "sha512-Ljb37LYafhQqKIasc0r32Cva8gIh6VeSMjlwO6V03tCjHd18gmjP0F4UD+8/a59sGTysAgA8Rb9lIC2DVxRz2Q==", + "dev": true, + "dependencies": { + "@vue/reactivity": "3.3.1", + "@vue/shared": "3.3.1" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.3.1.tgz", + "integrity": "sha512-NBjYbQPtMklb7lsJsM2Juv5Ygry6mvZP7PdH1GZqrzfLkvlplQT3qCtQMd/sib6yiy8t9m/Y4hVU7X9nzb9Oeg==", + "dev": true, + "dependencies": { + "@vue/runtime-core": "3.3.1", + "@vue/shared": "3.3.1", + "csstype": "^3.1.1" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.3.1.tgz", + "integrity": "sha512-sod8ggOwbkQXw3lBjfzrbdxRS9lw/lNHoMaXghHawNYowf+4WoaLWD5ouz6fPZadUqNKAsqK95p8DYb1vcVfPA==", + "dev": true, + "dependencies": { + "@vue/compiler-ssr": "3.3.1", + "@vue/shared": "3.3.1" + }, + "peerDependencies": { + "vue": "3.3.1" + } + }, + "node_modules/@vue/shared": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.3.1.tgz", + "integrity": "sha512-ybDBtQ+479HL/bkeIOIAwgpeAEACzztkvulJLbK3JMFuTOv4qDivmV3AIsR8RHYJ+RD9tQxcHWBsX4GqEcYrfw==", + "dev": true + }, + "node_modules/@vueuse/core": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/@vueuse/core/-/core-10.1.2.tgz", + "integrity": "sha512-roNn8WuerI56A5uiTyF/TEYX0Y+VKlhZAF94unUfdhbDUI+NfwQMn4FUnUscIRUhv3344qvAghopU4bzLPNFlA==", + "dev": true, + "dependencies": { + "@types/web-bluetooth": "^0.0.17", + "@vueuse/metadata": "10.1.2", + "@vueuse/shared": "10.1.2", + "vue-demi": ">=0.14.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/core/node_modules/vue-demi": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.1.tgz", + "integrity": "sha512-rt+yuCtXvscYot9SQQj3WKZJVSriPNqVkpVBNEHPzSgBv7QIYzsS410VqVgvx8f9AAPgjg+XPKvmV3vOqqkJQQ==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/@vueuse/metadata": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/@vueuse/metadata/-/metadata-10.1.2.tgz", + "integrity": "sha512-3mc5BqN9aU2SqBeBuWE7ne4OtXHoHKggNgxZR2K+zIW4YLsy6xoZ4/9vErQs6tvoKDX6QAqm3lvsrv0mczAwIQ==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/@vueuse/shared/-/shared-10.1.2.tgz", + "integrity": "sha512-1uoUTPBlgyscK9v6ScGeVYDDzlPSFXBlxuK7SfrDGyUTBiznb3mNceqhwvZHjtDRELZEN79V5uWPTF1VDV8svA==", + "dev": true, + "dependencies": { + "vue-demi": ">=0.14.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared/node_modules/vue-demi": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.1.tgz", + "integrity": "sha512-rt+yuCtXvscYot9SQQj3WKZJVSriPNqVkpVBNEHPzSgBv7QIYzsS410VqVgvx8f9AAPgjg+XPKvmV3vOqqkJQQ==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/algoliasearch": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.17.0.tgz", + "integrity": "sha512-JMRh2Mw6sEnVMiz6+APsi7lx9a2jiDFF+WUtANaUVCv6uSU9UOLdo5h9K3pdP6frRRybaM2fX8b1u0nqICS9aA==", + "dev": true, + "dependencies": { + "@algolia/cache-browser-local-storage": "4.17.0", + "@algolia/cache-common": "4.17.0", + "@algolia/cache-in-memory": "4.17.0", + "@algolia/client-account": "4.17.0", + "@algolia/client-analytics": "4.17.0", + "@algolia/client-common": "4.17.0", + "@algolia/client-personalization": "4.17.0", + "@algolia/client-search": "4.17.0", + "@algolia/logger-common": "4.17.0", + "@algolia/logger-console": "4.17.0", + "@algolia/requester-browser-xhr": "4.17.0", + "@algolia/requester-common": "4.17.0", + "@algolia/requester-node-http": "4.17.0", + "@algolia/transporter": "4.17.0" + } + }, + "node_modules/ansi-sequence-parser": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.0.tgz", + "integrity": "sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ==", + "dev": true + }, + "node_modules/body-scroll-lock": { + "version": "4.0.0-beta.0", + "resolved": "https://registry.npmjs.org/body-scroll-lock/-/body-scroll-lock-4.0.0-beta.0.tgz", + "integrity": "sha512-a7tP5+0Mw3YlUJcGAKUqIBkYYGlYxk2fnCasq/FUph1hadxlTRjF+gAcZksxANnaMnALjxEddmSi/H3OR8ugcQ==", + "dev": true + }, + "node_modules/csstype": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", + "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.17.18", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.18.tgz", + "integrity": "sha512-z1lix43jBs6UKjcZVKOw2xx69ffE2aG0PygLL5qJ9OS/gy0Ewd1gW/PUQIOIQGXBHWNywSc0floSKoMFF8aK2w==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.17.18", + "@esbuild/android-arm64": "0.17.18", + "@esbuild/android-x64": "0.17.18", + "@esbuild/darwin-arm64": "0.17.18", + "@esbuild/darwin-x64": "0.17.18", + "@esbuild/freebsd-arm64": "0.17.18", + "@esbuild/freebsd-x64": "0.17.18", + "@esbuild/linux-arm": "0.17.18", + "@esbuild/linux-arm64": "0.17.18", + "@esbuild/linux-ia32": "0.17.18", + "@esbuild/linux-loong64": "0.17.18", + "@esbuild/linux-mips64el": "0.17.18", + "@esbuild/linux-ppc64": "0.17.18", + "@esbuild/linux-riscv64": "0.17.18", + "@esbuild/linux-s390x": "0.17.18", + "@esbuild/linux-x64": "0.17.18", + "@esbuild/netbsd-x64": "0.17.18", + "@esbuild/openbsd-x64": "0.17.18", + "@esbuild/sunos-x64": "0.17.18", + "@esbuild/win32-arm64": "0.17.18", + "@esbuild/win32-ia32": "0.17.18", + "@esbuild/win32-x64": "0.17.18" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, + "node_modules/magic-string": { + "version": "0.30.0", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.0.tgz", + "integrity": "sha512-LA+31JYDJLs82r2ScLrlz1GjSgu66ZV518eyWT+S8VhyQn/JL0u9MeBOvQMGYiPk1DBiSN9DDMOcXvigJZaViQ==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.13" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mark.js": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/mark.js/-/mark.js-8.11.1.tgz", + "integrity": "sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==", + "dev": true + }, + "node_modules/minisearch": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-6.0.1.tgz", + "integrity": "sha512-Ly1w0nHKnlhAAh6/BF/+9NgzXfoJxaJ8nhopFhQ3NcvFJrFIL+iCg9gw9e9UMBD+XIsp/RyznJ/o5UIe5Kw+kg==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/postcss": { + "version": "8.4.23", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.23.tgz", + "integrity": "sha512-bQ3qMcpF6A/YjR55xtoTr0jGOlnPOKAIMdOWiv0EIT6HVPEaJiJB4NLljSbiHoC2RX7DN5Uvjtpbg1NPdwv1oA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/preact": { + "version": "10.13.2", + "resolved": "https://registry.npmjs.org/preact/-/preact-10.13.2.tgz", + "integrity": "sha512-q44QFLhOhty2Bd0Y46fnYW0gD/cbVM9dUVtNTDKPcdXSMA7jfY+Jpd6rk3GB0lcQss0z5s/6CmVP0Z/hV+g6pw==", + "dev": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/preact" + } + }, + "node_modules/rollup": { + "version": "3.21.6", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.21.6.tgz", + "integrity": "sha512-SXIICxvxQxR3D4dp/3LDHZIJPC8a4anKMHd4E3Jiz2/JnY+2bEjqrOokAauc5ShGVNFHlEFjBXAXlaxkJqIqSg==", + "dev": true, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=14.18.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/shiki": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.2.tgz", + "integrity": "sha512-ltSZlSLOuSY0M0Y75KA+ieRaZ0Trf5Wl3gutE7jzLuIcWxLp5i/uEnLoQWNvgKXQ5OMpGkJnVMRLAuzjc0LJ2A==", + "dev": true, + "dependencies": { + "ansi-sequence-parser": "^1.1.0", + "jsonc-parser": "^3.2.0", + "vscode-oniguruma": "^1.7.0", + "vscode-textmate": "^8.0.0" + } + }, + "node_modules/source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/vite": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.3.5.tgz", + "integrity": "sha512-0gEnL9wiRFxgz40o/i/eTBwm+NEbpUeTWhzKrZDSdKm6nplj+z4lKz8ANDgildxHm47Vg8EUia0aicKbawUVVA==", + "dev": true, + "dependencies": { + "esbuild": "^0.17.5", + "postcss": "^8.4.23", + "rollup": "^3.21.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + }, + "peerDependencies": { + "@types/node": ">= 14", + "less": "*", + "sass": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vitepress": { + "version": "1.0.0-alpha.75", + "resolved": "https://registry.npmjs.org/vitepress/-/vitepress-1.0.0-alpha.75.tgz", + "integrity": "sha512-twpPZ/6UnDR8X0Nmj767KwKhXlTQQM9V/J1i2BP9ryO29/w4hpxBfEum6nvfpNhJ4H3h+cIhwzAK/e9crZ6HEQ==", + "dev": true, + "dependencies": { + "@docsearch/css": "^3.3.4", + "@docsearch/js": "^3.3.4", + "@vitejs/plugin-vue": "^4.2.1", + "@vue/devtools-api": "^6.5.0", + "@vueuse/core": "^10.1.0", + "body-scroll-lock": "4.0.0-beta.0", + "mark.js": "8.11.1", + "minisearch": "^6.0.1", + "shiki": "^0.14.2", + "vite": "^4.3.3", + "vue": "^3.2.47" + }, + "bin": { + "vitepress": "bin/vitepress.js" + } + }, + "node_modules/vscode-oniguruma": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", + "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==", + "dev": true + }, + "node_modules/vscode-textmate": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-8.0.0.tgz", + "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==", + "dev": true + }, + "node_modules/vue": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.3.1.tgz", + "integrity": "sha512-3Rwy4I5idbPVSDZu6I+fFh6tdDSZbauImCTqLxE7y0LpHtiDvPeY01OI7RkFPbva1nk4hoO0sv/NzosH2h60sg==", + "dev": true, + "dependencies": { + "@vue/compiler-dom": "3.3.1", + "@vue/compiler-sfc": "3.3.1", + "@vue/runtime-dom": "3.3.1", + "@vue/server-renderer": "3.3.1", + "@vue/shared": "3.3.1" + } + } + } +} diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 00000000..5e63370e --- /dev/null +++ b/docs/package.json @@ -0,0 +1,10 @@ +{ + "devDependencies": { + "vitepress": "^1.0.0-alpha.75" + }, + "scripts": { + "docs:dev": "vitepress dev", + "docs:build": "vitepress build", + "docs:preview": "vitepress preview" + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index 4dc27cf2..6c649304 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,36 @@ -module github.com/alibaba/RedisShake +module RedisShake -go 1.17 +go 1.20 require ( - github.com/pelletier/go-toml/v2 v2.0.0-beta.3 + github.com/dustin/go-humanize v1.0.1 + github.com/go-stack/stack v1.8.1 + github.com/mcuadros/go-defaults v1.2.0 + github.com/redis/go-redis/v9 v9.0.4 github.com/rs/zerolog v1.28.0 + github.com/spf13/viper v1.15.0 + github.com/theckman/go-flock v0.8.1 github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 55959bc7..411cbddb 100644 --- a/go.sum +++ b/go.sum @@ -1,33 +1,511 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/pelletier/go-toml/v2 v2.0.0-beta.3 h1:PNCTU4naEJ8mKal97P3A2qDU74QRQGlv4FXiL1XDqi4= -github.com/pelletier/go-toml/v2 v2.0.0-beta.3/go.mod h1:aNseLYu/uKskg0zpr/kbr2z8yGuWtotWf/0BpGIAL2Y= +github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc= +github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/redis/go-redis/v9 v9.0.4 h1:FC82T+CHJ/Q/PdyLW++GeCO+Ol59Y4T7R4jbgjvktgc= +github.com/redis/go-redis/v9 v9.0.4/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 h1:t0lM6y/M5IiUZyvbBTcngso8SZEZICH7is9B6g/obVU= -github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/theckman/go-flock v0.8.1 h1:kTixuOsFBOtGYSTLRLWK6GOs1hk/8OD11sR1pDd0dl4= +github.com/theckman/go-flock v0.8.1/go.mod h1:kjuth3y9VJ2aNlkNEO99G/8lp9fMIKaGyBmh84IBheM= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/client/func.go b/internal/client/func.go index dcd0c1e0..9195683a 100644 --- a/internal/client/func.go +++ b/internal/client/func.go @@ -1,9 +1,10 @@ package client import ( + "RedisShake/internal/client/proto" + "RedisShake/internal/log" "bytes" - "github.com/alibaba/RedisShake/internal/client/proto" - "github.com/alibaba/RedisShake/internal/log" + "strings" ) func EncodeArgv(argv []string, buf *bytes.Buffer) { @@ -15,6 +16,12 @@ func EncodeArgv(argv []string, buf *bytes.Buffer) { } err := writer.WriteArgs(argvInterface) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } } + +// IsCluster is for determining whether the server is in cluster mode. +func (r *Redis) IsCluster() bool { + reply := r.DoWithStringReply("INFO", "Cluster") + return strings.Contains(reply, "cluster_enabled:1") +} diff --git a/internal/client/redis.go b/internal/client/redis.go index 4d4d567c..d85cc8df 100644 --- a/internal/client/redis.go +++ b/internal/client/redis.go @@ -1,10 +1,10 @@ package client import ( + "RedisShake/internal/client/proto" + "RedisShake/internal/log" "bufio" "crypto/tls" - "github.com/alibaba/RedisShake/internal/client/proto" - "github.com/alibaba/RedisShake/internal/log" "net" "strconv" "time" @@ -17,19 +17,19 @@ type Redis struct { protoWriter *proto.Writer } -func NewRedisClient(address string, username string, password string, isTls bool) *Redis { +func NewRedisClient(address string, username string, password string, Tls bool) *Redis { r := new(Redis) var conn net.Conn var dialer net.Dialer var err error dialer.Timeout = 3 * time.Second - if isTls { + if Tls { conn, err = tls.DialWithDialer(&dialer, "tcp", address, &tls.Config{InsecureSkipVerify: true}) } else { conn, err = dialer.Dial("tcp", address) } if err != nil { - log.PanicError(err) + log.Panicf("dial failed. address=[%s], tls=[%v], err=[%v]", address, Tls, err) } r.reader = bufio.NewReader(conn) @@ -48,14 +48,10 @@ func NewRedisClient(address string, username string, password string, isTls bool if reply != "OK" { log.Panicf("auth failed with reply: %s", reply) } - log.Infof("auth successful. address=[%s]", address) - } else { - log.Infof("no password. address=[%s]", address) } // ping to test connection reply := r.DoWithStringReply("ping") - if reply != "PONG" { panic("ping failed with reply: " + reply) } @@ -68,7 +64,7 @@ func (r *Redis) DoWithStringReply(args ...string) string { replyInterface, err := r.Receive() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } reply := replyInterface.(string) return reply @@ -81,7 +77,7 @@ func (r *Redis) Send(args ...string) { } err := r.protoWriter.WriteArgs(argsInterface) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.flush() } @@ -89,7 +85,7 @@ func (r *Redis) Send(args ...string) { func (r *Redis) SendBytes(buf []byte) { _, err := r.writer.Write(buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.flush() } @@ -97,7 +93,7 @@ func (r *Redis) SendBytes(buf []byte) { func (r *Redis) flush() { err := r.writer.Flush() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } } @@ -120,7 +116,7 @@ func (r *Redis) Scan(cursor uint64) (newCursor uint64, keys []string) { r.Send("scan", strconv.FormatUint(cursor, 10), "count", "2048") reply, err := r.Receive() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } array := reply.([]interface{}) @@ -131,7 +127,7 @@ func (r *Redis) Scan(cursor uint64) (newCursor uint64, keys []string) { // cursor newCursor, err = strconv.ParseUint(array[0].(string), 10, 64) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } // keys keys = make([]string, 0) diff --git a/internal/client/reply.go b/internal/client/reply.go index ef385e71..b9a00cb2 100644 --- a/internal/client/reply.go +++ b/internal/client/reply.go @@ -1,10 +1,10 @@ package client -import "github.com/alibaba/RedisShake/internal/log" +import "RedisShake/internal/log" func ArrayString(replyInterface interface{}, err error) []string { if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } replyArray := replyInterface.([]interface{}) replyArrayString := make([]string, len(replyArray)) diff --git a/internal/commands/keys.go b/internal/commands/keys.go index 0899e055..6a2d4545 100644 --- a/internal/commands/keys.go +++ b/internal/commands/keys.go @@ -1,9 +1,9 @@ package commands import ( + "RedisShake/internal/log" + "RedisShake/internal/utils" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/utils" "math" "strconv" "strings" @@ -73,7 +73,7 @@ func CalcKeys(argv []string) (cmaName string, group string, keys []string) { } keyCount, err := strconv.Atoi(argv[keynumIdx]) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } firstKey := spec.findKeysKeynumFirstKey step := spec.findKeysKeynumKeyStep diff --git a/internal/config/config.go b/internal/config/config.go index eb303abc..07d5c0f7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,147 +1,88 @@ package config import ( - "bytes" "fmt" - "github.com/pelletier/go-toml/v2" - "io/ioutil" + "github.com/mcuadros/go-defaults" + "github.com/rs/zerolog" + "github.com/spf13/viper" "os" - "runtime" ) -type tomlSource struct { - // sync mode - Version float32 `toml:"version"` - Address string `toml:"address"` - Username string `toml:"username"` - Password string `toml:"password"` - IsTLS bool `toml:"tls"` - ElastiCachePSync string `toml:"elasticache_psync"` +type AdvancedOptions struct { + Dir string `mapstructure:"dir" default:"data"` - // restore mode - RDBFilePath string `toml:"rdb_file_path"` -} - -type tomlTarget struct { - Type string `toml:"type"` - Version float32 `toml:"version"` - Username string `toml:"username"` - Address string `toml:"address"` - Password string `toml:"password"` - IsTLS bool `toml:"tls"` -} + Ncpu int `mapstructure:"ncpu" default:"4"` -type tomlAdvanced struct { - Dir string `toml:"dir"` - - Ncpu int `toml:"ncpu"` - - PprofPort int `toml:"pprof_port"` - MetricsPort int `toml:"metrics_port"` + PprofPort int `mapstructure:"pprof_port" default:"0"` + StatusPort int `mapstructure:"status_port" default:"6479"` // log - LogFile string `toml:"log_file"` - LogLevel string `toml:"log_level"` - LogInterval int `toml:"log_interval"` - - // rdb restore - RDBRestoreCommandBehavior string `toml:"rdb_restore_command_behavior"` - - // for writer - PipelineCountLimit uint64 `toml:"pipeline_count_limit"` - TargetRedisClientMaxQuerybufLen uint64 `toml:"target_redis_client_max_querybuf_len"` - TargetRedisProtoMaxBulkLen uint64 `toml:"target_redis_proto_max_bulk_len"` + LogFile string `mapstructure:"log_file" default:"shake.log"` + LogLevel string `mapstructure:"log_level" default:"info"` + LogInterval int `mapstructure:"log_interval" default:"5"` + + // redis-shake gets key and value from rdb file, and uses RESTORE command to + // create the key in target redis. Redis RESTORE will return a "Target key name + // is busy" error when key already exists. You can use this configuration item + // to change the default behavior of restore: + // panic: redis-shake will stop when meet "Target key name is busy" error. + // rewrite: redis-shake will replace the key with new value. + // ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. + RDBRestoreCommandBehavior string `mapstructure:"rdb_restore_command_behavior" default:"panic"` + + PipelineCountLimit uint64 `mapstructure:"pipeline_count_limit" default:"1024"` + TargetRedisClientMaxQuerybufLen int64 `mapstructure:"target_redis_client_max_querybuf_len" default:"1024000000"` + TargetRedisProtoMaxBulkLen uint64 `mapstructure:"target_redis_proto_max_bulk_len" default:"512000000"` + + AwsPSync string `mapstructure:"aws_psync" default:""` // "ip:port@xxxpsync;ip:port@xxxpsync" } -type tomlShakeConfig struct { - Type string - Source tomlSource - Target tomlTarget - Advanced tomlAdvanced +func (opt *AdvancedOptions) GetPSyncCommand(address string) string { + return fmt.Sprintf("psync %s 1 0", address) } -var Config tomlShakeConfig - -func init() { - Config.Type = "sync" - - // source - Config.Source.Version = 5.0 - Config.Source.Address = "" - Config.Source.Username = "" - Config.Source.Password = "" - Config.Source.IsTLS = false - Config.Source.ElastiCachePSync = "" - // restore - Config.Source.RDBFilePath = "" - - // target - Config.Target.Type = "standalone" - Config.Target.Version = 5.0 - Config.Target.Address = "" - Config.Target.Username = "" - Config.Target.Password = "" - Config.Target.IsTLS = false - - // advanced - Config.Advanced.Dir = "data" - Config.Advanced.Ncpu = 4 - Config.Advanced.PprofPort = 0 - Config.Advanced.MetricsPort = 0 - Config.Advanced.LogFile = "redis-shake.log" - Config.Advanced.LogLevel = "info" - Config.Advanced.LogInterval = 5 - Config.Advanced.RDBRestoreCommandBehavior = "rewrite" - Config.Advanced.PipelineCountLimit = 1024 - Config.Advanced.TargetRedisClientMaxQuerybufLen = 1024 * 1000 * 1000 - Config.Advanced.TargetRedisProtoMaxBulkLen = 512 * 1000 * 1000 +type ShakeOptions struct { + Transform string `mapstructure:"transform" default:""` + Advanced AdvancedOptions } -func LoadFromFile(filename string) { - - buf, err := ioutil.ReadFile(filename) - if err != nil { - panic(err.Error()) - } +var Opt ShakeOptions - decoder := toml.NewDecoder(bytes.NewReader(buf)) - decoder.SetStrict(true) - err = decoder.Decode(&Config) - if err != nil { - missingError, ok := err.(*toml.StrictMissingError) - if ok { - panic(fmt.Sprintf("decode config error:\n%s", missingError.String())) - } - panic(err.Error()) - } +func LoadConfig() *viper.Viper { + defaults.SetDefaults(&Opt) - // dir - err = os.MkdirAll(Config.Advanced.Dir, os.ModePerm) - if err != nil { - panic(err.Error()) + v := viper.New() + if len(os.Args) > 2 { + fmt.Println("Usage: redis-shake [config file]") + fmt.Println("Example: ") + fmt.Println(" redis-shake sync.toml # load config from sync.toml") + fmt.Println(" redis-shake # load config from environment variables") + os.Exit(1) } - err = os.Chdir(Config.Advanced.Dir) - if err != nil { - panic(err.Error()) - } - - // cpu core - var ncpu int - if Config.Advanced.Ncpu == 0 { - ncpu = runtime.NumCPU() - } else { - ncpu = Config.Advanced.Ncpu + consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: "2006-01-02 15:04:05"} + logger := zerolog.New(consoleWriter).With().Timestamp().Logger() + // load config from file + if len(os.Args) == 2 { + logger.Info().Msgf("load config from file: %s", os.Args[1]) + configFile := os.Args[1] + v.SetConfigFile(configFile) + err := v.ReadInConfig() + if err != nil { + panic(err) + } } - runtime.GOMAXPROCS(ncpu) - if Config.Source.Version < 2.8 { - panic("source redis version must be greater than 2.8") - } - if Config.Target.Version < 2.8 { - panic("target redis version must be greater than 2.8") + // load config from environment variables + if len(os.Args) == 1 { + logger.Warn().Msg("load config from environment variables") + v.SetConfigType("env") + v.AutomaticEnv() } - if Config.Type != "sync" && Config.Type != "restore" && Config.Type != "scan" { - panic("type must be sync/restore/scan") + // unmarshal config + err := v.Unmarshal(&Opt) + if err != nil { + panic(err) } + return v } diff --git a/internal/entry/entry.go b/internal/entry/entry.go index 28dfc46a..0356be54 100644 --- a/internal/entry/entry.go +++ b/internal/entry/entry.go @@ -1,22 +1,23 @@ package entry -import "fmt" +import ( + "RedisShake/internal/client/proto" + "RedisShake/internal/log" + "bytes" + "strings" +) type Entry struct { - Id uint64 - IsBase bool // whether the command is decoded from dump.rdb file - DbId int - Argv []string - TimestampMs uint64 + DbId int + Argv []string CmdName string Group string Keys []string Slots []int - // for statistics - Offset int64 - EncodedSize uint64 // the size of the entry after encode + // for stat + SerializedSize int64 } func NewEntry() *Entry { @@ -24,6 +25,26 @@ func NewEntry() *Entry { return e } -func (e *Entry) ToString() string { - return fmt.Sprintf("%v", e.Argv) +func (e *Entry) String() string { + str := strings.Join(e.Argv, " ") + if len(str) > 100 { + str = str[:100] + "..." + } + return str +} + +func (e *Entry) Serialize() []byte { + buf := new(bytes.Buffer) + writer := proto.NewWriter(buf) + argvInterface := make([]interface{}, len(e.Argv)) + + for inx, item := range e.Argv { + argvInterface[inx] = item + } + err := writer.WriteArgs(argvInterface) + if err != nil { + log.Panicf(err.Error()) + } + e.SerializedSize = int64(buf.Len()) + return buf.Bytes() } diff --git a/internal/filter/filter.go b/internal/filter/filter.go deleted file mode 100644 index 0d93a309..00000000 --- a/internal/filter/filter.go +++ /dev/null @@ -1,55 +0,0 @@ -package filter - -import ( - "github.com/alibaba/RedisShake/internal/entry" - lua "github.com/yuin/gopher-lua" -) - -const ( - Allow = 0 - Disallow = 1 - Error = 2 -) - -var luaInstance *lua.LState - -func LoadFromFile(luaFile string) { - luaInstance = lua.NewState() - err := luaInstance.DoFile(luaFile) - if err != nil { - panic(err) - } -} - -func Filter(e *entry.Entry) int { - if luaInstance == nil { - return Allow - } - keys := luaInstance.NewTable() - for _, key := range e.Keys { - keys.Append(lua.LString(key)) - } - - slots := luaInstance.NewTable() - for _, slot := range e.Slots { - slots.Append(lua.LNumber(slot)) - } - - f := luaInstance.GetGlobal("filter") - luaInstance.Push(f) - luaInstance.Push(lua.LNumber(e.Id)) // id - luaInstance.Push(lua.LBool(e.IsBase)) // is_base - luaInstance.Push(lua.LString(e.Group)) // group - luaInstance.Push(lua.LString(e.CmdName)) // cmd name - luaInstance.Push(keys) // keys - luaInstance.Push(slots) // slots - luaInstance.Push(lua.LNumber(e.DbId)) // dbid - luaInstance.Push(lua.LNumber(e.TimestampMs)) // timestamp_ms - - luaInstance.Call(8, 2) - - code := int(luaInstance.Get(1).(lua.LNumber)) - e.DbId = int(luaInstance.Get(2).(lua.LNumber)) - luaInstance.Pop(2) - return code -} diff --git a/internal/log/func.go b/internal/log/func.go index f9604648..899cc67f 100644 --- a/internal/log/func.go +++ b/internal/log/func.go @@ -1,16 +1,11 @@ package log import ( - "runtime/debug" - "strings" + "fmt" + "github.com/go-stack/stack" + "os" ) -func Assert(condition bool, msg string) { - if !condition { - Panicf("Assert failed: %s", msg) - } -} - func Debugf(format string, args ...interface{}) { logger.Debug().Msgf(format, args...) } @@ -24,20 +19,11 @@ func Warnf(format string, args ...interface{}) { } func Panicf(format string, args ...interface{}) { - stack := string(debug.Stack()) - stack = strings.ReplaceAll(stack, "\n\t", "]<-") - stack = strings.ReplaceAll(stack, "\n", " [") - logger.Info().Msg(stack) - - logger.Panic().Msgf(format, args...) -} - -func PanicError(err error) { - Panicf(err.Error()) -} - -func PanicIfError(err error) { - if err != nil { - PanicError(err) + frames := stack.Trace().TrimRuntime() + msgs := fmt.Sprintf(format, args...) + for _, frame := range frames { + msgs += fmt.Sprintf("\n%+v -> %n()", frame, frame) } + logger.Error().Msg(msgs) + os.Exit(1) } diff --git a/internal/log/init.go b/internal/log/init.go index 793f0c58..cf36661e 100644 --- a/internal/log/init.go +++ b/internal/log/init.go @@ -1,18 +1,19 @@ package log import ( + "RedisShake/internal/config" "fmt" - "github.com/alibaba/RedisShake/internal/config" "github.com/rs/zerolog" "os" + "path/filepath" ) var logger zerolog.Logger -func Init() { +func Init(level string, file string) { // log level - switch config.Config.Advanced.LogLevel { + switch level { case "debug": zerolog.SetGlobalLevel(zerolog.DebugLevel) case "info": @@ -20,15 +21,31 @@ func Init() { case "warn": zerolog.SetGlobalLevel(zerolog.WarnLevel) default: - panic(fmt.Sprintf("unknown log level: %s", config.Config.Advanced.LogLevel)) + panic(fmt.Sprintf("unknown log level: %s", level)) } + // dir + dir, err := filepath.Abs(config.Opt.Advanced.Dir) + if err != nil { + panic(fmt.Sprintf("failed to determine current directory: %v", err)) + } + err = os.RemoveAll(dir) + if err != nil { + panic(fmt.Sprintf("remove dir failed. dir=[%s], error=[%v]", dir, err)) + } + err = os.MkdirAll(dir, 0777) + if err != nil { + panic(fmt.Sprintf("mkdir failed. dir=[%s], error=[%v]", dir, err)) + } + path := filepath.Join(dir, file) + // log file consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: "2006-01-02 15:04:05"} - fileWriter, err := os.OpenFile(config.Config.Advanced.LogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + fileWriter, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) if err != nil { - panic(fmt.Sprintf("open log file failed: %s", err)) + panic(fmt.Sprintf("open log file failed. file=[%s], err=[%s]", path, err)) } multi := zerolog.MultiLevelWriter(consoleWriter, fileWriter) logger = zerolog.New(multi).With().Timestamp().Logger() + Infof("log_level: [%v], log_file: [%v]", level, path) } diff --git a/internal/rdb/rdb.go b/internal/rdb/rdb.go index 45bd426b..46e5753a 100644 --- a/internal/rdb/rdb.go +++ b/internal/rdb/rdb.go @@ -1,16 +1,15 @@ package rdb import ( + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" + "RedisShake/internal/rdb/types" + "RedisShake/internal/utils" "bufio" "bytes" "encoding/binary" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" - "github.com/alibaba/RedisShake/internal/rdb/types" - "github.com/alibaba/RedisShake/internal/statistics" - "github.com/alibaba/RedisShake/internal/utils" "io" "os" "strconv" @@ -32,7 +31,7 @@ const ( ) type Loader struct { - replStreamDbId int // https://github.com/alibaba/RedisShake/pull/430#issuecomment-1099014464 + replStreamDbId int // https://RedisShake/pull/430#issuecomment-1099014464 nowDBId int expireMs int64 @@ -44,15 +43,22 @@ type Loader struct { ch chan *entry.Entry dumpBuffer bytes.Buffer + + name string + updateFunc func(int64) } -func NewLoader(filPath string, ch chan *entry.Entry) *Loader { +func NewLoader(name string, updateFunc func(int64), filPath string, ch chan *entry.Entry) *Loader { ld := new(Loader) ld.ch = ch ld.filPath = filPath + ld.name = name + ld.updateFunc = updateFunc return ld } +// ParseRDB parse rdb file +// return repl stream db id func (ld *Loader) ParseRDB() int { var err error ld.fp, err = os.OpenFile(ld.filPath, os.O_RDONLY, 0666) @@ -66,43 +72,41 @@ func (ld *Loader) ParseRDB() int { } }() rd := bufio.NewReader(ld.fp) - //magic + version + // magic + version buf := make([]byte, 9) _, err = io.ReadFull(rd, buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } if !bytes.Equal(buf[:5], []byte("REDIS")) { log.Panicf("verify magic string, invalid file format. bytes=[%v]", buf[:5]) } version, err := strconv.Atoi(string(buf[5:])) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } - log.Infof("RDB version: %d", version) + log.Infof("[%s] RDB version: %d", ld.name, version) // read entries ld.parseRDBEntry(rd) - // force update rdb_sent_size for issue: https://github.com/alibaba/RedisShake/issues/485 - fi, err := os.Stat(ld.filPath) - if err != nil { - log.Panicf("NewRDBReader: os.Stat error: %s", err.Error()) - } - statistics.Metrics.RdbSendSize = uint64(fi.Size()) return ld.replStreamDbId } func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { // for stat - UpdateRDBSentSize := func() { + updateProcessSize := func() { + if ld.updateFunc == nil { + return + } offset, err := ld.fp.Seek(0, io.SeekCurrent) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } - statistics.UpdateRDBSentSize(uint64(offset)) + ld.updateFunc(offset) } - defer UpdateRDBSentSize() + defer updateProcessSize() + // read one entry tick := time.Tick(time.Second * 1) for true { @@ -119,22 +123,21 @@ func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { var err error ld.replStreamDbId, err = strconv.Atoi(value) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } - log.Infof("RDB repl-stream-db: %d", ld.replStreamDbId) + log.Infof("[%s] RDB repl-stream-db: [%s]", ld.name, value) } else if key == "lua" { e := entry.NewEntry() e.Argv = []string{"script", "load", value} - e.IsBase = true ld.ch <- e - log.Infof("LUA script: [%s]", value) + log.Infof("[%s] LUA script: [%s]", ld.name, value) } else { - log.Infof("RDB AUX fields. key=[%s], value=[%s]", key, value) + log.Infof("[%s] RDB AUX: key=[%s], value=[%s]", ld.name, key, value) } case kFlagResizeDB: dbSize := structure.ReadLength(rd) expireSize := structure.ReadLength(rd) - log.Infof("RDB resize db. db_size=[%d], expire_size=[%d]", dbSize, expireSize) + log.Infof("[%s] RDB resize db: db_size=[%d], expire_size=[%d]", ld.name, dbSize, expireSize) case kFlagExpireMs: ld.expireMs = int64(structure.ReadUint64(rd)) - time.Now().UnixMilli() if ld.expireMs < 0 { @@ -154,40 +157,37 @@ func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { var value bytes.Buffer anotherReader := io.TeeReader(rd, &value) o := types.ParseObject(anotherReader, typeByte, key) - if uint64(value.Len()) > config.Config.Advanced.TargetRedisProtoMaxBulkLen { + if uint64(value.Len()) > config.Opt.Advanced.TargetRedisProtoMaxBulkLen { cmds := o.Rewrite() for _, cmd := range cmds { e := entry.NewEntry() - e.IsBase = true e.DbId = ld.nowDBId e.Argv = cmd ld.ch <- e } if ld.expireMs != 0 { e := entry.NewEntry() - e.IsBase = true e.DbId = ld.nowDBId e.Argv = []string{"PEXPIRE", key, strconv.FormatInt(ld.expireMs, 10)} ld.ch <- e } } else { e := entry.NewEntry() - e.IsBase = true e.DbId = ld.nowDBId v := ld.createValueDump(typeByte, value.Bytes()) e.Argv = []string{"restore", key, strconv.FormatInt(ld.expireMs, 10), v} - if config.Config.Advanced.RDBRestoreCommandBehavior == "rewrite" { - if config.Config.Target.Version < 3.0 { - log.Panicf("RDB restore command behavior is rewrite, but target redis version is %f, not support REPLACE modifier", config.Config.Target.Version) - } + if config.Opt.Advanced.RDBRestoreCommandBehavior == "rewrite" { + //if config.Opt.Target.Version < 3.0 { + // log.Panicf("RDB restore command behavior is rewrite, but target redis version is %f, not support REPLACE modifier", config.Config.Target.Version) + //} e.Argv = append(e.Argv, "replace") } - if ld.idle != 0 && config.Config.Target.Version >= 5.0 { - e.Argv = append(e.Argv, "idletime", strconv.FormatInt(ld.idle, 10)) - } - if ld.freq != 0 && config.Config.Target.Version >= 5.0 { - e.Argv = append(e.Argv, "freq", strconv.FormatInt(ld.freq, 10)) - } + //if ld.idle != 0 && config.Config.Target.Version >= 5.0 { + // e.Argv = append(e.Argv, "idletime", strconv.FormatInt(ld.idle, 10)) + //} + //if ld.freq != 0 && config.Config.Target.Version >= 5.0 { + // e.Argv = append(e.Argv, "freq", strconv.FormatInt(ld.freq, 10)) + //} ld.ch <- e } ld.expireMs = 0 @@ -196,7 +196,7 @@ func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { } select { case <-tick: - UpdateRDBSentSize() + updateProcessSize() default: } } diff --git a/internal/rdb/structure/byte.go b/internal/rdb/structure/byte.go index d52df8ec..d8b0bed5 100644 --- a/internal/rdb/structure/byte.go +++ b/internal/rdb/structure/byte.go @@ -1,7 +1,7 @@ package structure import ( - "github.com/alibaba/RedisShake/internal/log" + "RedisShake/internal/log" "io" ) @@ -14,7 +14,7 @@ func ReadBytes(rd io.Reader, n int) []byte { buf := make([]byte, n) _, err := io.ReadFull(rd, buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } return buf } diff --git a/internal/rdb/structure/float.go b/internal/rdb/structure/float.go index 8351d96a..a254f863 100644 --- a/internal/rdb/structure/float.go +++ b/internal/rdb/structure/float.go @@ -1,8 +1,8 @@ package structure import ( + "RedisShake/internal/log" "encoding/binary" - "github.com/alibaba/RedisShake/internal/log" "io" "math" "strconv" @@ -27,7 +27,7 @@ func ReadFloat(rd io.Reader) float64 { v, err := strconv.ParseFloat(string(buf), 64) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } return v } @@ -37,7 +37,7 @@ func ReadDouble(rd io.Reader) float64 { var buf = make([]byte, 8) _, err := io.ReadFull(rd, buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } num := binary.LittleEndian.Uint64(buf) return math.Float64frombits(num) diff --git a/internal/rdb/structure/length.go b/internal/rdb/structure/length.go index e62543fe..dc598c08 100644 --- a/internal/rdb/structure/length.go +++ b/internal/rdb/structure/length.go @@ -1,9 +1,9 @@ package structure import ( + "RedisShake/internal/log" "encoding/binary" "fmt" - "github.com/alibaba/RedisShake/internal/log" "io" ) @@ -22,7 +22,7 @@ func ReadLength(rd io.Reader) uint64 { log.Panicf("illegal length special=true, encoding: %d", length) } if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } return length } diff --git a/internal/rdb/structure/listpack.go b/internal/rdb/structure/listpack.go index 2d30374a..ab4600a5 100644 --- a/internal/rdb/structure/listpack.go +++ b/internal/rdb/structure/listpack.go @@ -1,8 +1,8 @@ package structure import ( + "RedisShake/internal/log" "bufio" - "github.com/alibaba/RedisShake/internal/log" "io" "math" "strconv" diff --git a/internal/rdb/structure/string.go b/internal/rdb/structure/string.go index f53559e3..101148a1 100644 --- a/internal/rdb/structure/string.go +++ b/internal/rdb/structure/string.go @@ -1,7 +1,7 @@ package structure import ( - "github.com/alibaba/RedisShake/internal/log" + "RedisShake/internal/log" "io" "strconv" ) @@ -16,7 +16,7 @@ const ( func ReadString(rd io.Reader) string { length, special, err := readEncodedLength(rd) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } if special { switch length { diff --git a/internal/rdb/structure/ziplist.go b/internal/rdb/structure/ziplist.go index 141ea026..4dcd37f3 100644 --- a/internal/rdb/structure/ziplist.go +++ b/internal/rdb/structure/ziplist.go @@ -1,9 +1,9 @@ package structure import ( + "RedisShake/internal/log" "bufio" "encoding/binary" - "github.com/alibaba/RedisShake/internal/log" "io" "strconv" "strings" diff --git a/internal/rdb/types/hash.go b/internal/rdb/types/hash.go index e27b67e8..2d373ae6 100644 --- a/internal/rdb/types/hash.go +++ b/internal/rdb/types/hash.go @@ -1,8 +1,8 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/interface.go b/internal/rdb/types/interface.go index 7d9fb3d5..1e237d75 100644 --- a/internal/rdb/types/interface.go +++ b/internal/rdb/types/interface.go @@ -1,7 +1,7 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" + "RedisShake/internal/log" "io" ) diff --git a/internal/rdb/types/list.go b/internal/rdb/types/list.go index 57e23c08..4f6b8dff 100644 --- a/internal/rdb/types/list.go +++ b/internal/rdb/types/list.go @@ -1,8 +1,8 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/module2.go b/internal/rdb/types/module2.go index 26ab46e0..73e01b06 100644 --- a/internal/rdb/types/module2.go +++ b/internal/rdb/types/module2.go @@ -1,8 +1,8 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/set.go b/internal/rdb/types/set.go index ac02f378..2f83b843 100644 --- a/internal/rdb/types/set.go +++ b/internal/rdb/types/set.go @@ -1,8 +1,8 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/stream.go b/internal/rdb/types/stream.go index 9b0a6b03..402affbc 100644 --- a/internal/rdb/types/stream.go +++ b/internal/rdb/types/stream.go @@ -1,10 +1,10 @@ package types import ( + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "encoding/binary" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" "strconv" ) diff --git a/internal/rdb/types/string.go b/internal/rdb/types/string.go index 2adf51c6..6321fd36 100644 --- a/internal/rdb/types/string.go +++ b/internal/rdb/types/string.go @@ -1,7 +1,7 @@ package types import ( - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/zset.go b/internal/rdb/types/zset.go index 34f99b37..16f1d8a1 100644 --- a/internal/rdb/types/zset.go +++ b/internal/rdb/types/zset.go @@ -1,9 +1,9 @@ package types import ( + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/reader/interface.go b/internal/reader/interface.go index 9d583e43..7ddc0d0a 100644 --- a/internal/reader/interface.go +++ b/internal/reader/interface.go @@ -1,7 +1,11 @@ package reader -import "github.com/alibaba/RedisShake/internal/entry" +import ( + "RedisShake/internal/entry" + "RedisShake/internal/status" +) type Reader interface { + status.Statusable StartRead() chan *entry.Entry } diff --git a/internal/reader/psync.go b/internal/reader/psync.go deleted file mode 100644 index 9745945c..00000000 --- a/internal/reader/psync.go +++ /dev/null @@ -1,247 +0,0 @@ -package reader - -import ( - "bufio" - "github.com/alibaba/RedisShake/internal/client" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb" - "github.com/alibaba/RedisShake/internal/reader/rotate" - "github.com/alibaba/RedisShake/internal/statistics" - "io" - "io/ioutil" - "os" - "strconv" - "strings" - "time" -) - -type psyncReader struct { - client *client.Redis - address string - ch chan *entry.Entry - DbId int - - rd *bufio.Reader - receivedOffset int64 - elastiCachePSync string -} - -func NewPSyncReader(address string, username string, password string, isTls bool, ElastiCachePSync string) Reader { - r := new(psyncReader) - r.address = address - r.elastiCachePSync = ElastiCachePSync - r.client = client.NewRedisClient(address, username, password, isTls) - r.rd = r.client.BufioReader() - log.Infof("psyncReader connected to redis successful. address=[%s]", address) - return r -} - -func (r *psyncReader) StartRead() chan *entry.Entry { - r.ch = make(chan *entry.Entry, 1024) - - go func() { - r.clearDir() - go r.sendReplconfAck() - r.saveRDB() - startOffset := r.receivedOffset - go r.saveAOF(r.rd) - r.sendRDB() - time.Sleep(1 * time.Second) // wait for saveAOF create aof file - r.sendAOF(startOffset) - }() - - return r.ch -} - -func (r *psyncReader) clearDir() { - files, err := ioutil.ReadDir("./") - if err != nil { - log.PanicError(err) - } - - for _, f := range files { - if strings.HasSuffix(f.Name(), ".rdb") || strings.HasSuffix(f.Name(), ".aof") { - err = os.Remove(f.Name()) - if err != nil { - log.PanicError(err) - } - log.Warnf("remove file. filename=[%s]", f.Name()) - } - } -} - -func (r *psyncReader) saveRDB() { - log.Infof("start save RDB. address=[%s]", r.address) - argv := []string{"replconf", "listening-port", "10007"} // 10007 is magic number - log.Infof("send %v", argv) - reply := r.client.DoWithStringReply(argv...) - if reply != "OK" { - log.Warnf("send replconf command to redis server failed. address=[%s], reply=[%s], error=[]", r.address, reply) - } - - // send psync - argv = []string{"PSYNC", "?", "-1"} - if r.elastiCachePSync != "" { - argv = []string{r.elastiCachePSync, "?", "-1"} - } - r.client.Send(argv...) - log.Infof("send %v", argv) - // format: \n\n\n$\r\n - for true { - // \n\n\n$ - b, err := r.rd.ReadByte() - if err != nil { - log.PanicError(err) - } - if b == '\n' { - continue - } - if b == '-' { - reply, err := r.rd.ReadString('\n') - if err != nil { - log.PanicError(err) - } - reply = strings.TrimSpace(reply) - log.Panicf("psync error. address=[%s], reply=[%s]", r.address, reply) - } - if b != '+' { - log.Panicf("invalid psync reply. address=[%s], b=[%s]", r.address, string(b)) - } - break - } - reply, err := r.rd.ReadString('\n') - if err != nil { - log.PanicError(err) - } - reply = strings.TrimSpace(reply) - log.Infof("receive [%s]", reply) - masterOffset, err := strconv.Atoi(strings.Split(reply, " ")[2]) - if err != nil { - log.PanicError(err) - } - r.receivedOffset = int64(masterOffset) - - log.Infof("source db is doing bgsave. address=[%s]", r.address) - statistics.Metrics.IsDoingBgsave = true - - timeStart := time.Now() - // format: \n\n\n$\r\n - for true { - // \n\n\n$ - b, err := r.rd.ReadByte() - if err != nil { - log.PanicError(err) - } - if b == '\n' { - continue - } - if b != '$' { - log.Panicf("invalid rdb format. address=[%s], b=[%s]", r.address, string(b)) - } - break - } - statistics.Metrics.IsDoingBgsave = false - log.Infof("source db bgsave finished. timeUsed=[%.2f]s, address=[%s]", time.Since(timeStart).Seconds(), r.address) - lengthStr, err := r.rd.ReadString('\n') - if err != nil { - log.PanicError(err) - } - lengthStr = strings.TrimSpace(lengthStr) - length, err := strconv.ParseInt(lengthStr, 10, 64) - if err != nil { - log.PanicError(err) - } - log.Infof("received rdb length. length=[%d]", length) - statistics.SetRDBFileSize(uint64(length)) - - // create rdb file - rdbFilePath := "dump.rdb" - log.Infof("create dump.rdb file. filename_path=[%s]", rdbFilePath) - rdbFileHandle, err := os.OpenFile(rdbFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) - if err != nil { - log.PanicError(err) - } - - // read rdb - remainder := length - const bufSize int64 = 32 * 1024 * 1024 // 32MB - buf := make([]byte, bufSize) - for remainder != 0 { - readOnce := bufSize - if remainder < readOnce { - readOnce = remainder - } - n, err := r.rd.Read(buf[:readOnce]) - if err != nil { - log.PanicError(err) - } - remainder -= int64(n) - statistics.UpdateRDBReceivedSize(uint64(length - remainder)) - _, err = rdbFileHandle.Write(buf[:n]) - if err != nil { - log.PanicError(err) - } - } - err = rdbFileHandle.Close() - if err != nil { - log.PanicError(err) - } - log.Infof("save RDB finished. address=[%s], total_bytes=[%d]", r.address, length) -} - -func (r *psyncReader) saveAOF(rd io.Reader) { - log.Infof("start save AOF. address=[%s]", r.address) - // create aof file - aofWriter := rotate.NewAOFWriter(r.receivedOffset) - defer aofWriter.Close() - buf := make([]byte, 16*1024) // 16KB is enough for writing file - for { - n, err := rd.Read(buf) - if err != nil { - log.PanicError(err) - } - r.receivedOffset += int64(n) - statistics.UpdateAOFReceivedOffset(uint64(r.receivedOffset)) - aofWriter.Write(buf[:n]) - } -} - -func (r *psyncReader) sendRDB() { - // start parse rdb - log.Infof("start send RDB. address=[%s]", r.address) - rdbLoader := rdb.NewLoader("dump.rdb", r.ch) - r.DbId = rdbLoader.ParseRDB() - log.Infof("send RDB finished. address=[%s], repl-stream-db=[%d]", r.address, r.DbId) -} - -func (r *psyncReader) sendAOF(offset int64) { - aofReader := rotate.NewAOFReader(offset) - defer aofReader.Close() - r.client.SetBufioReader(bufio.NewReader(aofReader)) - for { - argv := client.ArrayString(r.client.Receive()) - // select - if strings.EqualFold(argv[0], "select") { - DbId, err := strconv.Atoi(argv[1]) - if err != nil { - log.PanicError(err) - } - r.DbId = DbId - continue - } - - e := entry.NewEntry() - e.Argv = argv - e.DbId = r.DbId - e.Offset = aofReader.Offset() - r.ch <- e - } -} - -func (r *psyncReader) sendReplconfAck() { - for range time.Tick(time.Millisecond * 100) { - // send ack receivedOffset - r.client.Send("replconf", "ack", strconv.FormatInt(r.receivedOffset, 10)) - } -} diff --git a/internal/reader/rdb_reader.go b/internal/reader/rdb_reader.go index ee6444ed..23b5a30f 100644 --- a/internal/reader/rdb_reader.go +++ b/internal/reader/rdb_reader.go @@ -1,48 +1,72 @@ package reader import ( - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb" - "github.com/alibaba/RedisShake/internal/statistics" - "os" - "path/filepath" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/rdb" + "RedisShake/internal/utils" + "fmt" + "github.com/dustin/go-humanize" ) -type rdbReader struct { - path string - ch chan *entry.Entry +type RdbReaderOptions struct { + Filepath string `mapstructure:"filepath" default:""` } -func NewRDBReader(path string) Reader { - log.Infof("NewRDBReader: path=[%s]", path) - absolutePath, err := filepath.Abs(path) - if err != nil { - log.Panicf("NewRDBReader: filepath.Abs error: %s", err.Error()) +type rdbReader struct { + ch chan *entry.Entry + + stat struct { + Name string `json:"name"` + Status string `json:"status"` + Filepath string `json:"filepath"` + FileSizeBytes int64 `json:"file_size_bytes"` + FileSizeHuman string `json:"file_size_human"` + FileSentBytes int64 `json:"file_sent_bytes"` + FileSentHuman string `json:"file_sent_human"` + Percent string `json:"percent"` } - log.Infof("NewRDBReader: absolute path=[%s]", absolutePath) +} + +func NewRDBReader(opts *RdbReaderOptions) Reader { + absolutePath := utils.GetAbsPath(opts.Filepath) r := new(rdbReader) - r.path = absolutePath + r.stat.Name = "rdb_reader" + r.stat.Status = "init" + r.stat.Filepath = absolutePath + r.stat.FileSizeBytes = int64(utils.GetFileSize(absolutePath)) + r.stat.FileSizeHuman = humanize.Bytes(uint64(r.stat.FileSizeBytes)) return r } func (r *rdbReader) StartRead() chan *entry.Entry { + log.Infof("[%s] start read", r.stat.Name) r.ch = make(chan *entry.Entry, 1024) + updateFunc := func(offset int64) { + r.stat.FileSentBytes = offset + r.stat.FileSentHuman = humanize.Bytes(uint64(offset)) + r.stat.Percent = fmt.Sprintf("%.2f%%", float64(offset)/float64(r.stat.FileSizeBytes)*100) + r.stat.Status = fmt.Sprintf("[%s] rdb file synced: %s", r.stat.Name, r.stat.Percent) + } + rdbLoader := rdb.NewLoader(r.stat.Name, updateFunc, r.stat.Filepath, r.ch) go func() { - // start parse rdb - log.Infof("start send RDB. path=[%s]", r.path) - fi, err := os.Stat(r.path) - if err != nil { - log.Panicf("NewRDBReader: os.Stat error: %s", err.Error()) - } - statistics.Metrics.RdbFileSize = uint64(fi.Size()) - statistics.Metrics.RdbReceivedSize = uint64(fi.Size()) - rdbLoader := rdb.NewLoader(r.path, r.ch) _ = rdbLoader.ParseRDB() - log.Infof("send RDB finished. path=[%s]", r.path) + log.Infof("[%s] rdb file parse done", r.stat.Name) close(r.ch) }() return r.ch } + +func (r *rdbReader) Status() interface{} { + return r.stat +} + +func (r *rdbReader) StatusString() string { + return r.stat.Status +} + +func (r *rdbReader) StatusConsistent() bool { + return r.stat.FileSentBytes == r.stat.FileSizeBytes +} diff --git a/internal/reader/scan_cluster_reader.go b/internal/reader/scan_cluster_reader.go new file mode 100644 index 00000000..898a9a88 --- /dev/null +++ b/internal/reader/scan_cluster_reader.go @@ -0,0 +1,73 @@ +package reader + +import ( + "RedisShake/internal/entry" + "RedisShake/internal/utils" + "sync" +) + +type ScanClusterReaderOptions struct { + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` +} + +type scanClusterReader struct { + readers []Reader +} + +func NewScanClusterReader(opts *ScanClusterReaderOptions) Reader { + addresses, _ := utils.GetRedisClusterNodes(opts.Address, opts.Username, opts.Password, opts.Tls) + + rd := &scanClusterReader{} + for _, address := range addresses { + rd.readers = append(rd.readers, NewScanStandaloneReader(&ScanStandaloneReaderOptions{ + Address: address, + Username: opts.Username, + Password: opts.Password, + Tls: opts.Tls, + })) + } + return rd +} + +func (rd *scanClusterReader) StartRead() chan *entry.Entry { + ch := make(chan *entry.Entry, 1024) + var wg sync.WaitGroup + for _, r := range rd.readers { + wg.Add(1) + go func(r Reader) { + for e := range r.StartRead() { + ch <- e + } + wg.Done() + }(r) + } + go func() { + wg.Wait() + close(ch) + }() + return ch +} + +func (rd *scanClusterReader) Status() interface{} { + stat := make([]interface{}, 0) + for _, r := range rd.readers { + stat = append(stat, r.Status()) + } + return stat +} + +func (rd *scanClusterReader) StatusString() string { + return "scanClusterReader" +} + +func (rd *scanClusterReader) StatusConsistent() bool { + for _, r := range rd.readers { + if !r.StatusConsistent() { + return false + } + } + return true +} diff --git a/internal/reader/scan_reader.go b/internal/reader/scan_reader.go deleted file mode 100644 index b43d0297..00000000 --- a/internal/reader/scan_reader.go +++ /dev/null @@ -1,145 +0,0 @@ -package reader - -import ( - "strconv" - "strings" - - "github.com/alibaba/RedisShake/internal/client" - "github.com/alibaba/RedisShake/internal/client/proto" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/statistics" -) - -const ( - // cluster_enabled: Indicate Redis cluster is enabled. reference from https://redis.io/commands/info/ - clusterMode = "cluster_enabled:1" -) - -type dbKey struct { - db int - key string - isSelect bool -} - -type scanReader struct { - address string - - // client for scan keys - clientScan *client.Redis - innerChannel chan *dbKey - isCluster bool - - // client for dump keys - clientDump *client.Redis - clientDumpDbid int - ch chan *entry.Entry -} - -func NewScanReader(address string, username string, password string, isTls bool) Reader { - r := new(scanReader) - r.address = address - r.clientScan = client.NewRedisClient(address, username, password, isTls) - r.clientDump = client.NewRedisClient(address, username, password, isTls) - log.Infof("scanReader connected to redis successful. address=[%s]", address) - - r.isCluster = r.IsCluster() - return r -} - -// IsCluster is for determining whether the server is in cluster mode. -func (r *scanReader) IsCluster() bool { - reply := r.clientScan.DoWithStringReply("INFO", "Cluster") - return strings.Contains(reply, clusterMode) -} - -func (r *scanReader) StartRead() chan *entry.Entry { - r.ch = make(chan *entry.Entry, 1024) - r.innerChannel = make(chan *dbKey, 1024) - go r.scan() - go r.fetch() - return r.ch -} - -func (r *scanReader) scan() { - scanDbIdUpper := 15 - if r.isCluster { - log.Infof("scanReader node are in cluster mode, only scan db 0") - scanDbIdUpper = 0 - } - for dbId := 0; dbId <= scanDbIdUpper; dbId++ { - if !r.isCluster { - reply := r.clientScan.DoWithStringReply("SELECT", strconv.Itoa(dbId)) - if reply != "OK" { - log.Panicf("scanReader select db failed. db=[%d]", dbId) - } - - r.clientDump.Send("SELECT", strconv.Itoa(dbId)) - r.innerChannel <- &dbKey{dbId, "", true} - } - - var cursor uint64 = 0 - for { - var keys []string - cursor, keys = r.clientScan.Scan(cursor) - for _, key := range keys { - r.clientDump.Send("DUMP", key) - r.clientDump.Send("PTTL", key) - r.innerChannel <- &dbKey{dbId, key, false} - } - - // stat - statistics.Metrics.ScanDbId = dbId - statistics.Metrics.ScanCursor = cursor - - if cursor == 0 { - break - } - } - } - close(r.innerChannel) -} - -func (r *scanReader) fetch() { - var id uint64 = 0 - for item := range r.innerChannel { - if item.isSelect { - // select - receive, err := client.String(r.clientDump.Receive()) - if err != nil { - log.Panicf("scanReader select db failed. db=[%d], err=[%v]", item.db, err) - } - if receive != "OK" { - log.Panicf("scanReader select db failed. db=[%d]", item.db) - } - } else { - // dump - receive, err := client.String(r.clientDump.Receive()) - if err != proto.Nil && err != nil { // error! - log.PanicIfError(err) - } - - // pttl - pttl, pttlErr := client.Int64(r.clientDump.Receive()) - log.PanicIfError(pttlErr) - if pttl < 0 { - pttl = 0 - } - - if err == proto.Nil { // key not exist - continue - } - - id += 1 - argv := []string{"RESTORE", item.key, strconv.FormatInt(pttl, 10), receive} - r.ch <- &entry.Entry{ - Id: id, - IsBase: false, - DbId: item.db, - Argv: argv, - } - } - } - log.Infof("scanReader fetch finished. address=[%s]", r.address) - close(r.ch) -} diff --git a/internal/reader/scan_standalone_reader.go b/internal/reader/scan_standalone_reader.go new file mode 100644 index 00000000..dcd172b9 --- /dev/null +++ b/internal/reader/scan_standalone_reader.go @@ -0,0 +1,168 @@ +package reader + +import ( + "RedisShake/internal/client" + "RedisShake/internal/client/proto" + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "fmt" + "math/bits" + "strconv" + "strings" +) + +type dbKey struct { + db int + key string + isSelect bool +} + +type ScanStandaloneReaderOptions struct { + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` +} + +type scanStandaloneReader struct { + isCluster bool + + // client for scan keys + clientScan *client.Redis + innerChannel chan *dbKey + + // client for dump keys + clientDump *client.Redis + clientDumpDbid int + ch chan *entry.Entry + + stat struct { + Name string `json:"name"` + Finished bool `json:"finished"` + DbId int `json:"dbId"` + Cursor uint64 `json:"cursor"` + PercentByDbId string `json:"percent"` + } +} + +func NewScanStandaloneReader(opts *ScanStandaloneReaderOptions) Reader { + r := new(scanStandaloneReader) + r.stat.Name = "reader_" + strings.Replace(opts.Address, ":", "_", -1) + r.clientScan = client.NewRedisClient(opts.Address, opts.Username, opts.Password, opts.Tls) + r.clientDump = client.NewRedisClient(opts.Address, opts.Username, opts.Password, opts.Tls) + r.isCluster = r.clientScan.IsCluster() + return r +} + +func (r *scanStandaloneReader) StartRead() chan *entry.Entry { + r.ch = make(chan *entry.Entry, 1024) + r.innerChannel = make(chan *dbKey, 1024) + go r.scan() + go r.fetch() + return r.ch +} + +func (r *scanStandaloneReader) scan() { + scanDbIdUpper := 15 + if r.isCluster { + log.Infof("scanStandaloneReader node are in cluster mode, only scan db 0") + scanDbIdUpper = 0 + } + for dbId := 0; dbId <= scanDbIdUpper; dbId++ { + if !r.isCluster { + reply := r.clientScan.DoWithStringReply("SELECT", strconv.Itoa(dbId)) + if reply != "OK" { + log.Panicf("scanStandaloneReader select db failed. db=[%d]", dbId) + } + + r.clientDump.Send("SELECT", strconv.Itoa(dbId)) + r.innerChannel <- &dbKey{dbId, "", true} + } + + var cursor uint64 = 0 + for { + var keys []string + cursor, keys = r.clientScan.Scan(cursor) + for _, key := range keys { + r.clientDump.Send("DUMP", key) + r.clientDump.Send("PTTL", key) + r.innerChannel <- &dbKey{dbId, key, false} + } + + // stat + r.stat.Cursor = cursor + r.stat.DbId = dbId + r.stat.PercentByDbId = fmt.Sprintf("%.2f%%", float64(bits.Reverse64(cursor))/float64(^uint(0))*100) + + if cursor == 0 { + break + } + } + } + r.stat.Finished = true + close(r.innerChannel) +} + +func (r *scanStandaloneReader) fetch() { + var id uint64 = 0 + for item := range r.innerChannel { + if item.isSelect { + // select + receive, err := client.String(r.clientDump.Receive()) + if err != nil { + log.Panicf("scanStandaloneReader select db failed. db=[%d], err=[%v]", item.db, err) + } + if receive != "OK" { + log.Panicf("scanStandaloneReader select db failed. db=[%d]", item.db) + } + } else { + // dump + receive, err := client.String(r.clientDump.Receive()) + if err != proto.Nil && err != nil { // error! + log.Panicf(err.Error()) + } + + // pttl + pttl, pttlErr := client.Int64(r.clientDump.Receive()) + if pttlErr != nil { // error! + log.Panicf(pttlErr.Error()) + } + if pttl < 0 { + pttl = 0 + } + + if err == proto.Nil { // key not exist + continue + } + + id += 1 + argv := []string{"RESTORE", item.key, strconv.FormatInt(pttl, 10), receive} + if config.Opt.Advanced.RDBRestoreCommandBehavior == "rewrite" { + argv = append(argv, "replace") + } + log.Debugf("[%s] send command: [%v], dbid: [%v]", r.stat.Name, argv, item.db) + r.ch <- &entry.Entry{ + DbId: item.db, + Argv: argv, + } + } + } + log.Infof("[%s] scanStandaloneReader fetch finished.", r.stat.Name) + close(r.ch) +} + +func (r *scanStandaloneReader) Status() interface{} { + return r.stat +} + +func (r *scanStandaloneReader) StatusString() string { + if r.stat.Finished { + return fmt.Sprintf("[%s] finished", r.stat.Name) + } + return fmt.Sprintf("[%s] dbid: [%d], percent: [%s]", r.stat.Name, r.stat.DbId, r.stat.PercentByDbId) +} + +func (r *scanStandaloneReader) StatusConsistent() bool { + return r.stat.Finished +} diff --git a/internal/reader/sync_cluster_reader.go b/internal/reader/sync_cluster_reader.go new file mode 100644 index 00000000..74cd3b9f --- /dev/null +++ b/internal/reader/sync_cluster_reader.go @@ -0,0 +1,77 @@ +package reader + +import ( + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/utils" + "sync" +) + +type SyncClusterReaderOptions struct { + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` +} + +type syncClusterReader struct { + readers []Reader +} + +func NewSyncClusterReader(opts *SyncClusterReaderOptions) Reader { + addresses, _ := utils.GetRedisClusterNodes(opts.Address, opts.Username, opts.Password, opts.Tls) + log.Debugf("get redis cluster nodes:") + for _, address := range addresses { + log.Debugf("%s", address) + } + rd := &syncClusterReader{} + for _, address := range addresses { + rd.readers = append(rd.readers, NewSyncStandaloneReader(&SyncStandaloneReaderOptions{ + Address: address, + Username: opts.Username, + Password: opts.Password, + Tls: opts.Tls, + })) + } + return rd +} + +func (rd *syncClusterReader) StartRead() chan *entry.Entry { + ch := make(chan *entry.Entry, 1024) + var wg sync.WaitGroup + for _, r := range rd.readers { + wg.Add(1) + go func(r Reader) { + defer wg.Done() + for e := range r.StartRead() { + ch <- e + } + }(r) + } + go func() { + wg.Wait() + close(ch) + }() + return ch +} + +func (rd *syncClusterReader) Status() interface{} { + stat := make([]interface{}, 0) + for _, r := range rd.readers { + stat = append(stat, r.Status()) + } + return stat +} + +func (rd *syncClusterReader) StatusString() string { + return "syncClusterReader" +} + +func (rd *syncClusterReader) StatusConsistent() bool { + for _, r := range rd.readers { + if !r.StatusConsistent() { + return false + } + } + return true +} diff --git a/internal/reader/sync_standalone_reader.go b/internal/reader/sync_standalone_reader.go new file mode 100644 index 00000000..c7362e8f --- /dev/null +++ b/internal/reader/sync_standalone_reader.go @@ -0,0 +1,297 @@ +package reader + +import ( + "RedisShake/internal/client" + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/rdb" + "RedisShake/internal/utils" + "RedisShake/internal/utils/file_rotate" + "bufio" + "fmt" + "github.com/dustin/go-humanize" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +type SyncStandaloneReaderOptions struct { + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` +} + +type syncStandaloneReader struct { + client *client.Redis + + ch chan *entry.Entry + DbId int + + rd *bufio.Reader + + stat struct { + Name string `json:"name"` + Address string `json:"address"` + Dir string `json:"dir"` + + // status + Status string `json:"status"` + + // rdb info + RdbFilePath string `json:"rdb_file_path"` + RdbFileSizeBytes int64 `json:"rdb_file_size_bytes"` // bytes of the rdb file + RdbFIleSizeHuman string `json:"rdb_file_size_human"` + RdbReceivedBytes int64 `json:"rdb_received_bytes"` // bytes of RDB received from master + RdbReceivedHuman string `json:"rdb_received_human"` + RdbSentBytes int64 `json:"rdb_sent_bytes"` // bytes of RDB sent to chan + RdbSentHuman string `json:"rdb_sent_human"` + + // aof info + AofReceivedOffset int64 `json:"aof_received_offset"` // offset of AOF received from master + AofSentOffset int64 `json:"aof_sent_offset"` // offset of AOF sent to chan + AofReceivedBytes int64 `json:"aof_received_bytes"` // bytes of AOF received from master + AofReceivedHuman string `json:"aof_received_human"` + } +} + +func NewSyncStandaloneReader(opts *SyncStandaloneReaderOptions) Reader { + r := new(syncStandaloneReader) + r.client = client.NewRedisClient(opts.Address, opts.Username, opts.Password, opts.Tls) + r.rd = r.client.BufioReader() + r.stat.Name = "reader_" + strings.Replace(opts.Address, ":", "_", -1) + r.stat.Address = opts.Address + r.stat.Status = "init" + r.stat.Dir = utils.GetAbsPath(r.stat.Name) + utils.CreateEmptyDir(r.stat.Dir) + return r +} + +func (r *syncStandaloneReader) StartRead() chan *entry.Entry { + r.ch = make(chan *entry.Entry, 1024) + go func() { + r.sendReplconfListenPort() + r.sendPSync() + go r.sendReplconfAck() // start sent replconf ack + r.receiveRDB() + startOffset := r.stat.AofReceivedOffset + go r.receiveAOF(r.rd) + r.sendRDB() + r.sendAOF(startOffset) + }() + + return r.ch +} + +func (r *syncStandaloneReader) sendReplconfListenPort() { + // use status_port as redis-shake port + argv := []string{"replconf", "listening-port", strconv.Itoa(config.Opt.Advanced.StatusPort)} + r.client.Send(argv...) + reply, err := r.client.Receive() + if err != nil { + log.Warnf("[%s] send replconf command to redis server failed. reply=[%s], error=[%v]", r.stat.Name, reply, err) + } + if reply != "OK" { + log.Warnf("[%s] send replconf command to redis server failed. reply=[%s]", r.stat.Name, reply) + } +} + +func (r *syncStandaloneReader) sendPSync() { + // send PSync + argv := []string{"PSYNC", "?", "-1"} + if config.Opt.Advanced.AwsPSync != "" { + argv = []string{config.Opt.Advanced.AwsPSync, "?", "-1"} // TODO AWS PSYNC + } + r.client.Send(argv...) + + // format: \n\n\n+\r\n + for true { // TODO better way to parse psync reply + // \n\n\n+ + b, err := r.rd.ReadByte() + if err != nil { + log.Panicf(err.Error()) + } + if b == '\n' { + continue + } + if b == '-' { + reply, err := r.rd.ReadString('\n') + if err != nil { + log.Panicf(err.Error()) + } + reply = strings.TrimSpace(reply) + log.Panicf("psync error. name=[%s], reply=[%s]", r.stat.Name, reply) + } + if b != '+' { + log.Panicf("invalid psync reply. name=[%s], b=[%s]", r.stat.Name, string(b)) + } + break + } + reply, err := r.rd.ReadString('\n') + if err != nil { + log.Panicf(err.Error()) + } + reply = strings.TrimSpace(reply) + + masterOffset, err := strconv.Atoi(strings.Split(reply, " ")[2]) + if err != nil { + log.Panicf(err.Error()) + } + r.stat.AofReceivedOffset = int64(masterOffset) +} + +func (r *syncStandaloneReader) receiveRDB() { + log.Infof("[%s] source db is doing bgsave.", r.stat.Name) + r.stat.Status = "source db is doing bgsave" + timeStart := time.Now() + // format: \n\n\n$\r\n + for true { + b, err := r.rd.ReadByte() + if err != nil { + log.Panicf(err.Error()) + } + if b == '\n' { + continue + } + if b != '$' { + log.Panicf("[%s] invalid rdb format. b=[%s]", r.stat.Name, string(b)) + } + break + } + log.Infof("[%s] source db bgsave finished. timeUsed=[%.2f]s", r.stat.Name, time.Since(timeStart).Seconds()) + lengthStr, err := r.rd.ReadString('\n') + if err != nil { + log.Panicf(err.Error()) + } + lengthStr = strings.TrimSpace(lengthStr) + length, err := strconv.ParseInt(lengthStr, 10, 64) + if err != nil { + log.Panicf(err.Error()) + } + log.Infof("[%s] rdb length=[%d]", r.stat.Name, length) + r.stat.RdbFileSizeBytes = length + r.stat.RdbFIleSizeHuman = humanize.IBytes(uint64(length)) + + // create rdb file + r.stat.RdbFilePath, err = filepath.Abs(r.stat.Name + "/dump.rdb") + if err != nil { + log.Panicf(err.Error()) + } + log.Infof("[%s] start receiving RDB. path=[%s]", r.stat.Name, r.stat.RdbFilePath) + rdbFileHandle, err := os.OpenFile(r.stat.RdbFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + log.Panicf(err.Error()) + } + + // receive rdb + r.stat.Status = fmt.Sprintf("[%s]: receiving RDB", r.stat.Name) + remainder := length + const bufSize int64 = 32 * 1024 * 1024 // 32MB + buf := make([]byte, bufSize) + for remainder != 0 { + readOnce := bufSize + if remainder < readOnce { + readOnce = remainder + } + n, err := r.rd.Read(buf[:readOnce]) + if err != nil { + log.Panicf(err.Error()) + } + remainder -= int64(n) + _, err = rdbFileHandle.Write(buf[:n]) + if err != nil { + log.Panicf(err.Error()) + } + + r.stat.RdbReceivedBytes += int64(n) + r.stat.RdbReceivedHuman = humanize.IBytes(uint64(r.stat.RdbReceivedBytes)) + } + err = rdbFileHandle.Close() + if err != nil { + log.Panicf(err.Error()) + } + log.Infof("[%s] save RDB finished.", r.stat.Name) +} + +func (r *syncStandaloneReader) receiveAOF(rd io.Reader) { + log.Infof("[%s] start receiving aof data, and save to file", r.stat.Name) + aofWriter := rotate.NewAOFWriter(r.stat.Name, r.stat.Dir, r.stat.AofReceivedOffset) + defer aofWriter.Close() + buf := make([]byte, 16*1024) // 16KB is enough for writing file + for { + n, err := rd.Read(buf) + if err != nil { + log.Panicf(err.Error()) + } + r.stat.AofReceivedBytes += int64(n) + r.stat.AofReceivedHuman = humanize.IBytes(uint64(r.stat.AofReceivedBytes)) + aofWriter.Write(buf[:n]) + r.stat.AofReceivedOffset += int64(n) + } +} + +func (r *syncStandaloneReader) sendRDB() { + // start parse rdb + log.Infof("[%s] start sending RDB to target", r.stat.Name) + r.stat.Status = fmt.Sprintf("[%s]: sending RDB to target", r.stat.Name) + updateFunc := func(offset int64) { + r.stat.RdbSentBytes = offset + r.stat.RdbSentHuman = humanize.IBytes(uint64(offset)) + } + rdbLoader := rdb.NewLoader(r.stat.Name, updateFunc, r.stat.RdbFilePath, r.ch) + r.DbId = rdbLoader.ParseRDB() + log.Infof("[%s] send RDB finished", r.stat.Name) +} + +func (r *syncStandaloneReader) sendAOF(offset int64) { + time.Sleep(1 * time.Second) // wait for receiveAOF create aof file + aofReader := rotate.NewAOFReader(r.stat.Name, r.stat.Dir, offset) + defer aofReader.Close() + r.client.SetBufioReader(bufio.NewReader(aofReader)) + for { + argv := client.ArrayString(r.client.Receive()) + r.stat.AofSentOffset = aofReader.Offset() + // select + if strings.EqualFold(argv[0], "select") { + DbId, err := strconv.Atoi(argv[1]) + if err != nil { + log.Panicf(err.Error()) + } + r.DbId = DbId + continue + } + + e := entry.NewEntry() + e.Argv = argv + e.DbId = r.DbId + r.ch <- e + r.stat.Status = fmt.Sprintf("[%s]: sending aof to target", r.stat.Name) + } +} + +// sendReplconfAck send replconf ack to master to keep heartbeat between redis-shake and source redis. +func (r *syncStandaloneReader) sendReplconfAck() { + for range time.Tick(time.Millisecond * 100) { + if r.stat.AofReceivedOffset != 0 { + r.client.Send("replconf", "ack", strconv.FormatInt(r.stat.AofReceivedOffset, 10)) + } + } +} + +func (r *syncStandaloneReader) Status() interface{} { + return r.stat +} + +func (r *syncStandaloneReader) StatusString() string { + return r.stat.Status +} + +func (r *syncStandaloneReader) StatusConsistent() bool { + return r.stat.AofReceivedOffset != 0 && + r.stat.AofReceivedOffset == r.stat.AofSentOffset && + len(r.ch) == 0 +} diff --git a/internal/statistics/statistics.go b/internal/statistics/statistics.go deleted file mode 100644 index 9af801d5..00000000 --- a/internal/statistics/statistics.go +++ /dev/null @@ -1,157 +0,0 @@ -package statistics - -import ( - "encoding/json" - "fmt" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/log" - "math/bits" - "net/http" - "strings" - "time" -) - -type metrics struct { - // info - Address string `json:"address"` - - // entries - EntryId uint64 `json:"entry_id"` - AllowEntriesCount uint64 `json:"allow_entries_count"` - DisallowEntriesCount uint64 `json:"disallow_entries_count"` - - // rdb - IsDoingBgsave bool `json:"is_doing_bgsave"` - RdbFileSize uint64 `json:"rdb_file_size"` - RdbReceivedSize uint64 `json:"rdb_received_size"` - RdbSendSize uint64 `json:"rdb_send_size"` - - // aof - AofReceivedOffset uint64 `json:"aof_received_offset"` - AofAppliedOffset uint64 `json:"aof_applied_offset"` - - // for performance debug - InQueueEntriesCount uint64 `json:"in_queue_entries_count"` - UnansweredBytesCount uint64 `json:"unanswered_bytes_count"` - - // scan cursor - ScanDbId int `json:"scan_db_id"` - ScanCursor uint64 `json:"scan_cursor"` - - // for log - Msg string `json:"msg"` -} - -var Metrics = &metrics{} - -func Handler(w http.ResponseWriter, _ *http.Request) { - w.Header().Add("Content-Type", "application/json") - err := json.NewEncoder(w).Encode(Metrics) - if err != nil { - log.PanicError(err) - } -} - -func Init() { - go func() { - seconds := config.Config.Advanced.LogInterval - if seconds <= 0 { - log.Infof("statistics disabled. seconds=[%d]", seconds) - } - - lastAllowEntriesCount := Metrics.AllowEntriesCount - lastDisallowEntriesCount := Metrics.DisallowEntriesCount - - for range time.Tick(time.Duration(seconds) * time.Second) { - // scan - if config.Config.Type == "scan" { - Metrics.Msg = fmt.Sprintf("syncing. dbId=[%d], percent=[%.2f]%%, allowOps=[%.2f], disallowOps=[%.2f], entryId=[%d], InQueueEntriesCount=[%d], unansweredBytesCount=[%d]bytes", - Metrics.ScanDbId, - float64(bits.Reverse64(Metrics.ScanCursor))/float64(^uint(0))*100, - float32(Metrics.AllowEntriesCount-lastAllowEntriesCount)/float32(seconds), - float32(Metrics.DisallowEntriesCount-lastDisallowEntriesCount)/float32(seconds), - Metrics.EntryId, - Metrics.InQueueEntriesCount, - Metrics.UnansweredBytesCount) - log.Infof(strings.Replace(Metrics.Msg, "%", "%%", -1)) - lastAllowEntriesCount = Metrics.AllowEntriesCount - lastDisallowEntriesCount = Metrics.DisallowEntriesCount - continue - } - // sync or restore - if Metrics.RdbFileSize == 0 { - Metrics.Msg = "source db is doing bgsave" - } else if Metrics.RdbSendSize > Metrics.RdbReceivedSize { - Metrics.Msg = fmt.Sprintf("receiving rdb. percent=[%.2f]%%, rdbFileSize=[%.3f]G, rdbReceivedSize=[%.3f]G", - float64(Metrics.RdbReceivedSize)/float64(Metrics.RdbFileSize)*100, - float64(Metrics.RdbFileSize)/1024/1024/1024, - float64(Metrics.RdbReceivedSize)/1024/1024/1024) - } else if Metrics.RdbFileSize > Metrics.RdbSendSize { - Metrics.Msg = fmt.Sprintf("syncing rdb. percent=[%.2f]%%, allowOps=[%.2f], disallowOps=[%.2f], entryId=[%d], InQueueEntriesCount=[%d], unansweredBytesCount=[%d]bytes, rdbFileSize=[%.3f]G, rdbSendSize=[%.3f]G", - float64(Metrics.RdbSendSize)*100/float64(Metrics.RdbFileSize), - float32(Metrics.AllowEntriesCount-lastAllowEntriesCount)/float32(seconds), - float32(Metrics.DisallowEntriesCount-lastDisallowEntriesCount)/float32(seconds), - Metrics.EntryId, - Metrics.InQueueEntriesCount, - Metrics.UnansweredBytesCount, - float64(Metrics.RdbFileSize)/1024/1024/1024, - float64(Metrics.RdbSendSize)/1024/1024/1024) - } else { - Metrics.Msg = fmt.Sprintf("syncing aof. allowOps=[%.2f], disallowOps=[%.2f], entryId=[%d], InQueueEntriesCount=[%d], unansweredBytesCount=[%d]bytes, diff=[%d], aofReceivedOffset=[%d], aofAppliedOffset=[%d]", - float32(Metrics.AllowEntriesCount-lastAllowEntriesCount)/float32(seconds), - float32(Metrics.DisallowEntriesCount-lastDisallowEntriesCount)/float32(seconds), - Metrics.EntryId, - Metrics.InQueueEntriesCount, - Metrics.UnansweredBytesCount, - Metrics.AofReceivedOffset-Metrics.AofAppliedOffset, - Metrics.AofReceivedOffset, - Metrics.AofAppliedOffset) - } - log.Infof(strings.Replace(Metrics.Msg, "%", "%%", -1)) - lastAllowEntriesCount = Metrics.AllowEntriesCount - lastDisallowEntriesCount = Metrics.DisallowEntriesCount - } - }() -} - -// entry id - -func UpdateEntryId(id uint64) { - Metrics.EntryId = id -} -func AddAllowEntriesCount() { - Metrics.AllowEntriesCount++ -} -func AddDisallowEntriesCount() { - Metrics.DisallowEntriesCount++ -} - -// rdb - -func SetRDBFileSize(size uint64) { - Metrics.RdbFileSize = size -} -func UpdateRDBReceivedSize(size uint64) { - Metrics.RdbReceivedSize = size -} -func UpdateRDBSentSize(offset uint64) { - Metrics.RdbSendSize = offset -} - -// aof - -func UpdateAOFReceivedOffset(offset uint64) { - Metrics.AofReceivedOffset = offset -} -func UpdateAOFAppliedOffset(offset uint64) { - Metrics.AofAppliedOffset = offset -} - -// for debug - -func UpdateInQueueEntriesCount(count uint64) { - Metrics.InQueueEntriesCount = count -} -func UpdateUnansweredBytesCount(count uint64) { - Metrics.UnansweredBytesCount = count -} diff --git a/internal/status/entry_count.go b/internal/status/entry_count.go new file mode 100644 index 00000000..3b0c312c --- /dev/null +++ b/internal/status/entry_count.go @@ -0,0 +1,35 @@ +package status + +import ( + "fmt" + "time" +) + +type EntryCount struct { + Allow uint64 `json:"allow"` + Disallow uint64 `json:"disallow"` + AllowOps float64 `json:"allow_ops"` + DisallowOps float64 `json:"disallow_ops"` + + // update ops + lastAllow uint64 + lastDisallow uint64 + lastUpdateTimestampSec float64 +} + +// call this function every second +func (e *EntryCount) updateOPS() { + nowTimestampSec := float64(time.Now().UnixNano()) / 1e9 + if e.lastUpdateTimestampSec != 0 { + timeIntervalSec := nowTimestampSec - e.lastUpdateTimestampSec + e.AllowOps = float64(e.Allow-e.lastAllow) / timeIntervalSec + e.DisallowOps = float64(e.Disallow-e.lastDisallow) / timeIntervalSec + e.lastAllow = e.Allow + e.lastDisallow = e.Disallow + } + e.lastUpdateTimestampSec = nowTimestampSec +} + +func (e *EntryCount) String() string { + return fmt.Sprintf("allow: %.2fops/s, disallow: %.2fops/s", e.AllowOps, e.DisallowOps) +} diff --git a/internal/status/handler.go b/internal/status/handler.go new file mode 100644 index 00000000..c0069cf8 --- /dev/null +++ b/internal/status/handler.go @@ -0,0 +1,53 @@ +package status + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "encoding/json" + "fmt" + "net/http" + "time" +) + +func Handler(w http.ResponseWriter, _ *http.Request) { + w.Header().Add("Content-Type", "application/json") + + bytesChannel := make(chan []byte, 1) + + ch <- func() { + stat.Consistent = theReader.StatusConsistent() && theWriter.StatusConsistent() + jsonBytes, err := json.Marshal(stat) + if err != nil { + log.Warnf("marshal status info failed, err=[%v]", err) + bytesChannel <- []byte(fmt.Sprintf(`{"error": "%v"}`, err)) + return + } + bytesChannel <- jsonBytes + } + + select { + case bytes := <-bytesChannel: + _, err := w.Write(bytes) + if err != nil { + log.Warnf("write status info failed, err=[%v]", err) + } + case <-time.After(time.Second * 3): + log.Warnf("write status info timeout") + w.WriteHeader(http.StatusRequestTimeout) + } +} + +func setStatusPort() { + if config.Opt.Advanced.StatusPort != 0 { + go func() { + addr := fmt.Sprintf(":%d", config.Opt.Advanced.StatusPort) + if err := http.ListenAndServe(addr, http.HandlerFunc(Handler)); err != nil { + log.Panicf(err.Error()) + } + }() + log.Infof("status information: http://localhost:%v", config.Opt.Advanced.StatusPort) + log.Infof("status information: watch -n 0.3 'curl -s http://localhost:%v | python -m json.tool'", config.Opt.Advanced.StatusPort) + } else { + log.Infof("not set status port") + } +} diff --git a/internal/status/status.go b/internal/status/status.go new file mode 100644 index 00000000..e21b5de2 --- /dev/null +++ b/internal/status/status.go @@ -0,0 +1,110 @@ +package status + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "time" +) + +type Statusable interface { + Status() interface{} + StatusString() string + StatusConsistent() bool +} + +type Stat struct { + Time string `json:"start_time"` + Consistent bool `json:"consistent"` + // transform + TotalEntriesCount EntryCount `json:"total_entries_count"` + PerCmdEntriesCount map[string]EntryCount `json:"per_cmd_entries_count"` + // reader + Reader interface{} `json:"reader"` + // writer + Writer interface{} `json:"writer"` +} + +var ch = make(chan func(), 1000) +var stat = new(Stat) +var theReader Statusable +var theWriter Statusable + +func AddEntryCount(cmd string, allow bool) { + ch <- func() { + if stat.PerCmdEntriesCount == nil { + stat.PerCmdEntriesCount = make(map[string]EntryCount) + } + cmdEntryCount, ok := stat.PerCmdEntriesCount[cmd] + if !ok { + cmdEntryCount = EntryCount{} + stat.PerCmdEntriesCount[cmd] = cmdEntryCount + } + if allow { + stat.TotalEntriesCount.Allow += 1 + cmdEntryCount.Allow += 1 + } else { + stat.TotalEntriesCount.Disallow += 1 + cmdEntryCount.Disallow += 1 + } + stat.PerCmdEntriesCount[cmd] = cmdEntryCount + } +} + +func Init(r Statusable, w Statusable) { + theReader = r + theWriter = w + setStatusPort() + stat.Time = time.Now().Format("2006-01-02 15:04:05") + + // for update reader/writer stat + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + lastConsistent := false + for { + select { + case <-ticker.C: + ch <- func() { + // update reader/writer stat + stat.Reader = theReader.Status() + stat.Writer = theWriter.Status() + stat.Consistent = lastConsistent && theReader.StatusConsistent() && theWriter.StatusConsistent() + lastConsistent = stat.Consistent + // update OPS + stat.TotalEntriesCount.updateOPS() + for _, cmdEntryCount := range stat.PerCmdEntriesCount { + cmdEntryCount.updateOPS() + } + } + } + } + }() + + // for log to screen + go func() { + if config.Opt.Advanced.LogInterval <= 0 { + log.Infof("log interval is 0, will not log to screen") + return + } + ticker := time.NewTicker(time.Duration(config.Opt.Advanced.LogInterval) * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + ch <- func() { + log.Infof("%s, %s, %s", + stat.TotalEntriesCount.String(), + theReader.StatusString(), + theWriter.StatusString()) + } + } + } + }() + + // run all func in ch + go func() { + for f := range ch { + f() + } + }() +} diff --git a/internal/transform/transform.go b/internal/transform/transform.go new file mode 100644 index 00000000..39697fb0 --- /dev/null +++ b/internal/transform/transform.go @@ -0,0 +1,63 @@ +package transform + +import ( + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + lua "github.com/yuin/gopher-lua" + "strings" +) + +const ( + Allow = 0 + Disallow = 1 + Error = 2 +) + +var luaInstance *lua.LState + +func Init() { + luaString := config.Opt.Transform + luaString = strings.TrimSpace(luaString) + if len(luaString) == 0 { + log.Infof("no transform script") + return + } + luaInstance = lua.NewState() + err := luaInstance.DoString(luaString) + if err != nil { + log.Panicf("load transform script failed: %v", err) + } + log.Infof("load transform script success") +} + +func Transform(e *entry.Entry) int { + if luaInstance == nil { + return Allow + } + + keys := luaInstance.NewTable() + for _, key := range e.Keys { + keys.Append(lua.LString(key)) + } + + slots := luaInstance.NewTable() + for _, slot := range e.Slots { + slots.Append(lua.LNumber(slot)) + } + + f := luaInstance.GetGlobal("filter") + luaInstance.Push(f) + luaInstance.Push(lua.LString(e.Group)) // group + luaInstance.Push(lua.LString(e.CmdName)) // cmd name + luaInstance.Push(keys) // keys + luaInstance.Push(slots) // slots + luaInstance.Push(lua.LNumber(e.DbId)) // dbid + + luaInstance.Call(8, 2) + + code := int(luaInstance.Get(1).(lua.LNumber)) + e.DbId = int(luaInstance.Get(2).(lua.LNumber)) + luaInstance.Pop(2) + return code +} diff --git a/internal/utils/cluster_nodes.go b/internal/utils/cluster_nodes.go new file mode 100644 index 00000000..1c0eaead --- /dev/null +++ b/internal/utils/cluster_nodes.go @@ -0,0 +1,74 @@ +package utils + +import ( + "RedisShake/internal/client" + "RedisShake/internal/log" + "fmt" + "strconv" + "strings" +) + +func GetRedisClusterNodes(address string, username string, password string, Tls bool) (addresses []string, slots [][]int) { + c := client.NewRedisClient(address, username, password, Tls) + reply := c.DoWithStringReply("cluster", "nodes") + reply = strings.TrimSpace(reply) + slotsCount := 0 + for _, line := range strings.Split(reply, "\n") { + line = strings.TrimSpace(line) + words := strings.Split(line, " ") + if !strings.Contains(words[2], "master") { + continue + } + if len(words) < 9 { + log.Panicf("invalid cluster nodes line: %s", line) + } + log.Infof("redisClusterWriter load cluster nodes. line=%v", line) + + // address + address := strings.Split(words[1], "@")[0] + // handle ipv6 address + tok := strings.Split(address, ":") + if len(tok) > 2 { + // ipv6 address + port := tok[len(tok)-1] + + ipv6Addr := strings.Join(tok[:len(tok)-1], ":") + address = fmt.Sprintf("[%s]:%s", ipv6Addr, port) + } + addresses = append(addresses, address) + + // parse slots + slot := make([]int, 0) + for i := 8; i < len(words); i++ { + words[i] = strings.TrimSpace(words[i]) + var start, end int + var err error + if strings.Contains(words[i], "-") { + seg := strings.Split(words[i], "-") + start, err = strconv.Atoi(seg[0]) + if err != nil { + log.Panicf(err.Error()) + } + end, err = strconv.Atoi(seg[1]) + if err != nil { + log.Panicf(err.Error()) + } + } else { + start, err = strconv.Atoi(words[i]) + if err != nil { + log.Panicf(err.Error()) + } + end = start + } + for j := start; j <= end; j++ { + slot = append(slot, j) + slotsCount++ + } + slots = append(slots, slot) + } + } + if slotsCount != 16384 { + log.Panicf("invalid cluster nodes slots. slots_count=%v, address=%v", slotsCount, address) + } + return addresses, slots +} diff --git a/internal/utils/file.go b/internal/utils/file.go index 47fb7c07..28d3fec5 100644 --- a/internal/utils/file.go +++ b/internal/utils/file.go @@ -1,18 +1,49 @@ package utils import ( - "github.com/alibaba/RedisShake/internal/log" + "RedisShake/internal/log" "os" + "path/filepath" ) -func DoesFileExist(fileName string) bool { - _, err := os.Stat(fileName) +func CreateEmptyDir(dir string) { + if IsExist(dir) { + err := os.RemoveAll(dir) + if err != nil { + log.Panicf("remove dir failed. dir=[%s], error=[%v]", dir, err) + } + } + err := os.MkdirAll(dir, 0777) + if err != nil { + log.Panicf("mkdir failed. dir=[%s], error=[%v]", dir, err) + } + log.Debugf("CreateEmptyDir: dir=[%s]", dir) +} + +func IsExist(path string) bool { + _, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return false } else { - log.PanicError(err) + log.Panicf(err.Error()) } } return true } + +func GetFileSize(path string) uint64 { + fi, err := os.Stat(path) + if err != nil { + log.Panicf(err.Error()) + } + return uint64(fi.Size()) +} + +func GetAbsPath(path string) string { + absolutePath, err := filepath.Abs(path) + if err != nil { + log.Panicf(err.Error()) + } + return absolutePath +} diff --git a/internal/reader/rotate/aof_reader.go b/internal/utils/file_rotate/aof_reader.go similarity index 55% rename from internal/reader/rotate/aof_reader.go rename to internal/utils/file_rotate/aof_reader.go index f5ba676c..f0c79fce 100644 --- a/internal/reader/rotate/aof_reader.go +++ b/internal/utils/file_rotate/aof_reader.go @@ -1,44 +1,48 @@ package rotate import ( + "RedisShake/internal/log" + "RedisShake/internal/utils" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/utils" "io" "os" "time" ) type AOFReader struct { + name string + dir string file *os.File offset int64 pos int64 - filename string + filepath string } -func NewAOFReader(offset int64) *AOFReader { +func NewAOFReader(name string, dir string, offset int64) *AOFReader { r := new(AOFReader) + r.name = name + r.dir = dir r.openFile(offset) return r } func (r *AOFReader) openFile(offset int64) { - r.filename = fmt.Sprintf("%d.aof", offset) + r.filepath = fmt.Sprintf("%s/%d.aof", r.dir, r.offset) var err error - r.file, err = os.OpenFile(r.filename, os.O_RDONLY, 0644) + r.file, err = os.OpenFile(r.filepath, os.O_RDONLY, 0644) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.offset = offset r.pos = 0 - log.Infof("AOFReader open file. aof_filename=[%s]", r.filename) + log.Infof("[%s] open file for read. filename=[%s]", r.name, r.filepath) } func (r *AOFReader) readNextFile(offset int64) { - filename := fmt.Sprintf("%d.aof", offset) - if utils.DoesFileExist(filename) { + filepath := fmt.Sprintf("%s/%d.aof", r.dir, r.offset) + if utils.IsExist(filepath) { r.Close() - err := os.Remove(r.filename) + err := os.Remove(r.filepath) if err != nil { return } @@ -49,18 +53,18 @@ func (r *AOFReader) readNextFile(offset int64) { func (r *AOFReader) Read(buf []byte) (n int, err error) { n, err = r.file.Read(buf) for err == io.EOF { - if r.filename != fmt.Sprintf("%d.aof", r.offset) { + if r.filepath != fmt.Sprintf("%s/%d.aof", r.dir, r.offset) { r.readNextFile(r.offset) } time.Sleep(time.Millisecond * 10) _, err = r.file.Seek(0, 1) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } n, err = r.file.Read(buf) } if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.offset += int64(n) r.pos += int64(n) @@ -77,8 +81,8 @@ func (r *AOFReader) Close() { } err := r.file.Close() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.file = nil - log.Infof("AOFReader close file. aof_filename=[%s]", r.filename) + log.Infof("[%s] close file. filename=[%s]", r.name, r.filepath) } diff --git a/internal/reader/rotate/aof_writer.go b/internal/utils/file_rotate/aof_writer.go similarity index 55% rename from internal/reader/rotate/aof_writer.go rename to internal/utils/file_rotate/aof_writer.go index 0e58474f..b4828964 100644 --- a/internal/reader/rotate/aof_writer.go +++ b/internal/utils/file_rotate/aof_writer.go @@ -1,42 +1,47 @@ package rotate import ( + "RedisShake/internal/log" "fmt" - "github.com/alibaba/RedisShake/internal/log" "os" ) const MaxFileSize = 1024 * 1024 * 1024 // 1G type AOFWriter struct { + name string + dir string + file *os.File offset int64 - filename string + filepath string filesize int64 } -func NewAOFWriter(offset int64) *AOFWriter { - w := &AOFWriter{} +func NewAOFWriter(name string, dir string, offset int64) *AOFWriter { + w := new(AOFWriter) + w.name = name + w.dir = dir w.openFile(offset) return w } func (w *AOFWriter) openFile(offset int64) { - w.filename = fmt.Sprintf("%d.aof", offset) + w.filepath = fmt.Sprintf("%s/%d.aof", w.dir, w.offset) var err error - w.file, err = os.OpenFile(w.filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + w.file, err = os.OpenFile(w.filepath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } w.offset = offset w.filesize = 0 - log.Infof("AOFWriter open file. filename=[%s]", w.filename) + log.Infof("[%s] open file for write. filename=[%s]", w.name, w.filepath) } func (w *AOFWriter) Write(buf []byte) { _, err := w.file.Write(buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } w.offset += int64(len(buf)) w.filesize += int64(len(buf)) @@ -46,7 +51,7 @@ func (w *AOFWriter) Write(buf []byte) { } err = w.file.Sync() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } } @@ -56,11 +61,11 @@ func (w *AOFWriter) Close() { } err := w.file.Sync() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } err = w.file.Close() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } - log.Infof("AOFWriter close file. filename=[%s], filesize=[%d]", w.filename, w.filesize) + log.Infof("[%s] close file. filename=[%s], filesize=[%d]", w.name, w.filepath, w.filesize) } diff --git a/internal/utils/filelock.go b/internal/utils/filelock.go new file mode 100644 index 00000000..69b1acb4 --- /dev/null +++ b/internal/utils/filelock.go @@ -0,0 +1,46 @@ +package utils + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "github.com/theckman/go-flock" + "os" + "path/filepath" +) + +var filelock *flock.Flock + +func ChdirAndAcquireFileLock() { + // dir + dir, err := filepath.Abs(config.Opt.Advanced.Dir) + if err != nil { + log.Panicf("failed to determine current directory: %v", err) + } + // create dir + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + log.Panicf("failed to create dir. dir: %v, err: %v", dir, err) + } + filelock = flock.New(filepath.Join(dir, "pid.lockfile")) + locked, err := filelock.TryLock() + if err != nil { + log.Panicf("failed to lock pid file: %v", err) + } + if !locked { + log.Warnf("failed to lock pid file") + } + err = os.Chdir(dir) // change dir + if err != nil { + log.Panicf("failed to change dir. dir: %v, err: %v", dir, err) + } + log.Infof("changed work dir to [%s]", dir) +} + +func ReleaseFileLock() { + if filelock != nil { + err := filelock.Unlock() + if err != nil { + log.Warnf("failed to unlock pid file: %v", err) + } + } +} diff --git a/internal/utils/ncpu.go b/internal/utils/ncpu.go new file mode 100644 index 00000000..0d2adb39 --- /dev/null +++ b/internal/utils/ncpu.go @@ -0,0 +1,17 @@ +package utils + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "runtime" +) + +func SetNcpu() { + if config.Opt.Advanced.Ncpu != 0 { + log.Infof("set ncpu to %d", config.Opt.Advanced.Ncpu) + runtime.GOMAXPROCS(config.Opt.Advanced.Ncpu) + log.Infof("set GOMAXPROCS to %v", config.Opt.Advanced.Ncpu) + } else { + log.Infof("GOMAXPROCS defaults to the value of runtime.NumCPU %v", runtime.NumCPU()) + } +} diff --git a/internal/utils/pprof.go b/internal/utils/pprof.go new file mode 100644 index 00000000..a7884307 --- /dev/null +++ b/internal/utils/pprof.go @@ -0,0 +1,23 @@ +package utils + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "fmt" + "net/http" +) + +func SetPprofPort() { + // pprof_port + if config.Opt.Advanced.PprofPort != 0 { + go func() { + err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Opt.Advanced.PprofPort), nil) + if err != nil { + log.Panicf(err.Error()) + } + }() + log.Infof("pprof information: http://localhost:%d/debug/pprof/", config.Opt.Advanced.PprofPort) + } else { + log.Infof("not set pprof port") + } +} diff --git a/internal/writer/interface.go b/internal/writer/interface.go index 679d2001..523f14fb 100644 --- a/internal/writer/interface.go +++ b/internal/writer/interface.go @@ -1,8 +1,12 @@ package writer -import "github.com/alibaba/RedisShake/internal/entry" +import ( + "RedisShake/internal/entry" + "RedisShake/internal/status" +) type Writer interface { + status.Statusable Write(entry *entry.Entry) Close() } diff --git a/internal/writer/redis.go b/internal/writer/redis.go deleted file mode 100644 index c3502451..00000000 --- a/internal/writer/redis.go +++ /dev/null @@ -1,96 +0,0 @@ -package writer - -import ( - "bytes" - "github.com/alibaba/RedisShake/internal/client" - "github.com/alibaba/RedisShake/internal/client/proto" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/statistics" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" -) - -type redisWriter struct { - client *client.Redis - DbId int - - cmdBuffer *bytes.Buffer - chWaitReply chan *entry.Entry - chWg sync.WaitGroup - - UpdateUnansweredBytesCount uint64 // have sent in bytes -} - -func NewRedisWriter(address string, username string, password string, isTls bool) Writer { - rw := new(redisWriter) - rw.client = client.NewRedisClient(address, username, password, isTls) - log.Infof("redisWriter connected to redis successful. address=[%s]", address) - rw.cmdBuffer = new(bytes.Buffer) - rw.chWaitReply = make(chan *entry.Entry, config.Config.Advanced.PipelineCountLimit) - rw.chWg.Add(1) - go rw.flushInterval() - return rw -} - -func (w *redisWriter) Write(e *entry.Entry) { - // switch db if we need - if w.DbId != e.DbId { - w.switchDbTo(e.DbId) - } - - // send - w.cmdBuffer.Reset() - client.EncodeArgv(e.Argv, w.cmdBuffer) - e.EncodedSize = uint64(w.cmdBuffer.Len()) - for e.EncodedSize+atomic.LoadUint64(&w.UpdateUnansweredBytesCount) > config.Config.Advanced.TargetRedisClientMaxQuerybufLen { - time.Sleep(1 * time.Nanosecond) - } - w.chWaitReply <- e - atomic.AddUint64(&w.UpdateUnansweredBytesCount, e.EncodedSize) - w.client.SendBytes(w.cmdBuffer.Bytes()) -} - -func (w *redisWriter) switchDbTo(newDbId int) { - w.client.Send("select", strconv.Itoa(newDbId)) - w.DbId = newDbId - w.chWaitReply <- &entry.Entry{ - Argv: []string{"select", strconv.Itoa(newDbId)}, - CmdName: "select", - } -} - -func (w *redisWriter) flushInterval() { - for e := range w.chWaitReply { - reply, err := w.client.Receive() - if err == proto.Nil { - log.Warnf("redisWriter receive nil reply. argv=%v", e.Argv) - } else if err != nil { - if err.Error() == "BUSYKEY Target key name already exists." { - if config.Config.Advanced.RDBRestoreCommandBehavior == "skip" { - log.Warnf("redisWriter received BUSYKEY reply. argv=%v", e.Argv) - } else if config.Config.Advanced.RDBRestoreCommandBehavior == "panic" { - log.Panicf("redisWriter received BUSYKEY reply. argv=%v", e.Argv) - } - } else { - log.Panicf("redisWriter received error. error=[%v], argv=%v, slots=%v, reply=[%v]", err, e.Argv, e.Slots, reply) - } - } - if strings.EqualFold(e.CmdName, "select") { // skip select command - continue - } - atomic.AddUint64(&w.UpdateUnansweredBytesCount, ^(e.EncodedSize - 1)) - statistics.UpdateAOFAppliedOffset(uint64(e.Offset)) - statistics.UpdateUnansweredBytesCount(atomic.LoadUint64(&w.UpdateUnansweredBytesCount)) - } - w.chWg.Done() -} - -func (w *redisWriter) Close() { - close(w.chWaitReply) - w.chWg.Wait() -} diff --git a/internal/writer/redis_cluster.go b/internal/writer/redis_cluster_writer.go similarity index 59% rename from internal/writer/redis_cluster.go rename to internal/writer/redis_cluster_writer.go index de77a36a..145d1ad6 100644 --- a/internal/writer/redis_cluster.go +++ b/internal/writer/redis_cluster_writer.go @@ -1,34 +1,47 @@ package writer import ( + "RedisShake/internal/client" + "RedisShake/internal/entry" + "RedisShake/internal/log" "fmt" - "github.com/alibaba/RedisShake/internal/client" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" "strconv" "strings" ) const KeySlots = 16384 +type RedisClusterWriterOptions struct { + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` +} + type RedisClusterWriter struct { addresses []string writers []Writer router [KeySlots]Writer + + stat []interface{} } -func NewRedisClusterWriter(address string, username string, password string, isTls bool) Writer { +func NewRedisClusterWriter(opts *RedisClusterWriterOptions) Writer { rw := new(RedisClusterWriter) - - rw.loadClusterNodes(address, username, password, isTls) - + rw.loadClusterNodes(opts) log.Infof("redisClusterWriter connected to redis cluster successful. addresses=%v", rw.addresses) return rw } -func (r *RedisClusterWriter) loadClusterNodes(address string, username string, password string, isTls bool) { - client_ := client.NewRedisClient(address, username, password, isTls) - reply := client_.DoWithStringReply("cluster", "nodes") +func (r *RedisClusterWriter) Close() { + for _, writer := range r.writers { + writer.Close() + } +} + +func (r *RedisClusterWriter) loadClusterNodes(opts *RedisClusterWriterOptions) { + c := client.NewRedisClient(opts.Address, opts.Username, opts.Password, opts.Tls) + reply := c.DoWithStringReply("cluster", "nodes") reply = strings.TrimSpace(reply) for _, line := range strings.Split(reply, "\n") { line = strings.TrimSpace(line) @@ -54,8 +67,15 @@ func (r *RedisClusterWriter) loadClusterNodes(address string, username string, p } r.addresses = append(r.addresses, address) + // writers - redisWriter := NewRedisWriter(address, username, password, isTls) + opts := &RedisStandaloneWriterOptions{ + Address: address, + Username: opts.Username, + Password: opts.Password, + Tls: opts.Tls, + } + redisWriter := NewRedisStandaloneWriter(opts) r.writers = append(r.writers, redisWriter) // parse slots for i := 8; i < len(words); i++ { @@ -66,16 +86,16 @@ func (r *RedisClusterWriter) loadClusterNodes(address string, username string, p seg := strings.Split(words[i], "-") start, err = strconv.Atoi(seg[0]) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } end, err = strconv.Atoi(seg[1]) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } } else { start, err = strconv.Atoi(words[i]) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } end = start } @@ -114,8 +134,32 @@ func (r *RedisClusterWriter) Write(entry *entry.Entry) { r.router[lastSlot].Write(entry) } -func (r *RedisClusterWriter) Close() { +func (r *RedisClusterWriter) Consistent() bool { for _, writer := range r.writers { - writer.Close() + if !writer.StatusConsistent() { + return false + } + } + return true +} + +func (r *RedisClusterWriter) Status() interface{} { + r.stat = make([]interface{}, 0) + for _, writer := range r.writers { + r.stat = append(r.stat, writer.Status()) + } + return r.stat +} + +func (r *RedisClusterWriter) StatusString() string { + return "[redis_cluster_writer] writing to redis cluster" +} + +func (r *RedisClusterWriter) StatusConsistent() bool { + for _, writer := range r.writers { + if !writer.StatusConsistent() { + return false + } } + return true } diff --git a/internal/writer/redis_standalone_writer.go b/internal/writer/redis_standalone_writer.go new file mode 100644 index 00000000..2c4ebe5b --- /dev/null +++ b/internal/writer/redis_standalone_writer.go @@ -0,0 +1,119 @@ +package writer + +import ( + "RedisShake/internal/client" + "RedisShake/internal/client/proto" + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "fmt" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +type RedisStandaloneWriterOptions struct { + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` +} + +type redisStandaloneWriter struct { + address string + client *client.Redis + DbId int + + chWaitReply chan *entry.Entry + chWg sync.WaitGroup + + stat struct { + Name string `json:"name"` + UnansweredBytes int64 `json:"unanswered_bytes"` + UnansweredEntries int64 `json:"unanswered_entries"` + } +} + +func NewRedisStandaloneWriter(opts *RedisStandaloneWriterOptions) Writer { + rw := new(redisStandaloneWriter) + rw.address = opts.Address + rw.stat.Name = "writer_" + strings.Replace(opts.Address, ":", "_", -1) + rw.client = client.NewRedisClient(opts.Address, opts.Username, opts.Password, opts.Tls) + rw.chWaitReply = make(chan *entry.Entry, config.Opt.Advanced.PipelineCountLimit) + rw.chWg.Add(1) + go rw.processReply() + return rw +} + +func (w *redisStandaloneWriter) Close() { + close(w.chWaitReply) + w.chWg.Wait() +} + +func (w *redisStandaloneWriter) Write(e *entry.Entry) { + // switch db if we need + if w.DbId != e.DbId { + w.switchDbTo(e.DbId) + } + + // send + bytes := e.Serialize() + for e.SerializedSize+atomic.LoadInt64(&w.stat.UnansweredBytes) > config.Opt.Advanced.TargetRedisClientMaxQuerybufLen { + time.Sleep(1 * time.Nanosecond) + } + log.Debugf("[%s] send cmd. cmd=[%s]", w.stat.Name, e.String()) + w.chWaitReply <- e + atomic.AddInt64(&w.stat.UnansweredBytes, e.SerializedSize) + atomic.AddInt64(&w.stat.UnansweredEntries, 1) + w.client.SendBytes(bytes) +} + +func (w *redisStandaloneWriter) switchDbTo(newDbId int) { + log.Debugf("[%s] switch db to [%d]", w.stat.Name, newDbId) + w.client.Send("select", strconv.Itoa(newDbId)) + w.DbId = newDbId + w.chWaitReply <- &entry.Entry{ + Argv: []string{"select", strconv.Itoa(newDbId)}, + CmdName: "select", + } +} + +func (w *redisStandaloneWriter) processReply() { + for e := range w.chWaitReply { + reply, err := w.client.Receive() + log.Debugf("[%s] receive reply. reply=[%v], cmd=[%s]", w.stat.Name, reply, e.String()) + if err == proto.Nil { + log.Warnf("[%s] receive nil reply. cmd=[%s]", w.stat.Name, e.String()) + } else if err != nil { + if err.Error() == "BUSYKEY Target key name already exists." { + if config.Opt.Advanced.RDBRestoreCommandBehavior == "skip" { + log.Debugf("[%s] redisStandaloneWriter received BUSYKEY reply. cmd=[%s]", w.stat.Name, e.String()) + } else if config.Opt.Advanced.RDBRestoreCommandBehavior == "panic" { + log.Panicf("[%s] redisStandaloneWriter received BUSYKEY reply. cmd=[%s]", w.stat.Name, e.String()) + } + } else { + log.Panicf("[%s] receive reply failed. cmd=[%s], error=[%v]", w.stat.Name, e.String(), err) + } + } + if strings.EqualFold(e.CmdName, "select") { // skip select command + continue + } + atomic.AddInt64(&w.stat.UnansweredBytes, -e.SerializedSize) + atomic.AddInt64(&w.stat.UnansweredEntries, -1) + } + w.chWg.Done() +} + +func (w *redisStandaloneWriter) Status() interface{} { + return w.stat +} + +func (w *redisStandaloneWriter) StatusString() string { + return fmt.Sprintf("[%s]: unanswered_entries=%d", w.stat.Name, atomic.LoadInt64(&w.stat.UnansweredEntries)) +} + +func (w *redisStandaloneWriter) StatusConsistent() bool { + return atomic.LoadInt64(&w.stat.UnansweredBytes) == 0 && atomic.LoadInt64(&w.stat.UnansweredEntries) == 0 +} diff --git a/restore.toml b/restore.toml deleted file mode 100644 index 6ed1d1ae..00000000 --- a/restore.toml +++ /dev/null @@ -1,54 +0,0 @@ -type = "restore" - -[source] -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -# Path to the dump.rdb file. Absolute path or relative path. Note -# that relative paths are relative to the dir directory. -rdb_file_path = "dump.rdb" - -[target] -type = "standalone" # standalone or cluster -# When the target is a cluster, write the address of one of the nodes. -# redis-shake will obtain other nodes through the `cluster nodes` command. -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -address = "127.0.0.1:6379" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[advanced] -dir = "data" - -# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores -ncpu = 3 - -# pprof port, 0 means disable -pprof_port = 0 - -# metric port, 0 means disable -metrics_port = 0 - -# log -log_file = "redis-shake.log" -log_level = "info" # debug, info or warn -log_interval = 5 # in seconds - -# redis-shake gets key and value from rdb file, and uses RESTORE command to -# create the key in target redis. Redis RESTORE will return a "Target key name -# is busy" error when key already exists. You can use this configuration item -# to change the default behavior of restore: -# panic: redis-shake will stop when meet "Target key name is busy" error. -# rewrite: redis-shake will replace the key with new value. -# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. -rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip - -# pipeline -pipeline_count_limit = 1024 - -# Client query buffers accumulate new commands. They are limited to a fixed -# amount by default. This amount is normally 1gb. -target_redis_client_max_querybuf_len = 1024_000_000 - -# In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited to 512 mb. -target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file diff --git a/scan.toml b/scan.toml deleted file mode 100644 index 25f61add..00000000 --- a/scan.toml +++ /dev/null @@ -1,55 +0,0 @@ -type = "scan" - -[source] -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -address = "127.0.0.1:6379" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[target] -type = "standalone" # "standalone" or "cluster" -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -# When the target is a cluster, write the address of one of the nodes. -# redis-shake will obtain other nodes through the `cluster nodes` command. -address = "127.0.0.1:6380" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[advanced] -dir = "data" - -# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores -ncpu = 0 - -# pprof port, 0 means disable -pprof_port = 0 - -# metric port, 0 means disable -metrics_port = 0 - -# log -log_file = "redis-shake.log" -log_level = "info" # debug, info or warn -log_interval = 5 # in seconds - -# redis-shake gets key and value from rdb file, and uses RESTORE command to -# create the key in target redis. Redis RESTORE will return a "Target key name -# is busy" error when key already exists. You can use this configuration item -# to change the default behavior of restore: -# panic: redis-shake will stop when meet "Target key name is busy" error. -# rewrite: redis-shake will replace the key with new value. -# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. -rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip - -# pipeline -pipeline_count_limit = 1024 - -# Client query buffers accumulate new commands. They are limited to a fixed -# amount by default. This amount is normally 1gb. -target_redis_client_max_querybuf_len = 1024_000_000 - -# In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited to 512 mb. -target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file diff --git a/scripts/cluster_helper/cluster_helper.py b/scripts/cluster_helper/cluster_helper.py deleted file mode 100644 index 8a9a68ec..00000000 --- a/scripts/cluster_helper/cluster_helper.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 -import datetime -import os -import shutil -import signal -import sys -import time -from pathlib import Path - -import redis -import requests -import toml - -from launcher import Launcher - -USAGE = """ -cluster_helper is a helper script to start many redis-shake for syncing from cluster. - -Usage: - $ python3 cluster_helper.py ./bin/redis-shake sync.toml - or - $ python3 cluster_helper.py ./bin/redis-shake sync.toml ./bin/filters/key_prefix.lua -""" - -REDIS_SHAKE_PATH = "" -LUA_FILTER_PATH = "" -SLEEP_SECONDS = 5 -stopped = False -toml_template = {} - - -class Shake: - def __init__(self): - self.metrics_port = 0 - self.launcher = None - - -nodes = {} - - -def parse_args(): - if len(sys.argv) != 3 and len(sys.argv) != 4: - print(USAGE) - exit(1) - global REDIS_SHAKE_PATH, LUA_FILTER_PATH, toml_template - - # 1. check redis-shake path - REDIS_SHAKE_PATH = sys.argv[1] - if not Path(REDIS_SHAKE_PATH).is_file(): - print(f"redis-shake path [{REDIS_SHAKE_PATH}] is not a file") - print(USAGE) - exit(1) - print(f"redis-shake path: {REDIS_SHAKE_PATH}") - REDIS_SHAKE_PATH = os.path.abspath(REDIS_SHAKE_PATH) - print(f"redis-shake abs path: {REDIS_SHAKE_PATH}") - - # 2. check and load toml file - toml_template = toml.load(sys.argv[2]) - print(toml_template) - if "username" not in toml_template["source"]: - toml_template["source"]["username"] = "" - if "password" not in toml_template["source"]: - toml_template["source"]["password"] = "" - if "tls" not in toml_template["source"]: - toml_template["source"]["tls"] = False - if "advanced" not in toml_template: - toml_template["advanced"] = {} - - # 3. check filter - if len(sys.argv) == 4: - LUA_FILTER_PATH = sys.argv[3] - if not Path(LUA_FILTER_PATH).is_file(): - print(f"filter path [{LUA_FILTER_PATH}] is not a file") - print(USAGE) - exit(1) - print(f"filter path: {LUA_FILTER_PATH}") - LUA_FILTER_PATH = os.path.abspath(LUA_FILTER_PATH) - print(f"filter abs path: {LUA_FILTER_PATH}") - - -def stop(): - for shake in nodes.values(): - shake.launcher.stop() - exit(0) - - -def loop(): - while True: - if stopped: - stop() - print( - f"================ {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ================" - ) - - metrics = [] - for address, shake in nodes.items(): - try: - ret = requests.get(f"http://localhost:{shake.metrics_port}").json() - metrics.append(ret) - except requests.exceptions.RequestException as e: - print(f"get metrics from [{address}] failed: {e}") - - for metric in sorted(metrics, key=lambda x: x["address"]): - print(f"{metric['address']} {metric['msg']} ") - - if len(metrics) == 0: - print("no redis-shake is running") - break - - time.sleep(SLEEP_SECONDS) - - -def main(): - parse_args() - - # parse args - address = toml_template["source"]["address"] - host, port = address.split(":") - username = toml_template["source"]["username"] - password = toml_template["source"]["password"] - tls = toml_template["source"]["tls"] - print( - f"host: {host}, port: {port}, username: {username}, password: {password}, tls: {tls}" - ) - cluster = redis.RedisCluster( - host=host, port=port, username=username, password=password, ssl=tls - ) - print("cluster nodes:", cluster.cluster_nodes()) - - # parse cluster nodes - for address, node in cluster.cluster_nodes().items(): - if "master" in node["flags"]: - nodes[address] = Shake() - print(f"addresses:") - for k in nodes.keys(): - print(k) - - # create workdir and start redis-shake - if os.path.exists("data"): - shutil.rmtree("data") - os.mkdir("data") - os.chdir("data") - start_port = ( - 11007 - if toml_template.get("advanced").get("metrics_port", 0) == 0 - else toml_template["advanced"]["metrics_port"] - ) - for address in nodes.keys(): - workdir = address.replace(".", "_").replace(":", "_") - - os.mkdir(workdir) - tmp_toml = toml_template - tmp_toml["source"]["address"] = address - start_port += 1 - tmp_toml["advanced"]["metrics_port"] = start_port - - with open(f"{workdir}/sync.toml", "w") as f: - toml.dump(tmp_toml, f) - - # start redis-shake - args = [REDIS_SHAKE_PATH, f"sync.toml"] - if LUA_FILTER_PATH != "": - args.append(LUA_FILTER_PATH) - launcher = Launcher(args=args, work_dir=workdir) - nodes[address].launcher = launcher - nodes[address].metrics_port = start_port - - signal.signal(signal.SIGINT, signal_handler) - print("start syncing...") - print("sleep 3 seconds to wait redis-shake start") - time.sleep(3) - loop() - for node in nodes.values(): - node.launcher.stop() - - -def signal_handler(sig, frame): - global stopped - print("\nYou pressed Ctrl+C!") - stopped = True - - -if __name__ == "__main__": - main() diff --git a/scripts/cluster_helper/launcher.py b/scripts/cluster_helper/launcher.py deleted file mode 100644 index f1da8944..00000000 --- a/scripts/cluster_helper/launcher.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import signal -import subprocess -from pathlib import Path - - -class Launcher: - def __init__(self, args, work_dir): - self.started = True - self.args = args - self.work_dir = work_dir - if not os.path.exists(work_dir): - Path(self.work_dir).mkdir(parents=True, exist_ok=True) - self.stdout_file = open(work_dir + "/stdout", 'a') - self.stderr_file = open(work_dir + "/stderr", 'a') - self.process = subprocess.Popen(self.args, stdout=self.stdout_file, - stderr=self.stderr_file, cwd=self.work_dir, - encoding="utf-8") - - def __del__(self): - assert not self.started, "Every Launcher should be closed manually! work_dir:" + self.work_dir - - def get_pid(self): - return self.process.pid - - def stop(self): - if self.started: - self.started = False - print(f"Waiting for process {self.process.pid} to exit...") - self.stdout_file.close() - self.stderr_file.close() - self.process.send_signal(signal.SIGINT) - self.process.wait() - print(f"process {self.process.pid} exited.") diff --git a/scripts/cluster_helper/requirements.txt b/scripts/cluster_helper/requirements.txt deleted file mode 100644 index 5cd405fc..00000000 --- a/scripts/cluster_helper/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -redis==4.3.4 -requests==2.27.1 -toml==0.10.2 diff --git a/test.sh b/test.sh index c41811b6..3edc5908 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,9 @@ #!/bin/bash set -e +# unit test go test ./... -v -cd test -python main.py \ No newline at end of file +# black box test +cd tests/ +pybbt cases \ No newline at end of file diff --git a/test/assets/empty.toml b/test/assets/empty.toml deleted file mode 100644 index 6ac5ea65..00000000 --- a/test/assets/empty.toml +++ /dev/null @@ -1,54 +0,0 @@ -type = "sync" - -[source] -address = "127.0.0.1:6379" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false -elasticache_psync = "" # using when source is ElastiCache. ref: https://github.com/alibaba/RedisShake/issues/373 - -[target] -type = "cluster" # standalone or cluster -# When the target is a cluster, write the address of one of the nodes. -# redis-shake will obtain other nodes through the `cluster nodes` command. -address = "127.0.0.1:30001" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[advanced] -dir = "data" - -# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores -ncpu = 3 - -# pprof port, 0 means disable -pprof_port = 0 - -# metric port, 0 means disable -metrics_port = 0 - -# log -log_file = "redis-shake.log" -log_level = "info" # debug, info or warn -log_interval = 5 # in seconds - -# redis-shake gets key and value from rdb file, and uses RESTORE command to -# create the key in target redis. Redis RESTORE will return a "Target key name -# is busy" error when key already exists. You can use this configuration item -# to change the default behavior of restore: -# panic: redis-shake will stop when meet "Target key name is busy" error. -# rewrite: redis-shake will replace the key with new value. -# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. -rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip - -# pipeline -pipeline_count_limit = 1024 - -# Client query buffers accumulate new commands. They are limited to a fixed -# amount by default. This amount is normally 1gb. -target_redis_client_max_querybuf_len = 1024_000_000 - -# In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited to 512 mb. -target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file diff --git a/test/cases/auth.py b/test/cases/auth.py deleted file mode 100644 index 1194b7f4..00000000 --- a/test/cases/auth.py +++ /dev/null @@ -1,27 +0,0 @@ -import time - -from utils import * - - -def main(): - r0 = Redis() - r0.client.config_set("requirepass", "password") - r0.client.execute_command("auth", "password") # for Redis 4.0 - r1 = Redis() - r1.client.config_set("requirepass", "password") - r1.client.execute_command("auth", "password") # for Redis 4.0 - - t = get_empty_config() - t["source"]["address"] = r0.get_address() - t["source"]["password"] = "password" - t["target"]["type"] = "standalone" - t["target"]["address"] = r1.get_address() - t["target"]["password"] = "password" - - rs = RedisShake() - rs.run(t) - - # wait sync need use http interface - r0.client.set("finished", "1") - time.sleep(2) - assert r1.client.get("finished") == b"1" diff --git a/test/cases/auth_acl.py b/test/cases/auth_acl.py deleted file mode 100644 index 66438c75..00000000 --- a/test/cases/auth_acl.py +++ /dev/null @@ -1,35 +0,0 @@ -import time - -from utils import * - - -def main(): - r0 = Redis() - try: - r0.client.acl_list() - except Exception: - return - r0.client.execute_command("acl", "setuser", "user0", ">password0", "~*", "+@all") - r0.client.execute_command("acl", "setuser", "user0", "on") - r0.client.execute_command("auth", "user0", "password0") # for Redis 4.0 - r1 = Redis() - r1.client.execute_command("acl", "setuser", "user1", ">password1", "~*", "+@all") - r1.client.execute_command("acl", "setuser", "user1", "on") - r1.client.execute_command("auth", "user1", "password1") # for Redis 4.0 - - t = get_empty_config() - t["source"]["address"] = r0.get_address() - t["source"]["username"] = "user0" - t["source"]["password"] = "password0" - t["target"]["type"] = "standalone" - t["target"]["address"] = r1.get_address() - t["target"]["username"] = "user1" - t["target"]["password"] = "password1" - - rs = RedisShake() - rs.run(t) - - # wait sync need use http interface - r0.client.set("finished", "1") - time.sleep(2) - assert r1.client.get("finished") == b"1" diff --git a/test/cases/cluster/__init__.py b/test/cases/cluster/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/cases/cluster/sync.py b/test/cases/cluster/sync.py deleted file mode 100644 index 4bb57e5a..00000000 --- a/test/cases/cluster/sync.py +++ /dev/null @@ -1,5 +0,0 @@ -from utils import * - - -def main(): - pass diff --git a/test/cases/example.py b/test/cases/example.py deleted file mode 100644 index 8cf3afaa..00000000 --- a/test/cases/example.py +++ /dev/null @@ -1,9 +0,0 @@ -from utils import * - - -def main(): - assert True - - -if __name__ == '__main__': - pass diff --git a/test/cases/issues/__init__.py b/test/cases/issues/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/cases/types/__init__.py b/test/cases/types/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/cases/types/type_hash.py b/test/cases/types/type_hash.py deleted file mode 100644 index a60f604a..00000000 --- a/test/cases/types/type_hash.py +++ /dev/null @@ -1,25 +0,0 @@ -import redis - -prefix = "hash" - - -def add_rdb_data(c: redis.Redis): - c.hset(f"{prefix}_rdb_k", "key0", "value0") - for i in range(10000): - c.hset(f"{prefix}_rdb_k_large", f"key{i}", f"value{i}") - - -def add_aof_data(c: redis.Redis): - c.hset(f"{prefix}_aof_k", "key0", "value0") - for i in range(10000): - c.hset(f"{prefix}_aof_k_large", f"key{i}", f"value{i}") - - -def check_data(c: redis.Redis): - assert c.hget(f"{prefix}_rdb_k", "key0") == b"value0" - assert c.hmget(f"{prefix}_rdb_k_large", *[f"key{i}" for i in range(10000)]) == [f"value{i}".encode() for i in - range(10000)] - - assert c.hget(f"{prefix}_aof_k", "key0") == b"value0" - assert c.hmget(f"{prefix}_aof_k_large", *[f"key{i}" for i in range(10000)]) == [f"value{i}".encode() for i in - range(10000)] diff --git a/test/cases/types/type_list.py b/test/cases/types/type_list.py deleted file mode 100644 index f823a958..00000000 --- a/test/cases/types/type_list.py +++ /dev/null @@ -1,23 +0,0 @@ -import redis - -prefix = "list" - -elements = [f"element_{i}" for i in range(10000)] - - -def add_rdb_data(c: redis.Redis): - c.rpush(f"{prefix}_rdb_k", 0, 1, 2, 3, 4, 5, 6, 7) - c.rpush(f"{prefix}_rdb_k0", *elements) - - -def add_aof_data(c: redis.Redis): - c.rpush(f"{prefix}_aof_k", 0, 1, 2, 3, 4, 5, 6, 7) - c.rpush(f"{prefix}_aof_k0", *elements) - - -def check_data(c: redis.Redis): - assert c.lrange(f"{prefix}_rdb_k", 0, -1) == [b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7"] - assert c.lrange(f"{prefix}_rdb_k0", 0, -1) == [f"element_{i}".encode() for i in range(10000)] - - assert c.lrange(f"{prefix}_aof_k", 0, -1) == [b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7"] - assert c.lrange(f"{prefix}_aof_k0", 0, -1) == [f"element_{i}".encode() for i in range(10000)] diff --git a/test/cases/types/type_set.py b/test/cases/types/type_set.py deleted file mode 100644 index 288d95ab..00000000 --- a/test/cases/types/type_set.py +++ /dev/null @@ -1,23 +0,0 @@ -import redis - -prefix = "set" - - -def add_rdb_data(c: redis.Redis): - c.sadd(f"{prefix}_rdb_k", 0, 1, 2, 3, 4, 5, 6, 7) - elements = [f"element_{i}" for i in range(10000)] - c.sadd(f"{prefix}_rdb_k0", *elements) - - -def add_aof_data(c: redis.Redis): - c.sadd(f"{prefix}_aof_k", 0, 1, 2, 3, 4, 5, 6, 7) - elements = [f"element_{i}" for i in range(10000)] - c.sadd(f"{prefix}_aof_k0", *elements) - - -def check_data(c: redis.Redis): - assert c.smembers(f"{prefix}_rdb_k") == {b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7"} - assert c.smembers(f"{prefix}_rdb_k0") == {f"element_{i}".encode() for i in range(10000)} - - assert c.smembers(f"{prefix}_aof_k") == {b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7"} - assert c.smembers(f"{prefix}_aof_k0") == {f"element_{i}".encode() for i in range(10000)} diff --git a/test/cases/types/type_stream.py b/test/cases/types/type_stream.py deleted file mode 100644 index d1f207e5..00000000 --- a/test/cases/types/type_stream.py +++ /dev/null @@ -1,34 +0,0 @@ -import redis - -prefix = "stream" - -fields = {f"field_{i}": f"value_{i}" for i in range(64)} -STREAM_LENGTH = 128 - - -def add_rdb_data(c: redis.Redis): - c.xadd(f"{prefix}_rdb_k", {"key0": "value0"}, "*") - for i in range(STREAM_LENGTH): - c.xadd(f"{prefix}_rdb_k_large", fields=fields, id="*") - - -def add_aof_data(c: redis.Redis): - c.xadd(f"{prefix}_aof_k", {"key0": "value0"}, "*") - for i in range(STREAM_LENGTH): - c.xadd(f"{prefix}_aof_k_large", fields=fields, id="*") - - -def check_data(c: redis.Redis): - ret = c.xread(streams={f"{prefix}_rdb_k": "0-0"}, count=1)[0][1] - assert ret[0][1] == {b"key0": b"value0"} - - ret = c.xread(streams={f"{prefix}_rdb_k_large": "0-0"}, count=STREAM_LENGTH)[0][1] - for i in range(STREAM_LENGTH): - assert ret[i][1] == {k.encode(): v.encode() for k, v in fields.items()} - - ret = c.xread(streams={f"{prefix}_aof_k": "0-0"}, count=1)[0][1] - assert ret[0][1] == {b"key0": b"value0"} - - ret = c.xread(streams={f"{prefix}_aof_k_large": "0-0"}, count=STREAM_LENGTH)[0][1] - for i in range(STREAM_LENGTH): - assert ret[i][1] == {k.encode(): v.encode() for k, v in fields.items()} diff --git a/test/cases/types/type_string.py b/test/cases/types/type_string.py deleted file mode 100644 index c3c00b15..00000000 --- a/test/cases/types/type_string.py +++ /dev/null @@ -1,29 +0,0 @@ -import redis - -prefix = "string" - - -def add_rdb_data(c: redis.Redis): - c.set(f"{prefix}_rdb_k", "v") - c.set(f"{prefix}_rdb_int", 0) - c.set(f"{prefix}_rdb_int0", -1) - c.set(f"{prefix}_rdb_int1", 123456789) - - -def add_aof_data(c: redis.Redis): - c.set(f"{prefix}_aof_k", "v") - c.set(f"{prefix}_aof_int", 0) - c.set(f"{prefix}_aof_int0", -1) - c.set(f"{prefix}_aof_int1", 123456789) - - -def check_data(c: redis.Redis): - assert c.get(f"{prefix}_rdb_k") == b"v" - assert c.get(f"{prefix}_rdb_int") == b'0' - assert c.get(f"{prefix}_rdb_int0") == b'-1' - assert c.get(f"{prefix}_rdb_int1") == b'123456789' - - assert c.get(f"{prefix}_aof_k") == b"v" - assert c.get(f"{prefix}_aof_int") == b'0' - assert c.get(f"{prefix}_aof_int0") == b'-1' - assert c.get(f"{prefix}_aof_int1") == b'123456789' diff --git a/test/cases/types/type_zset.py b/test/cases/types/type_zset.py deleted file mode 100644 index 6fba0410..00000000 --- a/test/cases/types/type_zset.py +++ /dev/null @@ -1,27 +0,0 @@ -import redis - -prefix = "zset" -float_maps = {"a": 1.1111, "b": 2.2222, "c": 3.3333} -maps = {str(item): item for item in range(10000)} - - -def add_rdb_data(c: redis.Redis): - c.zadd(f"{prefix}_rdb_k_float", float_maps) - c.zadd(f"{prefix}_rdb_k", maps) - - -def add_aof_data(c: redis.Redis): - c.zadd(f"{prefix}_aof_k_float", float_maps) - c.zadd(f"{prefix}_aof_k", maps) - - -def check_data(c: redis.Redis): - for k, v in c.zrange(f"{prefix}_rdb_k_float", 0, -1, withscores=True): - assert float_maps[k.decode()] == v - for k, v in c.zrange(f"{prefix}_rdb_k", 0, -1, withscores=True): - assert maps[k.decode()] == v - - for k, v in c.zrange(f"{prefix}_aof_k_float", 0, -1, withscores=True): - assert float_maps[k.decode()] == v - for k, v in c.zrange(f"{prefix}_aof_k", 0, -1, withscores=True): - assert maps[k.decode()] == v diff --git a/test/cases/types/types.py b/test/cases/types/types.py deleted file mode 100644 index 743ce3c5..00000000 --- a/test/cases/types/types.py +++ /dev/null @@ -1,62 +0,0 @@ -import time - -import jury -from utils import * - -from . import type_string, type_list, type_set, type_hash, type_zset, type_stream - - -def main(): - rs = RedisShake() - r0 = Redis() - r1 = Redis() - t = get_empty_config() - t["source"]["address"] = r0.get_address() - t["target"]["type"] = "standalone" - t["target"]["address"] = r1.get_address() - - timer = jury.Timer() - type_string.add_rdb_data(r0.client) - type_list.add_rdb_data(r0.client) - type_set.add_rdb_data(r0.client) - type_hash.add_rdb_data(r0.client) - type_zset.add_rdb_data(r0.client) - type_stream.add_rdb_data(r0.client) - jury.log(f"add_rdb_data: {timer.elapsed_time()}s") - - # run redis-shake - rs.run(t) - time.sleep(1) - - timer = jury.Timer() - type_string.add_aof_data(r0.client) - type_list.add_aof_data(r0.client) - type_set.add_aof_data(r0.client) - type_hash.add_aof_data(r0.client) - type_zset.add_aof_data(r0.client) - type_stream.add_aof_data(r0.client) - jury.log(f"add_aof_data: {timer.elapsed_time()}s") - - # wait sync need use http interface - timer = jury.Timer() - r0.client.set("finished", "1") - cnt = 0 - while r1.client.get("finished") != b"1": - time.sleep(0.5) - cnt += 1 - if cnt > 20: - raise Exception("sync timeout") - jury.log(f"sync time: {timer.elapsed_time()}s") - - timer = jury.Timer() - type_string.check_data(r1.client) - type_list.check_data(r1.client) - type_set.check_data(r1.client) - type_hash.check_data(r1.client) - type_zset.check_data(r1.client) - type_stream.check_data(r1.client) - jury.log(f"check_data: {timer.elapsed_time()}s") - - -if __name__ == '__main__': - main() diff --git a/test/cases/types/types_rewrite.py b/test/cases/types/types_rewrite.py deleted file mode 100644 index 8ffc8ec4..00000000 --- a/test/cases/types/types_rewrite.py +++ /dev/null @@ -1,63 +0,0 @@ -import time - -import jury -from utils import * - -from . import type_string, type_list, type_set, type_hash, type_zset, type_stream - - -def main(): - rs = RedisShake() - r0 = Redis() - r1 = Redis() - t = get_empty_config() - t["advanced"]["target_redis_proto_max_bulk_len"] = 0 - t["source"]["address"] = r0.get_address() - t["target"]["type"] = "standalone" - t["target"]["address"] = r1.get_address() - - timer = jury.Timer() - type_string.add_rdb_data(r0.client) - type_list.add_rdb_data(r0.client) - type_set.add_rdb_data(r0.client) - type_hash.add_rdb_data(r0.client) - type_zset.add_rdb_data(r0.client) - type_stream.add_rdb_data(r0.client) - jury.log(f"add_rdb_data: {timer.elapsed_time()}s") - - # run redis-shake - rs.run(t) - time.sleep(1) - - timer = jury.Timer() - type_string.add_aof_data(r0.client) - type_list.add_aof_data(r0.client) - type_set.add_aof_data(r0.client) - type_hash.add_aof_data(r0.client) - type_zset.add_aof_data(r0.client) - type_stream.add_aof_data(r0.client) - jury.log(f"add_aof_data: {timer.elapsed_time()}s") - - # wait sync need use http interface - timer = jury.Timer() - r0.client.set("finished", "1") - cnt = 0 - while r1.client.get("finished") != b"1": - time.sleep(0.5) - cnt += 1 - if cnt > 20: - raise Exception("sync timeout") - jury.log(f"sync time: {timer.elapsed_time()}s") - - timer = jury.Timer() - type_string.check_data(r1.client) - type_list.check_data(r1.client) - type_set.check_data(r1.client) - type_hash.check_data(r1.client) - type_zset.check_data(r1.client) - type_stream.check_data(r1.client) - jury.log(f"check_data: {timer.elapsed_time()}s") - - -if __name__ == '__main__': - main() diff --git a/test/main.py b/test/main.py deleted file mode 100644 index 95da2c8a..00000000 --- a/test/main.py +++ /dev/null @@ -1,19 +0,0 @@ -import jury - -cases = [ - "cases/example", - "cases/types/types", - "cases/types/types_rewrite", - "cases/cluster/sync", - "cases/auth", - "cases/auth_acl", -] - - -def main(): - j = jury.Jury(cases) - j.run() - - -if __name__ == '__main__': - main() diff --git a/test/requirements.txt b/test/requirements.txt deleted file mode 100644 index c2b8943a..00000000 --- a/test/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -jury-test==0.0.3 -redis==4.3.3 -toml==0.10.2 diff --git a/test/utils/__init__.py b/test/utils/__init__.py deleted file mode 100644 index 4eba8f97..00000000 --- a/test/utils/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .redis_ import Redis -from .cluster_ import Cluster -from .redis_shake import get_empty_config, RedisShake diff --git a/test/utils/cluster_.py b/test/utils/cluster_.py deleted file mode 100644 index 5b8a956e..00000000 --- a/test/utils/cluster_.py +++ /dev/null @@ -1,18 +0,0 @@ -import os - -from .redis_ import Redis -import jury - - -class Cluster: - def __init__(self, num=3): - self.nodes = [] - self.num = num - for i in range(num): - self.nodes.append(Redis(args=["--cluster-enabled", "yes"])) - host_port_list = [f"{node.host}:{node.port}" for node in self.nodes] - jury.log_yellow(f"Redis cluster created, {self.num} nodes. {host_port_list}") - os.system("redis-cli --cluster-yes --cluster create " + " ".join(host_port_list)) - - def push_table(self): - pass diff --git a/test/utils/constant.py b/test/utils/constant.py deleted file mode 100644 index 9598674b..00000000 --- a/test/utils/constant.py +++ /dev/null @@ -1,6 +0,0 @@ -from pathlib import Path - -BASE_PATH = f"{Path(__file__).parent.parent.parent.absolute()}" # project path - -PATH_REDIS_SHAKE = f"{BASE_PATH}/bin/redis-shake" -PATH_EMPTY_CONFIG_FILE = f"{BASE_PATH}/test/assets/empty.toml" diff --git a/test/utils/redis_.py b/test/utils/redis_.py deleted file mode 100644 index 2ac5ed92..00000000 --- a/test/utils/redis_.py +++ /dev/null @@ -1,42 +0,0 @@ -import time - -import jury -import redis - - -class Redis: - def __init__(self, args=None): - self.args = args - if args is None: - self.args = [] - - self.redis = None - self.host = "127.0.0.1" - self.port = jury.get_free_port() - self.dir = f"{jury.get_case_dir()}/redis_{self.port}" - self.server = jury.Launcher(args=["redis-server", "--port", str(self.port)] + self.args, work_dir=self.dir) - self.__wait_start() - jury.log_yellow(f"Redis started(pid={self.server.get_pid()}). redis-cli -p {self.port}") - - def get_address(self): - return f"127.0.0.1:{self.port}" - - def __wait_start(self, timeout=5): - timer = jury.Timer() - while True: - try: - self.__create_client() - self.client.ping() - break - except redis.exceptions.ConnectionError: - time.sleep(0.01) - except redis.exceptions.ResponseError as e: - if str(e) == "LOADING Redis is loading the dataset in memory": - time.sleep(0.01) - else: - raise e - if timer.elapsed_time() > timeout: - raise Exception(f"tair start timeout, {self.dir}") - - def __create_client(self): - self.client = redis.Redis(port=self.port, single_connection_client=True) diff --git a/test/utils/redis_shake.py b/test/utils/redis_shake.py deleted file mode 100644 index 5d7a581c..00000000 --- a/test/utils/redis_shake.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -from pathlib import Path - -import jury -import toml - -from .constant import PATH_REDIS_SHAKE, PATH_EMPTY_CONFIG_FILE - - -def get_empty_config(): - with open(PATH_EMPTY_CONFIG_FILE, "r") as f: - return toml.load(f) - - -class RedisShake: - def __init__(self): - self.server = None - self.redis = None - self.dir = f"{jury.get_case_dir()}/redis_shake" - if not os.path.exists(self.dir): - Path(self.dir).mkdir(parents=True, exist_ok=True) - - def run(self, toml_config): - with open(f"{self.dir}/redis-shake.toml", "w") as f: - toml.dump(toml_config, f) - self.server = jury.Launcher(args=[PATH_REDIS_SHAKE, "redis-shake.toml"], work_dir=self.dir) diff --git a/test/.gitignore b/tests/.gitignore similarity index 100% rename from test/.gitignore rename to tests/.gitignore diff --git a/tests/cases/auth_acl.py b/tests/cases/auth_acl.py new file mode 100644 index 00000000..61e26ccf --- /dev/null +++ b/tests/cases/auth_acl.py @@ -0,0 +1,50 @@ +import pybbt as p + +import helpers as h + + +@p.subcase() +def acl(): + src = h.Redis() + dst = h.Redis() + + src.client.execute_command("acl", "setuser", "user0", ">password0", "~*", "+@all") + src.client.execute_command("acl", "setuser", "user0", "on") + src.client.execute_command("auth", "user0", "password0") # for Redis 4.0 + + dst.client.execute_command("acl", "setuser", "user1", ">password1", "~*", "+@all") + dst.client.execute_command("acl", "setuser", "user1", "on") + dst.client.execute_command("auth", "user1", "password1") # for Redis 4.0 + + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=True) + + opts = h.ShakeOpts.create_sync_opts(src, dst) + opts["SyncStandaloneReader"]["username"] = "user0" + opts["SyncStandaloneReader"]["password"] = "password0" + opts["SyncStandaloneWriter"]["username"] = "user1" + opts["SyncStandaloneWriter"]["password"] = "password1" + p.log(f"opts: {opts}") + shake = h.Shake(opts) + print(shake.dir) + + # wait sync done + p.ASSERT_TRUE_TIMEOUT(lambda: shake.is_consistent()) + p.log(shake.get_status()) + + # check data + inserter.check_data(src, cross_slots_cmd=True) + inserter.check_data(dst, cross_slots_cmd=True) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + + +@p.case(tags=["acl"]) +def main(): + if h.REDIS_SERVER_VERSION < 6.0: + return + + acl() + + +if __name__ == '__main__': + main() diff --git a/tests/cases/rdb.py b/tests/cases/rdb.py new file mode 100644 index 00000000..e020b939 --- /dev/null +++ b/tests/cases/rdb.py @@ -0,0 +1,45 @@ +import pybbt as p + +import helpers as h + + +def test(src, dst): + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_TRUE(src.do("save")) + + opts = h.ShakeOpts.create_rdb_opts(f"{src.dir}/dump.rdb", dst) + p.log(f"opts: {opts}") + h.Shake.run_once(opts) + + # check data + inserter.check_data(src, cross_slots_cmd=cross_slots_cmd) + inserter.check_data(dst, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + + +@p.subcase() +def rdb_to_standalone(): + src = h.Redis() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def rdb_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Redis() + dst = h.Cluster() + test(src, dst) + + +@p.case(tags=["sync"]) +def main(): + rdb_to_standalone() + rdb_to_cluster() + + +if __name__ == '__main__': + main() diff --git a/tests/cases/scan.py b/tests/cases/scan.py new file mode 100644 index 00000000..1023c703 --- /dev/null +++ b/tests/cases/scan.py @@ -0,0 +1,68 @@ +import pybbt as p + +import helpers as h + + +def test(src, dst): + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_TRUE(src.do("save")) + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) # add data again + + opts = h.ShakeOpts.create_scan_opts(src, dst) + p.log(f"opts: {opts}") + + # run shake + h.Shake.run_once(opts) + + # check data + inserter.check_data(src, cross_slots_cmd=cross_slots_cmd) + inserter.check_data(dst, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + + +@p.subcase() +def standalone_to_standalone(): + src = h.Redis() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def standalone_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Redis() + dst = h.Cluster() + test(src, dst) + + +@p.subcase() +def cluster_to_standalone(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Cluster() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def cluster_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Cluster() + dst = h.Cluster() + test(src, dst) + + +@p.case(tags=["scan"]) +def main(): + standalone_to_standalone() + standalone_to_cluster() + cluster_to_standalone() + cluster_to_cluster() + + +if __name__ == '__main__': + main() diff --git a/tests/cases/sync.py b/tests/cases/sync.py new file mode 100644 index 00000000..dfd4f41f --- /dev/null +++ b/tests/cases/sync.py @@ -0,0 +1,76 @@ +import pybbt as p + +import helpers as h + + +def test(src, dst): + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + + p.ASSERT_TRUE(src.do("save")) + + opts = h.ShakeOpts.create_sync_opts(src, dst) + p.log(f"opts: {opts}") + shake = h.Shake(opts) + + # wait sync done + p.ASSERT_TRUE_TIMEOUT(lambda: shake.is_consistent(), timeout=10) + + # add data again + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + + # wait sync done + p.ASSERT_TRUE_TIMEOUT(lambda: shake.is_consistent()) + p.log(shake.get_status()) + + # check data + inserter.check_data(src, cross_slots_cmd=cross_slots_cmd) + inserter.check_data(dst, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + + +@p.subcase() +def standalone_to_standalone(): + src = h.Redis() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def standalone_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Redis() + dst = h.Cluster() + test(src, dst) + + +@p.subcase() +def cluster_to_standalone(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Cluster() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def cluster_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Cluster() + dst = h.Cluster() + test(src, dst) + + +@p.case(tags=["sync"]) +def main(): + standalone_to_standalone() + standalone_to_cluster() + cluster_to_standalone() + cluster_to_cluster() + + +if __name__ == '__main__': + main() diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..01eae554 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,5 @@ +from .cluster import Cluster +from .constant import REDIS_SERVER_VERSION +from .data_inserter import DataInserter +from .redis import Redis +from .shake import Shake, ShakeOpts diff --git a/tests/helpers/cluster.py b/tests/helpers/cluster.py new file mode 100644 index 00000000..a9b85608 --- /dev/null +++ b/tests/helpers/cluster.py @@ -0,0 +1,47 @@ +import pybbt as p +import redis +from redis.cluster import ClusterNode + +from helpers.redis import Redis + + +class Cluster: + def __init__(self): + self.num = 2 + self.nodes = [] + for i in range(self.num): + self.nodes.append(Redis(args=["--cluster-enabled", "yes"])) + p.ASSERT_EQ(self.nodes[0].do("cluster", "addslots", *range(0, 8192)), b"OK") + p.ASSERT_EQ(self.nodes[1].do("cluster", "addslots", *range(8192, 16384)), b"OK") + p.ASSERT_EQ(self.nodes[0].do("cluster", "meet", self.nodes[1].host, self.nodes[1].port), b"OK") + p.ASSERT_EQ(self.nodes[1].do("cluster", "meet", self.nodes[0].host, self.nodes[0].port), b"OK") + self.client = redis.RedisCluster(startup_nodes=[ + ClusterNode(self.nodes[0].host, self.nodes[0].port), + ClusterNode(self.nodes[1].host, self.nodes[1].port) + ], require_full_coverage=True) + p.ASSERT_EQ_TIMEOUT(lambda: self.client.cluster_info()["cluster_state"], "ok", 10) + p.log(f"cluster started at {self.nodes[0].get_address()}") + p.log(self.client.cluster_nodes()) + + def do(self, *args): + try: + ret = self.client.execute_command(*args) + except redis.exceptions.ResponseError as e: + return f"-{str(e)}" + return ret + + def pipeline(self): + return self.client.pipeline(transaction=False) + + def get_address(self): + return self.nodes[0].get_address() + + def dbsize(self): + size = 0 + for node in self.nodes: + size += node.dbsize() + return size + + @staticmethod + def is_cluster(): + return True diff --git a/tests/helpers/commands/__init__.py b/tests/helpers/commands/__init__.py new file mode 100644 index 00000000..a1027747 --- /dev/null +++ b/tests/helpers/commands/__init__.py @@ -0,0 +1,2 @@ +from .select import SelectChecker +from .string import StringChecker diff --git a/tests/helpers/commands/checker.py b/tests/helpers/commands/checker.py new file mode 100644 index 00000000..756d306b --- /dev/null +++ b/tests/helpers/commands/checker.py @@ -0,0 +1,9 @@ +from helpers.redis import Redis + + +class Checker: + def add_data(self, r: Redis, cross_slots_cmd: bool): + ... + + def check_data(self, r: Redis, cross_slots_cmd: bool): + ... diff --git a/tests/helpers/commands/list.py b/tests/helpers/commands/list.py new file mode 100644 index 00000000..de4d0c80 --- /dev/null +++ b/tests/helpers/commands/list.py @@ -0,0 +1,31 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.redis import Redis + + +class ListChecker(Checker): + PREFIX = "list" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + p = r.pipeline() + p.lpush(f"{self.PREFIX}_{self.cnt}_list", 0) + p.lpush(f"{self.PREFIX}_{self.cnt}_list", 1) + p.lpush(f"{self.PREFIX}_{self.cnt}_list", 2) + p.lpush(f"{self.PREFIX}_{self.cnt}_list_str", "string0") + p.lpush(f"{self.PREFIX}_{self.cnt}_list_str", "string1") + p.lpush(f"{self.PREFIX}_{self.cnt}_list_str", "string2") + ret = p.execute() + pybbt.ASSERT_EQ(ret, [1, 2, 3, 1, 2, 3]) + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + for i in range(self.cnt): + p = r.pipeline() + p.lrange(f"{self.PREFIX}_{i}_list", 0, -1) + p.lrange(f"{self.PREFIX}_{i}_list_str", 0, -1) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [[b"2", b"1", b"0"], [b"string2", b"string1", b"string0"]]) diff --git a/tests/helpers/commands/select.py b/tests/helpers/commands/select.py new file mode 100644 index 00000000..c39c3716 --- /dev/null +++ b/tests/helpers/commands/select.py @@ -0,0 +1,41 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.redis import Redis + + +class SelectChecker(Checker): + PREFIX = "select" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + if not cross_slots_cmd: + return + p = r.pipeline() + p.select(1) + p.set(f"{self.PREFIX}_{self.cnt}_db1", "db1") + p.select(2) + p.set(f"{self.PREFIX}_{self.cnt}_db2", "db2") + p.select(3) + p.set(f"{self.PREFIX}_{self.cnt}_db3", "db3") + p.select(0) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [True, True, True, True, True, True, True]) + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + if not cross_slots_cmd: + return + for i in range(self.cnt): + p = r.pipeline() + p.select(1) + p.get(f"{self.PREFIX}_{i}_db1") + p.select(2) + p.get(f"{self.PREFIX}_{i}_db2") + p.select(3) + p.get(f"{self.PREFIX}_{i}_db3") + p.select(0) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [True, b'db1', True, b'db2', True, b'db3', True]) diff --git a/tests/helpers/commands/string.py b/tests/helpers/commands/string.py new file mode 100644 index 00000000..0e5f0395 --- /dev/null +++ b/tests/helpers/commands/string.py @@ -0,0 +1,31 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.redis import Redis + + +class StringChecker(Checker): + PREFIX = "string" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + p = r.pipeline() + p.set(f"{self.PREFIX}_{self.cnt}_str", "string") + p.set(f"{self.PREFIX}_{self.cnt}_int", 0) + p.set(f"{self.PREFIX}_{self.cnt}_int0", -1) + p.set(f"{self.PREFIX}_{self.cnt}_int1", 123456789) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [True, True, True, True]) + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + for i in range(self.cnt): + p = r.pipeline() + p.get(f"{self.PREFIX}_{i}_str") + p.get(f"{self.PREFIX}_{i}_int") + p.get(f"{self.PREFIX}_{i}_int0") + p.get(f"{self.PREFIX}_{i}_int1") + ret = p.execute() + pybbt.ASSERT_EQ(ret, [b"string", b"0", b"-1", b"123456789"]) diff --git a/tests/helpers/constant.py b/tests/helpers/constant.py new file mode 100644 index 00000000..46f957aa --- /dev/null +++ b/tests/helpers/constant.py @@ -0,0 +1,17 @@ +import shutil +import subprocess +from pathlib import Path + +BASE_PATH = f"{Path(__file__).parent.parent.parent.absolute()}" # project path + +PATH_REDIS_SHAKE = f"{BASE_PATH}/bin/redis-shake" +PATH_REDIS_SERVER = shutil.which('redis-server') +output = subprocess.check_output(f"{PATH_REDIS_SERVER} --version", shell=True) +output_str = output.decode("utf-8") +REDIS_SERVER_VERSION = float(output_str.split("=")[1].split(" ")[0][:3]) + +if __name__ == '__main__': + print(BASE_PATH) + print(PATH_REDIS_SHAKE) + print(PATH_REDIS_SERVER) + print(REDIS_SERVER_VERSION) diff --git a/tests/helpers/data_inserter.py b/tests/helpers/data_inserter.py new file mode 100644 index 00000000..d513205f --- /dev/null +++ b/tests/helpers/data_inserter.py @@ -0,0 +1,18 @@ +from helpers.commands import SelectChecker, StringChecker +from helpers.redis import Redis + + +class DataInserter: + def __init__(self, ): + self.checkers = [ + StringChecker(), + SelectChecker(), + ] + + def add_data(self, r: Redis, cross_slots_cmd: bool): + for checker in self.checkers: + checker.add_data(r, cross_slots_cmd) + + def check_data(self, r: Redis, cross_slots_cmd: bool): + for checker in self.checkers: + checker.check_data(r, cross_slots_cmd) diff --git a/tests/helpers/redis.py b/tests/helpers/redis.py new file mode 100644 index 00000000..e88469f4 --- /dev/null +++ b/tests/helpers/redis.py @@ -0,0 +1,53 @@ +import pybbt +import redis + +from helpers.constant import PATH_REDIS_SERVER +from helpers.utils.network import get_free_port +from helpers.utils.timer import Timer + + +class Redis: + def __init__(self, args=None): + self.case_ctx = pybbt.get_case_context() + if args is None: + args = [] + self.host = "127.0.0.1" + self.port = get_free_port() + self.dir = f"{self.case_ctx.dir}/redis_{self.port}" + args.extend(["--port", str(self.port)]) + self.server = pybbt.Launcher(args=[PATH_REDIS_SERVER] + args, work_dir=self.dir) + self._wait_start() + self.client = redis.Redis(host=self.host, port=self.port) + self.case_ctx.add_exit_hook(lambda: self.server.stop()) + pybbt.log_yellow(f"redis server started at {self.host}:{self.port}, redis-cli -p {self.port}") + + def _wait_start(self, timeout=5): + timer = Timer() + while True: + try: + r = redis.Redis(host=self.host, port=self.port) + r.ping() + return + except redis.exceptions.ConnectionError: + pass + if timer.elapsed() > timeout: + raise TimeoutError("redis server not started") + + def do(self, *args): + try: + ret = self.client.execute_command(*args) + except redis.exceptions.ResponseError as e: + return f"-{str(e)}" + return ret + + def pipeline(self): + return self.client.pipeline(transaction=False) + + def get_address(self): + return f"{self.host}:{self.port}" + + def is_cluster(self): + return self.client.info()["cluster_enabled"] + + def dbsize(self): + return self.client.dbsize() diff --git a/tests/helpers/shake.py b/tests/helpers/shake.py new file mode 100644 index 00000000..2d02a79e --- /dev/null +++ b/tests/helpers/shake.py @@ -0,0 +1,105 @@ +import time +import typing + +import pybbt +import requests +import toml + +from helpers.constant import PATH_REDIS_SHAKE +from helpers.redis import Redis +from helpers.utils.filesystem import create_empty_dir +from helpers.utils.network import get_free_port +from helpers.utils.timer import Timer + + +# [SyncClusterReader] +# address = "127.0.0.1:6379" +# username = "" # keep empty if not using ACL +# password = "" # keep empty if no authentication is required +# tls = false +# +# [RedisClusterWriter] +# address = "127.0.0.1:6380" +# username = "" # keep empty if not using ACL +# password = "" # keep empty if no authentication is required +# tls = false + +class ShakeOpts: + @staticmethod + def create_sync_opts(src: Redis, dst: Redis) -> typing.Dict: + d = {} + if src.is_cluster(): + d["SyncClusterReader"] = {"address": src.get_address()} + else: + d["SyncStandaloneReader"] = {"address": src.get_address()} + if dst.is_cluster(): + d["RedisClusterWriter"] = {"address": dst.get_address()} + else: + d["RedisStandaloneWriter"] = {"address": dst.get_address()} + return d + + @staticmethod + def create_scan_opts(src: Redis, dst: Redis) -> typing.Dict: + d = {} + if src.is_cluster(): + d["ScanClusterReader"] = {"address": src.get_address()} + else: + d["ScanStandaloneReader"] = {"address": src.get_address()} + if dst.is_cluster(): + d["RedisClusterWriter"] = {"address": dst.get_address()} + else: + d["RedisStandaloneWriter"] = {"address": dst.get_address()} + return d + + @staticmethod + def create_rdb_opts(rdb_path: str, dts: Redis) -> typing.Dict: + d = {"RdbReader": {"filepath": rdb_path}} + if dts.is_cluster(): + d["RedisClusterWriter"] = {"address": dts.get_address()} + else: + d["RedisStandaloneWriter"] = {"address": dts.get_address()} + return d + + +class Shake: + def __init__(self, opts: typing.Dict): + self.case_ctx = pybbt.get_case_context() + self.status_port = get_free_port() + self.status_url = f"http://localhost:{self.status_port}" + opts["advanced"] = {"status_port": self.status_port, "log_level": "debug"} + + self.dir = f"{self.case_ctx.dir}/shake{self.status_port}" + create_empty_dir(self.dir) + with open(f"{self.dir}/shake.toml", "w") as f: + toml.dump(opts, f) + self.server = pybbt.Launcher(args=[PATH_REDIS_SHAKE, "shake.toml"], work_dir=self.dir) + self.case_ctx.add_exit_hook(lambda: self.server.stop()) + self._wait_start() + + @staticmethod + def run_once(opts: typing.Dict): + status_port = get_free_port() + run_dir = f"{pybbt.get_case_context().dir}/shake{status_port}" + create_empty_dir(run_dir) + with open(f"{run_dir}/shake.toml", "w") as f: + toml.dump(opts, f) + server = pybbt.Launcher(args=[PATH_REDIS_SHAKE, "shake.toml"], work_dir=run_dir) + server.wait_stop() + + def get_status(self): + ret = requests.get(self.status_url) + return ret.json() + + def _wait_start(self, timeout=5): + timer = Timer() + while True: + try: + self.get_status() + return + except requests.exceptions.ConnectionError: + pass + if timer.elapsed() > timeout: + raise Exception(f"Shake server not started in {timeout} seconds") + + def is_consistent(self): + return self.get_status()["consistent"] diff --git a/test/cases/__init__.py b/tests/helpers/utils/__init__.py similarity index 100% rename from test/cases/__init__.py rename to tests/helpers/utils/__init__.py diff --git a/tests/helpers/utils/filesystem.py b/tests/helpers/utils/filesystem.py new file mode 100644 index 00000000..ba6067de --- /dev/null +++ b/tests/helpers/utils/filesystem.py @@ -0,0 +1,25 @@ +import os +import shutil + + +def file_size(file): + return os.stat(file).st_size + + +def file_truncate(file, size): + with open(file, "rb+") as f: + content = f.read() + f.truncate(size) + return content[size:] + + +def file_append(file, content): + with open(file, "ab") as f: + f.write(content) + + +def create_empty_dir(path): + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(path) + diff --git a/tests/helpers/utils/network.py b/tests/helpers/utils/network.py new file mode 100644 index 00000000..0193f806 --- /dev/null +++ b/tests/helpers/utils/network.py @@ -0,0 +1,45 @@ +import random +import socket +import threading + + +def is_port_available(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(('localhost', port)) + s.close() + return True + except OSError: + return False + + +MIN_PORT = 30000 +MAX_PORT = 40000 + +port_cursor = random.choice(range(MIN_PORT, MAX_PORT, 1000)) + +g_lock = threading.Lock() + + +def get_free_port(): + global port_cursor + global g_lock + with g_lock: + while True: + port_cursor += 1 + if port_cursor == MAX_PORT: + port_cursor = MIN_PORT + + if is_port_available(port_cursor): + return port_cursor + + +__all__ = [ + "is_port_available", + "get_free_port", +] + +if __name__ == '__main__': + # test + for i in range(10): + print(get_free_port()) diff --git a/tests/helpers/utils/rand.py b/tests/helpers/utils/rand.py new file mode 100644 index 00000000..4d598cfe --- /dev/null +++ b/tests/helpers/utils/rand.py @@ -0,0 +1,8 @@ +import random +import string + + +def random_string(length=8) -> str: + chars = string.ascii_letters + string.digits + random_str = ''.join(random.choices(chars, k=length)) + return random_str diff --git a/tests/helpers/utils/timer.py b/tests/helpers/utils/timer.py new file mode 100644 index 00000000..4cfc5ed2 --- /dev/null +++ b/tests/helpers/utils/timer.py @@ -0,0 +1,9 @@ +import time + + +class Timer: + def __init__(self): + self.start_time = time.perf_counter() + + def elapsed(self): + return time.perf_counter() - self.start_time diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..e58a91a5 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +requests>=2.31.0 +toml>=0.10.2 +pybbt>=1.0.1