From c60b7bcf895ad6029b24b5a70b6c108f77c35aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=A4=E5=B2=A1=20=E8=B1=8A=E4=BA=BA?= <toyohito.murooka@P0724-M.local> Date: Mon, 17 Jun 2019 00:39:14 +0900 Subject: [PATCH] init --- .gitignore | 8 + LICENSE | 202 ++++++++++++++++++++++++ README.md | 193 +++++++++++++++++++++++ cmd/conf.go | 118 ++++++++++++++ cmd/explain.go | 363 ++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 65 ++++++++ go.mod | 13 ++ go.sum | 154 +++++++++++++++++++ main.go | 21 +++ model/conf.go | 20 +++ model/explain.go | 70 +++++++++ model/sqlinfo.go | 8 + option.yaml | 19 +++ service/common.go | 18 +++ service/conf.go | 285 ++++++++++++++++++++++++++++++++++ service/db.go | 154 +++++++++++++++++++ service/error.go | 65 ++++++++ service/explain.go | 199 ++++++++++++++++++++++++ service/loadlog.go | 106 +++++++++++++ service/parsesql.go | 105 +++++++++++++ view/render.go | 124 +++++++++++++++ 21 files changed, 2310 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 cmd/conf.go create mode 100644 cmd/explain.go create mode 100644 cmd/root.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 model/conf.go create mode 100644 model/explain.go create mode 100644 model/sqlinfo.go create mode 100644 option.yaml create mode 100644 service/common.go create mode 100644 service/conf.go create mode 100644 service/db.go create mode 100644 service/error.go create mode 100644 service/explain.go create mode 100644 service/loadlog.go create mode 100644 service/parsesql.go create mode 100644 view/render.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5d48f18 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# .DS_Store +.DS_Store + +# _test.go +service/*_test.go + +# .goreleaser.yml +.goreleaser.yml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b99d34e --- /dev/null +++ b/README.md @@ -0,0 +1,193 @@ + +# expl + +MySQL explain checker. +This tool runs multiple SQL explain from SQL log. + +- Read Log or DB(mysql.general_log table) and explain multiple SQL +- Content filtering, Display explain result with 'SELECT_TYPE', 'TABLE', 'TYPE', 'EXTRA' specified + +# simple usage + +``` +$expl explain simple "select * from memo" --database database1 --host localhost --user root --pass "" + + DataBase: database1 + SQL: select * from memo ++----+------------+-------+------------+------+--------------+-----+--------+-----+------+----------+-------+ +| ID | SELECTTYPE | TABLE | PARTITIONS | TYPE | POSSIBLEKEYS | KEY | KEYLEN | REF | ROWS | FILTERED | EXTRA | ++----+------------+-------+------------+------+--------------+-----+--------+-----+------+----------+-------+ +| 1 | SIMPLE | memo | 0 | ALL | | | 0 | | 39 | 100.0000 | | ++----+------------+-------+------------+------+--------------+-----+--------+-----+------+----------+-------+ +``` + +# the case with logs of multiple databases in one file + +## 1. Create config file ("conf" sub command) + +Make config file in YAML format. +This includes relationships between tables and databases. + +- add setting in config file + +``` +# expl conf add host database user pass -c config_file_path + +$expl conf add localhost database1 root "" -c config.yaml +$expl conf add localhost database2 root "" -c config.yaml +``` + +## 2. Execute Explian ("explain" sub command) + +Execute explain multipule SQL + +1. make mapping relationships between tables and databases in config YAML file +2. execute explain SQL using using table-database mapping + +This has advantages such as using the "Combine SQL" (see below) + +``` +#expl explain mode -c config_file_path --format format_type --log sql_log_file_path + +$expl explain log -c config.yaml --format simple --log simple.yaml +``` + +# explain sub command + +### mode parameter + +2nd Parameter + +- simple : SQL direct input. The third parameter is sql. +- log : Getting SQL from log file. (official generate log or custom log file) +- log-db : Getting SQL from database. (mysql.general_log table) + +``` +# simple mode +$expl explain simple "select * from memo" --database database1 --host localhost --user root --pass "" + +# log mode +$expl explain log --conf config.yaml --format official --log sql.log + +# log-db mode +$expl explain log-db --conf config.yaml --format official +``` + +### conf option + +This is Config file path. The config file includes host, database, user, password and table-database mapping. + + +### log option + +This is Log file path. + +### format option + +format of one line in SQL log file. + +- simple : Raw SQL +- official : Same log format of MySQL general_log. https://dev.mysql.com/doc/refman/5.6/en/query-log.html +- command : Edit by OS command + +``` +# simple format +$expl explain simple "select * from memo" --database database1 --host localhost --user root --pass "" + +# official format +$expl explain log --conf config.yaml --format official --log /var/lib/mysql/general_sql.log + +# command format +$expl explain log --conf config.yaml --log custom_sql.log --format command --format-cmd "cut -c 21-" +``` + +#### format-cmd option + +Using only "command" format. +OS command for edit line of log to raw SQL. + +``` +$expl explain log --conf config.yaml --log custom_sql.log --format command --format-cmd "cut -c 21-" + +# same (using pipe simple mode) +$cut -c 21- custom_sql.log | xargs -I$ expl explain simple "$" --conf config.yaml --format command --format-cmd "cut -c 21-" +``` + +### Combine SQL option + +Display the same type of SQL results in one view. +The two SQL statements below are identical to the explain result. Thus, SQLs of the same type are displayed together in one + +``` +# sql1 +select * from memo where id = 1; + +# sql2 +select * from memo where id = 100; +``` + +``` +$expl explain log --conf config.yaml --format official --log /var/lib/mysql/general_sql.log --combine-sql +``` + +### filter option + +Filtering the explain results + +| option | meaning | +----|---- +| filter-select-type | Show only results included in the specified "Select Type" of explain | +| filter-no-select-type | Show only results that are not included in the specified "Select Type" of explain | +| filter-table | Show only results that contain the specified table | +| filter-no-table | Show only results that do not contain the specified table | +| filter-type | Show only results included in the specified "Type" of explain | +| filter-no-type | Show only results that are not included in the specified "Type" of explain | +| filter-extra | Show only results included in the specified "Extra" of explain | +| filter-no-extra | Show only results that are not included in the specified "Extra" of explain | + +``` +# view only results where includes "ALL" in TYPE column. + +$expl explain log --conf config.yaml --format official --log /var/lib/mysql/general_sql.log --filter-type ALL + + DataBase: memo_sample + SQL: select tag.* from tag, tag_memo where tag.id = tag_memo.tag_id ++----+------------+----------+------------+------+--------------+---------+--------+--------------------+------+----------+-------------+ +| ID | SELECTTYPE | TABLE | PARTITIONS | TYPE | POSSIBLEKEYS | KEY | KEYLEN | REF | ROWS | FILTERED | EXTRA | ++----+------------+----------+------------+------+--------------+---------+--------+--------------------+------+----------+-------------+ +| 1 | SIMPLE | tag | 0 | ALL | PRIMARY | | 0 | | 22 | 100.0000 | | +| 1 | SIMPLE | tag_memo | 0 | ref | PRIMARY | PRIMARY | 4 | memo_sample.tag.id | 1 | 100.0000 | Using index | ++----+------------+----------+------------+------+--------------+---------+--------+--------------------+------+----------+-------------+ +``` + +### ignore error option + +This is to ignore the "Explain SQL Error" or "SQL Parse Error". + +``` +$expl explain log --conf config.yaml --format official --log /var/lib/mysql/general_sql.log --ignore-error +``` +With SQL parse or explain SQL errors, let's try use this option. + +- If table "user" exists in "database1" and "database2", this tool will try to explain both "database1" and "database2". In this case, this option is useful to ignore the error and do the following processing. +- If the log contains only one unparsable string, using this option will not stop the execution of subsequent correct SQL statement lines. + +### option file option + +Using file for option settings +You can use YAML files instead of directly specifying options in the command + +If there are duplicate definitions, priority is given in the following order +1. Command +2. env +3. option file + +``` +$expl explain log --option-file ./option.yaml --filter-extra "using where" +``` + +### verbose output option + +Display the value of the option just before execution + + diff --git a/cmd/conf.go b/cmd/conf.go new file mode 100644 index 0000000..5a73ed2 --- /dev/null +++ b/cmd/conf.go @@ -0,0 +1,118 @@ +// Copyright © 2019 NAME HERE <EMAIL ADDRESS> +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "errors" + "expl/service" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +// confCmd represents the conf command +var confCmd = &cobra.Command{ + Use: "conf", + Short: "conf action", + Long: `conf action [option] + action: + add: add database hosting + rm: rm database hosting + `, + + //SilenceUsage: true, + + Args: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return errors.New("requires action") + } + return nil + }, + + RunE: func(cmd *cobra.Command, args []string) error { + + ctx := context.Background() + + // config file path + filePath, err := cmd.Flags().GetString("conf") + if err != nil { + return err + } + if filePath == "" { + filePath = os.Getenv("EXPL_CONF") + } + + action := args[0] + + switch action { + case "add": + host := args[1] + database := args[2] + user := args[3] + pass := args[4] + + err = service.AddHostAndDatabase(ctx, user, pass, host, database, filePath) + fmt.Printf("conf %s %s %s %s %s %s\n", action, host, database, user, pass, filePath) + case "rm": + host := args[1] + database := args[2] + user := args[3] + pass := args[4] + + err = service.RemoveHostAndDatabase(ctx, user, pass, host, database, filePath) + fmt.Printf("conf %s %s %s %s %s %s\n", action, host, database, user, pass, filePath) + case "mapping": + err = service.ReloadAllTableInfo(ctx, filePath) + fmt.Printf("conf %s\n", action) + default: + return fmt.Errorf("%s is invalid action\n", action) + } + + if err != nil { + fmt.Print(service.Message(err)) + } + + return err + }, +} + +func init() { + rootCmd.AddCommand(confCmd) + + template := `expl conf action [parameter]... + action: + add: add database and host, user, password in setting used to explain sql. + ex) add host databasea user password + rm: remove database and host, user, password in setting used to explain sql. + ex) rm host database user password + mapping: create or update table-dtabase mapping. using sing Host and Databse settings created by the above add action command. + + parameter: + ex) + expl conf add localhost database1 root password -c configpath + expl conf rm localhost database2 root password -c configpath + expl conf mapping -c configpath // make table-database mapping file in database1 and database2. + + option: + -c, --conf: config file. You can set and use "EXPL_CONF" environment variable as a default value. +` + + confCmd.SetUsageTemplate(template) + confCmd.SetHelpTemplate(template) + + confCmd.Flags().StringP("conf", "c", "", "config. which includes database-table mapping file.") +} diff --git a/cmd/explain.go b/cmd/explain.go new file mode 100644 index 0000000..9cdb1ce --- /dev/null +++ b/cmd/explain.go @@ -0,0 +1,363 @@ +// Copyright © 2019 NAME HERE <EMAIL ADDRESS> +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "expl/model" + "expl/service" + "expl/view" + + "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var explainTemplate string = `expl explain mode [option] [sql] + +mode: + simple: explain one sql + log: explain sqls from log file + log-db: explain sqls from database. + +sql: it's necessary in only "simple" mode. + +option: + -d --database: string database. used by any sqls. (using onlh simple mode. not using in table mapping mode) + -H --host: string host used by any sqls.(using onlh simple mode. not using table in mapping mode) + -u --user: string database user used by any sqls.(using onlh simple mode. not using table in mapping mode) + -p --pass: string database password used by any sqls.(using onlh simple mode. not using table in mapping mode) + -c, --conf: string use table mapping. This is efficient for switching hosts and databases by table. + "EXPL_CONF" environment variable as a default value. + value: + [mapping file path]: database-table mapping file path. default file is ./table_map.yaml. + ex) + -c $GOPATH/bin/table-mapping.yaml + -l, --log: string log file path. This is used in log mode. "EXPL_LOG" environment variable as a default value. + -fm, --format: string sql format. + value: + simple (default): simple is only sql. + official: sql is offical mysql sql.log's format. + command: change text using os command. format-cmd option required. + ex) + -f simple + -f official + -f command + --format-cmd: string os command string used only when fomat option is "command" + ex) + --format-cmd "cut -c 14-" + --filter-select-type: strings filter results by target select types. + ex) + --filter-select-type simple, subquery + appear results with SIMPLE or SUBQUERY selected-types. + --filter-no-select-type: strings filter results without target select types. + --filter-table: strings filter results by target tables. + ex) + --filter-table user, group + appear results, table of which is "user" or "group". + --filter-no-table: strings filter results without target tables. + --filter-type: strings filter results by target types. + ex) + --filter-type index, ref + appear results wich "index" or "ref" types. + --filter-no-type: strings filter results without target types. + --filter-extra: strings filter results by target taypes. + ex) + --filter-extra filesort, "using where" + appear results wich "filesort" or "using where" types. + --filter-no-extra: strings filter results without target types. + + -U, --update-table-map: update table-database mapping file before do explain sql. use current database environment. + -I, --ignore-error: ignore parse sql error or explain sql error. + -C, --combine-sql: This is useful in log or log-db module. combine identical SQL into one. + + --option-file: you can use option setting file. with this file you do not have to enter the above optional parameters. + "EXPL_OPTION" environment variable as a default value. + -v, --verbose: verbose output. + -h, --help: help. show usage. +` + +func validateArgs(args []string) error { + if len(args) == 0 { + return fmt.Errorf("invalit parameter.") + } + + mode := args[0] + switch mode { + case "simple": + if len(args) < 2 { + return fmt.Errorf("sql is none.") + } + case "log": + return nil + case "log-db": + return nil + default: + return fmt.Errorf("invalid mode. mode mast be (simple, log, log-db).") + } + + return nil +} + +// explainCmd represents the explain command +var explainCmd = &cobra.Command{ + Use: "explain", + Short: "A brief description of your command", + RunE: func(cmd *cobra.Command, args []string) error { + + // validate parameter + if err := validateArgs(args); err != nil { + return err + } + + // mode + mode := args[0] + + // sql + sql := "" + if mode == "simple" { + sql = args[1] + } + + optionFilePath, err := cmd.Flags().GetString("option-file") + if err != nil { + return err + } + if optionFilePath == "" { + optionFilePath = os.Getenv("EXPL_OPTION") + } + if optionFilePath != "" { + viper.SetConfigFile(optionFilePath) + + if err := viper.ReadInConfig(); err != nil { + return err + } + } + + expOpt := new(model.ExplainOption) + + expOpt.DB = viper.GetString("database") + expOpt.DBHost = viper.GetString("host") + expOpt.DBUser = viper.GetString("user") + expOpt.DBPass = viper.GetString("pass") + expOpt.Config = viper.GetString("conf") + if expOpt.Config == "" { + expOpt.Config = os.Getenv("EXPL_CONF") + } + + // log file + logPath := viper.GetString("log") + if logPath == "" { + logPath = os.Getenv("EXPL_LOG") + } + + // format + format := viper.GetString("format") + formatCmd := viper.GetString("format-cmd") + if formatCmd != "" { + format = string(service.FormatCommand) + } + + // filter options + fiOpt := new(model.ExplainFilter) + + fiOpt.SelectType = viper.GetStringSlice("filter-select-type") + fiOpt.SelectTypeNot = viper.GetStringSlice("filter-no-select-type") + fiOpt.Table = viper.GetStringSlice("filter-table") + fiOpt.TableNot = viper.GetStringSlice("filter-no-table") + fiOpt.Type = viper.GetStringSlice("filter-type") + fiOpt.TypeNot = viper.GetStringSlice("filter-no-type") + fiOpt.Extra = viper.GetStringSlice("filter-extra") + fiOpt.ExtraNot = viper.GetStringSlice("filter-no-extra") + + expOpt.UseTableMap = viper.GetBool("update-table-map") + if expOpt.DB != "" && expOpt.DBHost != "" && expOpt.DBUser != "" { + expOpt.UseTableMap = false + } + + expOpt.NoError = viper.GetBool("ignore-error") + expOpt.Uniq = viper.GetBool("combine-sql") + + verbose, err := cmd.Flags().GetBool("verbose") + if err != nil { + return err + } + if verbose { + view.RenderOptions(expOpt, fiOpt, logPath, format, formatCmd) + } + + ctx := context.Background() + + switch mode { + case "simple": + err = func() error { + if expOpt.UseTableMap { + if err = service.ReloadAllTableInfo(ctx, expOpt.Config); err != nil { + return err + } + + if err = service.LoadDBInfo(ctx, expOpt.Config); err != nil { + return err + } + + } else { + service.SetDBOne( + expOpt.DBHost, + expOpt.DB, + expOpt.DBUser, + expOpt.DBPass, + ) + } + + sql, err = service.GetQueryByFormat(service.FormatType(format), sql, formatCmd) + if err != nil { + return err + } + + list, err := service.Explains(ctx, []string{sql}, expOpt, fiOpt) + if err == nil { + view.RenderExplains(list, false) + } + return err + }() + case "log": + err = func() error { + sqls, err := service.LoadQueriesFromLog(ctx, logPath, service.FormatType(format), formatCmd) + if err != nil { + return err + } + + if err = service.ReloadAllTableInfo(ctx, expOpt.Config); err != nil { + return err + } + + if err = service.LoadDBInfo(ctx, expOpt.Config); err != nil { + return err + } + + list, err := service.Explains(ctx, sqls, expOpt, fiOpt) + if err == nil { + view.RenderExplains(list, expOpt.Uniq) + } + return err + }() + + case "log-db": + err = func() error { + if err = service.ReloadAllTableInfo(ctx, expOpt.Config); err != nil { + return err + } + + if err = service.LoadDBInfo(ctx, expOpt.Config); err != nil { + return err + } + + sqls, err := service.LoadQueriesFromDB(ctx) + if err != nil { + return err + } + + list, err := service.Explains(ctx, sqls, expOpt, fiOpt) + if err == nil { + view.RenderExplains(list, expOpt.Uniq) + } + return err + }() + } + + if err != nil { + fmt.Println("error occured!") + fmt.Println(service.Message(err)) + } + + return err + }, +} + +func init() { + rootCmd.AddCommand(explainCmd) + explainCmd.SetUsageTemplate(explainTemplate) + explainCmd.SetHelpTemplate(explainTemplate) + + //viper.SetConfigType("yml") + + viper.SetDefault("format", "simple") + + viper.SetDefault("filter-select-type", []string{}) + viper.SetDefault("filter-no-select-type", []string{}) + viper.SetDefault("filter-table", []string{}) + viper.SetDefault("filter-no-table", []string{}) + viper.SetDefault("filter-type", []string{}) + viper.SetDefault("filter-no-type", []string{}) + viper.SetDefault("filter-extra", []string{}) + viper.SetDefault("filter-no-extra", []string{}) + + viper.SetDefault("update-table-map", true) + + explainCmd.PersistentFlags().StringP("database", "d", "", "database") + explainCmd.PersistentFlags().StringP("host", "H", "", "host") + explainCmd.PersistentFlags().StringP("user", "u", "", "database user") + explainCmd.PersistentFlags().StringP("pass", "p", "", "database password") + explainCmd.PersistentFlags().StringP("conf", "c", "", "config. which includes database-table mapping file.") + explainCmd.PersistentFlags().StringP("log", "l", "", "sql log file path.") + explainCmd.PersistentFlags().StringP("format", "f", "simple", "format of the line.") + explainCmd.PersistentFlags().StringP("format-cmd", "", "", "os command to update line.") + explainCmd.PersistentFlags().StringSlice("filter-select-type", []string{}, "filter results by target select types.") + explainCmd.PersistentFlags().StringSlice("filter-no-select-type", []string{}, "filter results without target select types.") + explainCmd.PersistentFlags().StringSlice("filter-table", []string{}, "filter results by target tables.") + explainCmd.PersistentFlags().StringSlice("filter-no-table", []string{}, "filter results without target tables.") + explainCmd.PersistentFlags().StringSlice("filter-type", []string{}, "filter results by target types.") + explainCmd.PersistentFlags().StringSlice("filter-no-type", []string{}, "filter results without target types.") + explainCmd.PersistentFlags().StringSlice("filter-extra", []string{}, "strings filter results by target taypes.") + explainCmd.PersistentFlags().StringSlice("filter-no-extra", []string{}, "filter results without target types.") + explainCmd.PersistentFlags().BoolP("update-table-map", "U", true, "update table-database mapping file before do explain sql. use current database environment.") + explainCmd.PersistentFlags().BoolP("ignore-error", "I", false, "ignore sql error.") + explainCmd.PersistentFlags().BoolP("combine-sql", "C", false, "This is useful in log or log-db module. combine identical SQL into one.") + + explainCmd.Flags().StringP("option-file", "", "", "option yaml file.") + explainCmd.Flags().BoolP("verbose", "v", false, "verbose output.") + + viper.BindPFlag("database", explainCmd.PersistentFlags().Lookup("database")) + viper.BindPFlag("database", explainCmd.PersistentFlags().ShorthandLookup("d")) + viper.BindPFlag("host", explainCmd.PersistentFlags().Lookup("host")) + viper.BindPFlag("host", explainCmd.PersistentFlags().ShorthandLookup("H")) + viper.BindPFlag("user", explainCmd.PersistentFlags().Lookup("user")) + viper.BindPFlag("user", explainCmd.PersistentFlags().ShorthandLookup("u")) + viper.BindPFlag("pass", explainCmd.PersistentFlags().Lookup("pass")) + viper.BindPFlag("pass", explainCmd.PersistentFlags().ShorthandLookup("p")) + viper.BindPFlag("conf", explainCmd.PersistentFlags().Lookup("conf")) + viper.BindPFlag("conf", explainCmd.PersistentFlags().ShorthandLookup("c")) + viper.BindPFlag("log", explainCmd.PersistentFlags().Lookup("log")) + viper.BindPFlag("log", explainCmd.PersistentFlags().ShorthandLookup("l")) + viper.BindPFlag("format", explainCmd.PersistentFlags().Lookup("format")) + viper.BindPFlag("format", explainCmd.PersistentFlags().ShorthandLookup("f")) + viper.BindPFlag("format-cmd", explainCmd.PersistentFlags().Lookup("format-cmd")) + viper.BindPFlag("filter-select-type", explainCmd.PersistentFlags().Lookup("filter-select-type")) + viper.BindPFlag("filter-no-select-type", explainCmd.PersistentFlags().Lookup("filter-no-select-type")) + viper.BindPFlag("filter-table", explainCmd.PersistentFlags().Lookup("filter-table")) + viper.BindPFlag("filter-no-table", explainCmd.PersistentFlags().Lookup("filter-no-table")) + viper.BindPFlag("filter-type", explainCmd.PersistentFlags().Lookup("filter-type")) + viper.BindPFlag("filter-no-type", explainCmd.PersistentFlags().Lookup("filter-no-type")) + viper.BindPFlag("filter-extra", explainCmd.PersistentFlags().Lookup("filter-extra")) + viper.BindPFlag("filter-no-extra", explainCmd.PersistentFlags().Lookup("filter-no-extra")) + viper.BindPFlag("update-table-map", explainCmd.PersistentFlags().Lookup("update-table-map")) + viper.BindPFlag("update-table-map", explainCmd.PersistentFlags().ShorthandLookup("U")) + viper.BindPFlag("ignore-error", explainCmd.PersistentFlags().Lookup("ignore-error")) + viper.BindPFlag("ignore-error", explainCmd.PersistentFlags().ShorthandLookup("I")) + viper.BindPFlag("combine-sql", explainCmd.PersistentFlags().Lookup("combine-sql")) + viper.BindPFlag("combine-sql", explainCmd.PersistentFlags().ShorthandLookup("C")) +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..cc7b0e9 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,65 @@ +// Copyright © 2019 NAME HERE <EMAIL ADDRESS> +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var cfgFile string + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "expl", + Short: "A brief description of your application", + Long: `A longer description that spans multiple lines and likely contains +examples and usage of using your application. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + // Uncomment the following line if your bare application + // has an action associated with it: + // Run: func(cmd *cobra.Command, args []string) { }, + Run: func(cmd *cobra.Command, args []string) { + versionFlag, err := cmd.Flags().GetBool("version") + if err == nil && versionFlag { + fmt.Println("1.0.0") + } + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + //rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.expl.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.Flags().BoolP("version", "", true, "version") +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..79c8feb --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module expl + +require ( + github.com/go-sql-driver/mysql v1.4.1 + github.com/mattn/go-runewidth v0.0.4 // indirect + github.com/olekukonko/tablewriter v0.0.1 + github.com/pkg/errors v0.8.1 + github.com/spf13/cobra v0.0.5 + github.com/spf13/viper v1.4.0 + github.com/srvc/fail v4.0.0+incompatible + github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0ba833d --- /dev/null +++ b/go.sum @@ -0,0 +1,154 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/srvc/fail v4.0.0+incompatible h1:NyG9nEm+AG/5RcwtmhsjCAwFgycb/MRGe3X78BR+w2U= +github.com/srvc/fail v4.0.0+incompatible/go.mod h1:O6qL3O5sUrJlIh2oGAZ5J9RKGGae3D1U6jMGyzM6RK4= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= +github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go new file mode 100644 index 0000000..3dea97d --- /dev/null +++ b/main.go @@ -0,0 +1,21 @@ +// Copyright © 2019 NAME HERE <EMAIL ADDRESS> +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file service.ept in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either service.ress or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "expl/cmd" + +func main() { + cmd.Execute() +} diff --git a/model/conf.go b/model/conf.go new file mode 100644 index 0000000..bf4deea --- /dev/null +++ b/model/conf.go @@ -0,0 +1,20 @@ +package model + +type DBInfo struct { + Hosts []*DBHost +} + +type DBHost struct { + Address string + User string + Password string + Databases []*DBDatabase +} + +type DBDatabase struct { + Name string + Tables []string +} + +// key:table value:database +type TableDBMap map[string][]string diff --git a/model/explain.go b/model/explain.go new file mode 100644 index 0000000..9a5a486 --- /dev/null +++ b/model/explain.go @@ -0,0 +1,70 @@ +package model + +// Explain struct of expl query result +type Explain struct { + ID int64 + SelectType string + Table string + Partitions int64 + Type string + PossibleKeys string + Key string + KeyLen int64 + Ref string + Rows int64 + Filtered float64 + Extra string +} + +// ExplainInfo struct of expl query result +type ExplainInfo struct { + DataBase string + PrepareSQL string + SQL string + Values []*Explain +} + +// ExplainFilter filter option +type ExplainFilter struct { + SelectType []string + SelectTypeNot []string + Table []string + TableNot []string + Type []string + TypeNot []string + Extra []string + ExtraNot []string +} + +// ExplainOption option of service.Explain +type ExplainOption struct { + // useTableMap + UseTableMap bool + + // table-database map + TableMap TableDBMap + + // Config path + Config string + + // database (no use table mapping) + DB string + + // database host (no use table mapping) + DBHost string + + // database user (no use table mapping) + DBUser string + + // database password (no use table mapping) + DBPass string + + // uniq + Uniq bool + // no error + NoError bool + // table + Table map[string]uint + // without table + TableNot map[string]uint +} diff --git a/model/sqlinfo.go b/model/sqlinfo.go new file mode 100644 index 0000000..e666a81 --- /dev/null +++ b/model/sqlinfo.go @@ -0,0 +1,8 @@ +package model + +// SQLInfo SQL情報 +type SQLInfo struct { + PrepareSQL string + Table string + Databse string +} diff --git a/option.yaml b/option.yaml new file mode 100644 index 0000000..e8be096 --- /dev/null +++ b/option.yaml @@ -0,0 +1,19 @@ +database: +host: +user: +pass: +conf: +log: +format: +format-cmd: +filter-select-type: [] +filter-no-select-type: [] +filter-table: [] +filter-no-table: [] +filter-type: [] +filter-no-type: [] +filter-extra: [] +filter-no-extra: [] +update-table-map: +ignore-error: +combine-sql: diff --git a/service/common.go b/service/common.go new file mode 100644 index 0000000..976ebc7 --- /dev/null +++ b/service/common.go @@ -0,0 +1,18 @@ +package service + +import ( + "os" + "path" + "path/filepath" +) + +func getPath(filePath string) (string, error) { + + if path.IsAbs(filePath) { + return filePath, nil + } + + p, err := os.Getwd() + + return path.Clean(filepath.Join(p, filePath)), err +} diff --git a/service/conf.go b/service/conf.go new file mode 100644 index 0000000..6a1e3d0 --- /dev/null +++ b/service/conf.go @@ -0,0 +1,285 @@ +package service + +import ( + "context" + "expl/model" + "fmt" + + "io/ioutil" + "os" + + yaml "gopkg.in/yaml.v2" +) + +type config struct { + Hosts []*host `yaml:"hosts"` + Databases []*database `yaml:"databases"` +} + +type host struct { + Key int `yaml:"key"` + Address string `yaml:"address"` + User string `yaml:"user"` + Password string `yaml:"password"` +} + +type database struct { + HostKey int `yaml:"hostkey"` + Name string `yaml:"name"` + Tables []string `yaml:"tables"` +} + +var dbInfo *model.DBInfo + +// setConfig write config +func setConfig(ctx context.Context, conf *config, filePath string) error { + + filePath, err := getPath(filePath) + if err != nil { + return err + } + + buf, err := yaml.Marshal(conf) + if err != nil { + return ErrWrap(err, UserInputError) + } + + err = ioutil.WriteFile(filePath, buf, os.ModePerm) + if err != nil { + return ErrWrap(err, UserInputError) + } + + return nil +} + +// getConfig get config +func getConfig(ctx context.Context, filePath string) (*config, error) { + // 外部からconfの中身を参照できるようにする + var c config + + filePath, err := getPath(filePath) + if err != nil { + return nil, err + } + + buf, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, ErrWrap(err, UserInputError) + } + + err = yaml.Unmarshal(buf, &c) + if err != nil { + return nil, ErrWrap(err, UserInputError) + } + + return &c, nil +} + +func AddHostAndDatabase(ctx context.Context, user, pass, address, dbName, filePath string) error { + + conf := new(config) + if _, err := os.Stat(filePath); err == nil { + conf, err = getConfig(ctx, filePath) + if err != nil { + return err + } + } + + if conf.Hosts == nil { + conf.Hosts = []*host{} + } + + if conf.Databases == nil { + conf.Databases = []*database{} + } + + // add Host Info + var ho *host + for _, h := range conf.Hosts { + if h.User == user && h.Password == pass && h.Address == address { + ho = h + break + } + } + + if ho == nil { + ho = &host{ + Key: len(conf.Hosts) + 1, + User: user, + Password: pass, + Address: address, + } + + conf.Hosts = append(conf.Hosts, ho) + } + + hostKey := ho.Key + + var db *database + + // add Database Info + for _, d := range conf.Databases { + if d.HostKey == hostKey && d.Name == dbName { + db = d + break + } + } + + if db == nil { + db = &database{ + HostKey: hostKey, + Name: dbName, + } + + conf.Databases = append(conf.Databases, db) + } + + return setConfig(ctx, conf, filePath) +} + +func RemoveHostAndDatabase(ctx context.Context, user, pass, address, dbName, filePath string) error { + + conf, err := getConfig(ctx, filePath) + if err != nil { + return err + } + + // add Host Info + var ho *host + for _, h := range conf.Hosts { + if h.User == user && h.Password == pass && h.Address == address { + ho = h + break + } + } + + if ho == nil { + return ErrWrap( + fmt.Errorf("none data user:%s, pass:%s, address:%s", user, pass, address), + UserInputError, + ) + } + + hostKey := ho.Key + + var db *database + + dbs := make([]*database, 0, len(conf.Databases)) + + // add Database Info + for _, d := range conf.Databases { + if d.HostKey == hostKey && d.Name == dbName { + db = d + continue + } + dbs = append(dbs, d) + } + + if db == nil { + return ErrWrap( + fmt.Errorf("none data user:%s, pass:%s, address:%s, database:%s", + user, pass, address, dbName, + ), + UserInputError, + ) + } + conf.Databases = dbs + + return setConfig(ctx, conf, filePath) +} + +func ReloadAllTableInfo(ctx context.Context, filePath string) error { + conf, err := getConfig(ctx, filePath) + if err != nil { + return err + } + + for _, db := range conf.Databases { + for _, h := range conf.Hosts { + if db.HostKey != h.Key { + continue + } + + err := openAdditional(ctx, h.User, h.Password, h.Address, db.Name) + if err != nil { + return err + } + + tables, err := showtables(db.Name) + if err != nil { + return err + } + + db.Tables = tables + } + } + + return setConfig(ctx, conf, filePath) +} + +func GetDBInfo(ctx context.Context) *model.DBInfo { + return dbInfo +} + +func SetDBInfo(ctx context.Context, info *model.DBInfo) { + dbInfo = info +} + +func SetDBOne(host, database, user, pass string) { + dbInfo = &model.DBInfo{ + Hosts: []*model.DBHost{ + &model.DBHost{ + Address: host, + Databases: []*model.DBDatabase{&model.DBDatabase{Name: database}}, + User: user, + Password: pass, + }, + }, + } +} + +func LoadDBInfo(ctx context.Context, filePath string) error { + conf, err := getConfig(ctx, filePath) + if err != nil { + return err + } + + dbs := []*model.DBHost{} + for _, ho := range conf.Hosts { + dbh := &model.DBHost{Address: ho.Address, User: ho.User, Password: ho.Password, Databases: []*model.DBDatabase{}} + for _, d := range conf.Databases { + if d.HostKey != ho.Key { + continue + } + + dbd := &model.DBDatabase{Name: d.Name, Tables: d.Tables} + dbh.Databases = append(dbh.Databases, dbd) + } + dbs = append(dbs, dbh) + } + + SetDBInfo(ctx, &model.DBInfo{dbs}) + + return nil +} + +func GetTableDBMap(ctx context.Context) model.TableDBMap { + info := GetDBInfo(ctx) + + tbMap := map[string][]string{} + + for _, h := range info.Hosts { + for _, db := range h.Databases { + for _, tb := range db.Tables { + tDbs := []string{} + if v, ok := tbMap[tb]; ok { + tDbs = v + } + tDbs = append(tDbs, db.Name) + tbMap[tb] = tDbs + } + } + } + + return model.TableDBMap(tbMap) +} diff --git a/service/db.go b/service/db.go new file mode 100644 index 0000000..4764628 --- /dev/null +++ b/service/db.go @@ -0,0 +1,154 @@ +package service + +import ( + "context" + "database/sql" + "expl/model" + + "fmt" + + "github.com/pkg/errors" + + _ "github.com/go-sql-driver/mysql" +) + +func init() { + dbMap = map[string]*sql.DB{} + dbType = "mysql" + officialDB = "mysql" +} + +var dbMap map[string]*sql.DB + +type dbm struct{} + +var dbType string + +var officialDB string + +type explainResult struct { + ID int64 + SelectType string + Table string + Partitions sql.NullInt64 + Type sql.NullString + PossibleKeys sql.NullString + Key sql.NullString + KeyLen sql.NullInt64 + Ref sql.NullString + Rows sql.NullInt64 + Filtered sql.NullFloat64 + Extra sql.NullString +} + +// openAdditional +func openAdditional(ctx context.Context, user, pass, address, database string) error { + + db, err := open(user, pass, address, database) + if err != nil { + return ErrWrap(err, OtherError) + } + + dbMap[database] = db + + return nil +} + +// open +func open(user, pass, address, database string) (*sql.DB, error) { + if address == "localhost" { + address = "" + } + + dataSourse := fmt.Sprintf("%s:%s@%s/%s", user, pass, address, database) + return sql.Open(dbType, dataSourse) +} + +// explain +func explain(ctx context.Context, database, sql string) ([]*model.Explain, error) { + list := make([]*model.Explain, 0) + + exQuery := fmt.Sprintf("explain %s", sql) + + rows, err := query(database, exQuery) + if err != nil { + return list, ErrWrap(err, ExeExplainError) + } + + for rows.Next() { + ex := &explainResult{} + err := rows.Scan(&ex.ID, &ex.SelectType, &ex.Table, &ex.Partitions, &ex.Type, &ex.PossibleKeys, &ex.Key, &ex.KeyLen, &ex.Ref, &ex.Rows, &ex.Filtered, &ex.Extra) + if err != nil { + return list, ErrWrap(err, ExeExplainError) + } + exm := &model.Explain{ + ID: ex.ID, + SelectType: ex.SelectType, + Table: ex.Table, + Partitions: ex.Partitions.Int64, + Type: ex.Type.String, + PossibleKeys: ex.PossibleKeys.String, + Key: ex.Key.String, + KeyLen: ex.KeyLen.Int64, + Ref: ex.Ref.String, + Rows: ex.Rows.Int64, + Filtered: ex.Filtered.Float64, + Extra: ex.Extra.String, + } + + list = append(list, exm) + } + + defer rows.Close() + + if err := rows.Err(); err != nil { + return list, ErrWrap(err, ExeExplainError) + } + + return list, nil +} + +// query +func query(database, querySQL string) (*sql.Rows, error) { + + var db *sql.DB + var ok bool + if db, ok = dbMap[database]; !ok { + return nil, errors.Errorf("database is none. database:%s", database) + } + + return db.Query(querySQL) +} + +func showtables(database string) ([]string, error) { + tables := []string{} + + rows, err := query(database, "show tables") + if err != nil { + return nil, ErrWrap(err, ShowTablesError) + } + + for rows.Next() { + tb := "" + err := rows.Scan(&tb) + if err != nil { + return nil, ErrWrap(err, ShowTablesError) + } + tables = append(tables, tb) + } + + defer rows.Close() + + if err := rows.Err(); err != nil { + return tables, ErrWrap(err, ShowTablesError) + } + + return tables, nil +} + +// close +func closeAll(ctx context.Context) { + for _, db := range dbMap { + db.Close() + } +} diff --git a/service/error.go b/service/error.go new file mode 100644 index 0000000..89803ce --- /dev/null +++ b/service/error.go @@ -0,0 +1,65 @@ +package service + +import ( + "fmt" + "strings" + + "github.com/srvc/fail" +) + +type ErrorCode int + +const ( + SQLParseError ErrorCode = iota + 1 + ExeExplainError + ShowTablesError + UserInputError + OtherError +) + +var errorMessage = map[ErrorCode]string{ + SQLParseError: "sql parse error", + ExeExplainError: "explain error", + ShowTablesError: "execute show tables sql error", + UserInputError: "invalid parameter you inputed", + OtherError: "error", +} + +func ErrWrap(err error, code ErrorCode) error { + return fail.Wrap( + err, + fail.WithCode(int(code)), + fail.WithIgnorable(), + ) +} + +func ErrWrapWithMessage(err error, code ErrorCode, msg string) error { + return fail.Wrap( + err, + fail.WithCode(int(code)), + fail.WithIgnorable(), + fail.WithMessage(msg), + ) +} + +func Message(err error) string { + msg := strings.Join(fail.Unwrap(err).Messages, "\n") + + return fmt.Sprintf("%s %s\nStackTrace:%+v\n", + errorMessage[ErrorCode(ErrCode(err))], + msg, + fail.Unwrap(err).StackTrace, + ) +} + +func LogMessage(err error) string { + return fmt.Sprintf("%T\nCode:%d\nStackTrace:%+v\n", + err, + fail.Unwrap(err).Code, + fail.Unwrap(err).StackTrace, + ) +} + +func ErrCode(err error) int { + return fail.Unwrap(err).Code.(int) +} diff --git a/service/explain.go b/service/explain.go new file mode 100644 index 0000000..9b7adbd --- /dev/null +++ b/service/explain.go @@ -0,0 +1,199 @@ +package service + +import ( + "context" + "expl/model" + "strings" +) + +// Explains execute explain queries +func Explains( + ctx context.Context, + queries []string, + option *model.ExplainOption, + fi *model.ExplainFilter, +) ([]*model.ExplainInfo, error) { + infos := make([]*model.ExplainInfo, 0) + + option.TableMap = GetTableDBMap(ctx) // TODO: ここでやるべき? + + if err := openAdditonal(ctx, GetDBInfo(ctx)); err != nil { + return infos, err + } + + infos, err := exeExplains(ctx, queries, option) + if err != nil { + return infos, err + } + + return filterResults(infos, fi), nil +} + +func openAdditonal(ctx context.Context, dbi *model.DBInfo) error { + for _, h := range dbi.Hosts { + for _, db := range h.Databases { + err := openAdditional(ctx, h.User, h.Password, h.Address, db.Name) + if err != nil { + return err + } + } + } + return nil +} + +func exeExplains( + ctx context.Context, queries []string, option *model.ExplainOption, +) ([]*model.ExplainInfo, error) { + + list := []*model.ExplainInfo{} + + queryMap := map[string]*model.SQLInfo{} + + for _, q := range queries { + // SQL Parse + info, err := getSQLInfo(ctx, q) + if err != nil { + if option.NoError { + if ErrCode(err) == int(SQLParseError) { + continue + } + } + return nil, err + } + if info.Table == "" { + continue + } + + // uniqフラグ指定の場合、重複SQLの除外 + if _, ok := queryMap[info.PrepareSQL]; ok && option.Uniq { + continue + } + queryMap[info.PrepareSQL] = info + + if option.UseTableMap { + for _, db := range option.TableMap[info.Table] { + // Explain実行 + expInfo, err := exeExplain(ctx, db, q, info.PrepareSQL) + if err != nil { + if option.NoError { + if ErrCode(err) == int(ExeExplainError) { + continue + } + } + return nil, err + } + + list = append(list, expInfo) + } + } else { + // Explain実行 + db := option.DB + expInfo, err := exeExplain(ctx, db, q, info.PrepareSQL) + if err != nil { + if option.NoError { + if ErrCode(err) == int(ExeExplainError) { + continue + } + } + return nil, err + } + + list = append(list, expInfo) + } + + } + + return list, nil +} + +func exeExplain(ctx context.Context, db, sql, prepareSQL string) (*model.ExplainInfo, error) { + exps, err := explain(ctx, db, sql) + if err != nil { + return nil, err + } + + return &model.ExplainInfo{ + DataBase: db, + PrepareSQL: prepareSQL, + SQL: sql, + Values: exps, + }, nil +} + +func filterResults(infos []*model.ExplainInfo, fi *model.ExplainFilter) []*model.ExplainInfo { + + if fi == (&model.ExplainFilter{}) { + return infos + } + + list := make([]*model.ExplainInfo, 0, len(infos)) + + for _, info := range infos { + + add := true + for i, exp := range info.Values { + + if i == 0 || !add { + // SelectType + add = getAddFlagForFiltering(add, fi.SelectType, exp.SelectType, false, false) + + // Table + add = getAddFlagForFiltering(add, fi.Table, exp.Table, false, false) + + // Type + add = getAddFlagForFiltering(add, fi.Type, exp.Type, false, false) + + // Extra + add = getAddFlagForFiltering(add, fi.Extra, exp.Extra, false, true) + } + + // SelectTypeNot + add = getAddFlagForFiltering(add, fi.SelectTypeNot, exp.SelectType, true, false) + + // TableNot + add = getAddFlagForFiltering(add, fi.TableNot, exp.Table, true, false) + + // TypeNot + add = getAddFlagForFiltering(add, fi.TypeNot, exp.Type, true, false) + + // ExtraNot + add = getAddFlagForFiltering(add, fi.ExtraNot, exp.Extra, true, true) + } + + if add { + list = append(list, info) + } + } + + return list +} + +func getAddFlagForFiltering(add bool, list []string, target string, not, isExp bool) bool { + + if list == nil { + return add + } + + for _, val := range list { + if isTrueForFiltering(val, target, isExp) == not { + add = false + if not { + break + } + } else { + add = true + if !not { + break + } + } + } + return add +} + +func isTrueForFiltering(val, target string, isExp bool) bool { + if isExp { + return (strings.Index(strings.ToLower(val), strings.ToLower(target)) > -1) + } else { + return (val == target) + } +} diff --git a/service/loadlog.go b/service/loadlog.go new file mode 100644 index 0000000..d0b6771 --- /dev/null +++ b/service/loadlog.go @@ -0,0 +1,106 @@ +package service + +import ( + "bufio" + "bytes" + "context" + "os" + "os/exec" + "strings" +) + +type FormatType string + +const ( + FormatSimple FormatType = "simple" + FormatOfficial FormatType = "official" + FormatCommand FormatType = "command" +) + +func LoadQueriesFromLog(ctx context.Context, filePath string, format FormatType, cmd string) ([]string, error) { + queries := []string{} + + filePath, err := getPath(filePath) + if err != nil { + return queries, ErrWrap(err, UserInputError) + } + + f, err := os.Open(filePath) + if err != nil { + return queries, ErrWrap(err, UserInputError) + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + query, err := GetQueryByFormat(format, scanner.Text(), cmd) + if err != nil { + return queries, ErrWrap(err, UserInputError) + } + + if query == "" { + continue + } + queries = append(queries, query) + } + if err := scanner.Err(); err != nil { + return queries, ErrWrap(err, UserInputError) + } + + return queries, nil +} + +func GetQueryByFormat(format FormatType, line, cmd string) (string, error) { + query := "" + + if format == FormatOfficial { + ws := strings.Split(line, "\t") + + if len(ws) == 3 { + if strings.Index(ws[1], "Execute") > -1 || strings.Index(ws[1], "Query") > -1 { + query = ws[2] + } + } + } else if format == FormatCommand { + + c := exec.Command("sh", "-c", cmd) + c.Stdin = strings.NewReader(line) + + var out bytes.Buffer + c.Stdout = &out + c.Start() + c.Wait() + + query = out.String() + + } else { + query = line + } + + return query, nil +} + +func LoadQueriesFromDB(ctx context.Context) ([]string, error) { + list := []string{} + + rows, err := query(officialDB, "select argument from general_log where command_type in ('Query', 'Execute')") + if err != nil { + return nil, ErrWrap(err, OtherError) + } + + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + return nil, ErrWrap(err, OtherError) + } + list = append(list, value) + } + + defer rows.Close() + + if err := rows.Err(); err != nil { + return list, ErrWrap(err, OtherError) + } + + return list, nil +} diff --git a/service/parsesql.go b/service/parsesql.go new file mode 100644 index 0000000..0260d6a --- /dev/null +++ b/service/parsesql.go @@ -0,0 +1,105 @@ +package service + +import ( + "context" + "expl/model" + "fmt" + + "strings" + + "github.com/xwb1989/sqlparser" + querypb "github.com/xwb1989/sqlparser/dependency/querypb" +) + +// getSQLInfo SQL関連情報を取得 +func getSQLInfo(ctx context.Context, query string) (*model.SQLInfo, error) { + // Parse + bv := map[string]*querypb.BindVariable{} + sqlStripped, comments := sqlparser.SplitMarginComments(query) + + stmt, err := sqlparser.Parse(sqlStripped) + if err != nil { + msg := fmt.Sprintf("error query line:%s", query) + return nil, ErrWrapWithMessage(err, SQLParseError, msg) + } + + // Normalize + prefix := "" + sqlparser.Normalize(stmt, bv, prefix) + + // query + preareQuery := comments.Leading + sqlparser.String(stmt) + comments.Trailing + + // table + table := "" + switch stmt.(type) { + case *sqlparser.Select, *sqlparser.Update, *sqlparser.Delete: + + table = getFirstTableName(stmt) + if table == "" { + table = getTableNameFromDBDot(query) + } + } + + return &model.SQLInfo{PrepareSQL: preareQuery, Table: table}, nil +} + +// getFirstTableName SQLから最初のTable名を取得 +func getFirstTableName(stmt sqlparser.Statement) string { + var expr sqlparser.SimpleTableExpr + + switch stmt.(type) { + case *sqlparser.Select: + expr = stmt.(*sqlparser.Select).From[0].(*sqlparser.AliasedTableExpr).Expr + + case *sqlparser.Update: + expr = stmt.(*sqlparser.Update).TableExprs[0].(*sqlparser.AliasedTableExpr).Expr + + case *sqlparser.Delete: + expr = stmt.(*sqlparser.Delete).TableExprs[0].(*sqlparser.AliasedTableExpr).Expr + } + + switch expr.(type) { + case sqlparser.TableName: + out := sqlparser.GetTableName(expr) + if out.String() != "" { + return out.String() + } + + case *sqlparser.Subquery: + return getFirstTableName(expr.(*sqlparser.Subquery).Select) + } + + return "" +} + +// getTableNameFromDBDot DB.Table名の形式の場合、SQLからTable名を取得 +func getTableNameFromDBDot(query string) string { + tokens := strings.Fields(query) + index := 0 + for i, v := range tokens { + if strings.ToLower(v) == "from" { + index = i + break + } + + if strings.ToLower(v) == "update" { + index = i + break + } + } + if len(tokens) <= index+1 { + return "" + } + db_tb := tokens[index+1] + + ind := strings.Index(db_tb, ".") + if ind == -1 { + return "" + } + + ind = ind + 1 + table := db_tb[ind:] + + return table +} diff --git a/view/render.go b/view/render.go new file mode 100644 index 0000000..738f90c --- /dev/null +++ b/view/render.go @@ -0,0 +1,124 @@ +package view + +import ( + "expl/model" + "fmt" + "os" + "strconv" + "strings" + + "github.com/olekukonko/tablewriter" +) + +var tableHeader []string = []string{ + "ID", + "SelectType", + "Table", + "Partitions", + "Type", + "PossibleKeys", + "Key", + "KeyLen", + "Ref", + "Rows", + "Filtered", + "Extra", +} + +var optionHeader []string = []string{ + "inpnut option", + "value", +} + +// RenderExplains render expl results +func RenderExplains(infos []*model.ExplainInfo, isParseEnable bool) { + + for _, info := range infos { + + headerDatas := make([][]string, 0, 3) + headerDatas = append(headerDatas, []string{"DataBase:", info.DataBase}) + if isParseEnable { + headerDatas = append(headerDatas, []string{"ParsedSQL:", info.PrepareSQL}) + } + headerDatas = append(headerDatas, []string{"SQL:", info.SQL}) + + headerTable := tablewriter.NewWriter(os.Stdout) + headerTable.SetBorder(false) + headerTable.SetColumnSeparator("") + headerTable.SetAutoWrapText(false) + headerTable.SetAlignment(tablewriter.ALIGN_LEFT) + headerTable.AppendBulk(headerDatas) + headerTable.Render() + + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader(tableHeader) + + for _, v := range info.Values { + table.Append(getRecord(v)) + } + + table.Render() + + fmt.Print("\n") + } +} + +func getRecord(e *model.Explain) []string { + rec := []string{ + strconv.FormatInt(e.ID, 10), + e.SelectType, + e.Table, + strconv.FormatInt(e.Partitions, 10), + e.Type, + e.PossibleKeys, + e.Key, + strconv.FormatInt(e.KeyLen, 10), + e.Ref, + strconv.FormatInt(e.Rows, 10), + strconv.FormatFloat(e.Filtered, 'f', 4, 64), + e.Extra, + } + + return rec +} + +// RenderOptions render expl options +func RenderOptions( + expOpt *model.ExplainOption, + fiOpt *model.ExplainFilter, + logPath, format, formatCmd string, +) { + + optionDatas := [][]string{ + []string{"database", expOpt.DB}, + []string{"host", expOpt.DBHost}, + []string{"user", expOpt.DBUser}, + []string{"pass", expOpt.DBPass}, + []string{"conf", expOpt.Config}, + []string{"log", logPath}, + []string{"format", format}, + []string{"format-cmd", formatCmd}, + []string{"filter-select-type", strings.Join(fiOpt.SelectType, ",")}, + []string{"filter-no-select-type", strings.Join(fiOpt.SelectTypeNot, ",")}, + []string{"filter-table", strings.Join(fiOpt.Table, ",")}, + []string{"filter-no-table", strings.Join(fiOpt.TableNot, ",")}, + []string{"filter-type", strings.Join(fiOpt.Type, ",")}, + []string{"filter-no-type", strings.Join(fiOpt.TypeNot, ",")}, + []string{"filter-extra", strings.Join(fiOpt.Extra, ",")}, + []string{"filter-no-extra", strings.Join(fiOpt.ExtraNot, ",")}, + []string{"update-table-map", fmt.Sprintf("%v", expOpt.UseTableMap)}, + []string{"ignore-error", fmt.Sprintf("%v", expOpt.NoError)}, + []string{"combine-sql", fmt.Sprintf("%v", expOpt.Uniq)}, + } + + optionTable := tablewriter.NewWriter(os.Stdout) + optionTable.SetBorder(false) + optionTable.SetColumnSeparator("") + optionTable.SetAutoWrapText(false) + optionTable.SetAlignment(tablewriter.ALIGN_LEFT) + optionTable.SetHeader(optionHeader) + optionTable.AppendBulk(optionDatas) + optionTable.Render() + + fmt.Println("") +}