diff --git a/docker-compose.yaml b/docker-compose.yaml index 31c169e..cd75f59 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,10 +17,35 @@ services: - /var/tmp - /var/log - /var/cache + + mysql57: + image: mysql:latest + restart: always + environment: + MYSQL_DATABASE: last9 + MYSQL_USER: mysql + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password + ports: + # : < MySQL Port running inside container> + - 8306:3306 + expose: + # Opens port 3306 on the container + - '3306' + # Where our data will be persisted + volumes: + - my-db:/var/lib/mysql - wait_postgres: + wait_db: image: "waisbrot/wait" depends_on: - pg13 + - mysql57 environment: TARGETS: pg13:8432 + TARGETS: mysql57:3306 + +# Names our volume +volumes: + my-db: + driver: local \ No newline at end of file diff --git a/go/go.mod b/go/go.mod index 928fe07..e2cecab 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,16 +3,20 @@ module github.com/last9/last9-cdk/go go 1.17 require ( + github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba github.com/go-chi/chi/v5 v5.0.7 + github.com/go-sql-driver/mysql v1.6.0 github.com/gorilla/mux v1.8.0 github.com/last9/last9-cdk/go/proc v0.0.0-20211209093125-ceff0e8ad651 github.com/last9/last9-cdk/go/tests v0.0.0-20211209093818-d351efae43f0 github.com/last9/pat v0.0.0-20211111093525-daacb495b5a9 github.com/lib/pq v1.10.4 + github.com/pganalyze/pg_query_go v1.0.3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/client_model v0.2.0 github.com/shogo82148/go-sql-proxy v0.6.1 + github.com/tidwall/gjson v1.14.1 github.com/xo/dburl v0.9.0 gopkg.in/go-playground/assert.v1 v1.2.1 gotest.tools v2.2.0+incompatible @@ -23,9 +27,13 @@ require ( github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/golang/protobuf v1.4.3 // indirect github.com/google/go-cmp v0.5.5 // indirect + github.com/juju/errors v0.0.0-20170703010042-c7d06af17c68 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.6.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 // indirect + golang.org/x/text v0.3.6 // indirect google.golang.org/protobuf v1.26.0-rc.1 // indirect ) diff --git a/go/go.sum b/go/go.sum index da441d9..afe48bb 100644 --- a/go/go.sum +++ b/go/go.sum @@ -42,6 +42,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba h1:hBK2BWzm0OzYZrZy9yzvZZw59C5Do4/miZ8FhEwd5P8= +github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba/go.mod h1:FGQp+RNQwVmLzDq6HBrYCww9qJQyNwH9Qji/quTQII4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -67,6 +69,8 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -131,6 +135,12 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/juju/errors v0.0.0-20170703010042-c7d06af17c68 h1:d2hBkTvi7B89+OXY8+bBBshPlc+7JYacGrG/dFak8SQ= +github.com/juju/errors v0.0.0-20170703010042-c7d06af17c68/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI= +github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b h1:Rrp0ByJXEjhREMPGTt3aWYjoIsUGCbt21ekbeJcTWv0= +github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -138,7 +148,11 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/last9/last9-cdk/go/proc v0.0.0-20211209093125-ceff0e8ad651 h1:/obyeV4t14bYiOQOd7fWOsFWjOd1zHYJTrSBmkwtdms= github.com/last9/last9-cdk/go/proc v0.0.0-20211209093125-ceff0e8ad651/go.mod h1:v6CMOSapiBY2xUeeFNsQ0sc2CuxMwlh6MFr7OVyGlpg= @@ -156,6 +170,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pganalyze/pg_query_go v1.0.3 h1:cur7WhCeA63mUD3Y/hZCl4QbU8NudQr1tIZV/ctsXCQ= +github.com/pganalyze/pg_query_go v1.0.3/go.mod h1:tR53lU3ddnExxb0XeLyYuQIK3dkR03FjQ9sj8AV/up8= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -192,6 +208,12 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= +github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/xo/dburl v0.9.0 h1:ME8QfRqZz/YDwf+VVEe9sq4wgEZCAOdYcUTeuAf+wQQ= github.com/xo/dburl v0.9.0/go.mod h1:7Uupe87dIDxNrbKFRrpw6bAf2l3/rqU42iwlpq1nyjY= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -320,10 +342,12 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7q golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.0.0-20180302201248-b7ef84aaf62a/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -451,15 +475,20 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/go/httpmetrics/metrics_gochi_test.go b/go/httpmetrics/metrics_gochi_test.go index 72b4806..55b1ee3 100644 --- a/go/httpmetrics/metrics_gochi_test.go +++ b/go/httpmetrics/metrics_gochi_test.go @@ -40,7 +40,6 @@ func TestGoChiMux(t *testing.T) { assert.Equal(t, len(ids) > 0, true) rms := o["http_requests_duration_milliseconds"] - // fmt.Println(rms.GetMetric()) assert.Equal(t, 1, len(rms.GetMetric())) assert.Equal(t, 7, assertLabels("/api/{id}", getDomain(srv), rms)) diff --git a/go/sqlmetrics/README.md b/go/sqlmetrics/README.md new file mode 100644 index 0000000..19f98f4 --- /dev/null +++ b/go/sqlmetrics/README.md @@ -0,0 +1,6 @@ +## Sqlmetrics Unit tests +```bash +export LAST9_PQSQL_DSN="postgres://testuser:testpassword@localhost:5432/last9" +export LAST9_MYSQL_DSN="mysql:password@tcp(localhost:8306)/last9" +go test +``` \ No newline at end of file diff --git a/go/sqlmetrics/connection.go b/go/sqlmetrics/connection.go index aaca692..10f8616 100644 --- a/go/sqlmetrics/connection.go +++ b/go/sqlmetrics/connection.go @@ -32,8 +32,8 @@ var ( connMap sync.Map ) -func storeConnInfo(conn *proxy.Conn, dsn string) error { - info, err := parseDSN(dsn) +func storeConnInfo(conn *proxy.Conn, driverName, dsn string) error { + info, err := parseDSN(driverName, dsn) if err != nil { return errors.Wrap(err, "parse dsn") } @@ -51,8 +51,29 @@ func loadConnInfo(conn *proxy.Conn) *connInfo { return info.(*connInfo) } -func parseDSN(dsn string) (*connInfo, error) { - u, err := dburl.Parse(dsn) +// Note: the DB Url library expects a scheme in the DSN for +// all drivers, while the standard library `database/sql` expects some +// DSNs to be passed without a scheme (e.g. mysql) and some with +// a scheme (e.g. postgres). +// This function hence expects you to pass a driver name along with the DSN for +// successfully parsing the DSN. +func parseDSN(driver, dsn string) (*connInfo, error) { + + var SchemefullDSN string + + switch strings.Split(driver, ":")[0] { + case "postgres": + { + SchemefullDSN = dsn + } + case "mysql": + { + //dsn = scheme + username + ":" + password + "@" + host + ":" + port + "/" + dbname + SchemefullDSN = strings.Split(driver, ":")[0] + "://" + strings.Split(dsn, "@")[0] + "@" + strings.Split(strings.Split(dsn, "(")[1], ")")[0] + strings.Split(strings.Split(dsn, "(")[1], ")")[1] + } + } + + u, err := dburl.Parse(SchemefullDSN) if err != nil { return nil, errors.Wrap(err, "invalid dsn") } diff --git a/go/sqlmetrics/dbstats.go b/go/sqlmetrics/dbstats.go index e3d92af..66d033a 100644 --- a/go/sqlmetrics/dbstats.go +++ b/go/sqlmetrics/dbstats.go @@ -97,8 +97,8 @@ func emitStats(s sql.DBStats, labels LabelSet) { } // make Labels to be used for DB Stats -func makeStatsLabels(dsn string) (LabelSet, error) { - info, err := parseDSN(dsn) +func makeStatsLabels(driver, dsn string) (LabelSet, error) { + info, err := parseDSN(driver, dsn) if err != nil { return nil, errors.Wrap(err, "parse dsn emit db") } @@ -110,6 +110,7 @@ func makeStatsLabels(dsn string) (LabelSet, error) { labels[proc.LabelProgram] = proc.GetProgamName() labels[proc.LabelHostname] = proc.GetHostname() + return labels.Merge(info.LabelSet()), nil } @@ -117,8 +118,8 @@ func makeStatsLabels(dsn string) (LabelSet, error) { // to emit the gauges and counters corresponding the connections spawned // by this binary. It's fairly light weight with minimal allocation so // performance should not really be a concern here. -func EmitDBStats(db *sql.DB, dsn string) error { - l, err := makeStatsLabels(dsn) +func EmitDBStats(db *sql.DB, driver, dsn string) error { + l, err := makeStatsLabels(driver, dsn) if err != nil { return errors.Wrap(err, "stats labels") } diff --git a/go/sqlmetrics/label_maker.go b/go/sqlmetrics/label_maker.go index cf68b08..97c1ded 100644 --- a/go/sqlmetrics/label_maker.go +++ b/go/sqlmetrics/label_maker.go @@ -1,5 +1,15 @@ package sqlmetrics +import ( + "encoding/json" + "log" + "strings" + + "github.com/blastrain/vitess-sqlparser/sqlparser" + pg_query "github.com/pganalyze/pg_query_go" + "github.com/tidwall/gjson" +) + type LabelSet map[string]string // Merge accepts a set of LabelSets and merge them into this LabelSet. @@ -22,7 +32,7 @@ func (l LabelSet) ToMap() map[string]string { // A type that user can extend to Parse a query and extract less verbose // or more relevant labels out of it. -type LabelMaker func(string) LabelSet +type LabelMaker func(string, string) LabelSet const idealLabelLen = 20 @@ -31,12 +41,14 @@ const idealLabelLen = 20 // the metric server to its knees if left untapped. If this behaviour is not // desired, a user can anwyay implement their own QToLabelSet and emit the // raw query as-it-is. -func defaultLabelMaker(q string) LabelSet { +func defaultLabelMaker(driver, q string) LabelSet { + statementName := getStatementName(driver, q) + if len(q) > idealLabelLen { q = q[:idealLabelLen] + "..." } - return LabelSet{"per": q} + return LabelSet{"per": q, "statement": statementName} } // queryStatus is an enumeration. @@ -52,3 +64,48 @@ const ( func (q queryStatus) String() string { return [...]string{"success", "failure"}[q] } + +// Accepts a driver name and query and returns the statement name of the first level query. +func getStatementName(driver, q string) string { + + switch strings.Split(driver, ":")[0] { + case "postgres": + { + // Parse query and get query AST + tree, err := pg_query.ParseToJSON(q) + if err != nil { + log.Println("Error parsing query:", q, "Error occured :", err) + return "" + } + + // Get the statement name of the first level query + statement := gjson.Get(tree[1:len(tree)-1], "..0.RawStmt.stmt.@keys.0") + + return statement.String() + } + case "mysql": + { + // Parse query and get query AST + stmt, err := sqlparser.Parse(q) + if err != nil { + log.Println("Error parsing query:", q, "Error :", err) + return "" + } + + // convert AST to json string + stmtJson, err := json.Marshal(stmt) + if err != nil { + log.Println("Error in conversion to json :", q, "Error :", err) + return "" + } + + // Parse the json and get the statement name of the first level query + if gjson.Get(string(stmtJson), "SelectExprs").String() != "" { + return "select" + } + return gjson.Get(string(stmtJson), "Action").String() + } + } + + return "" +} diff --git a/go/sqlmetrics/metrics.go b/go/sqlmetrics/metrics.go index 786ab9e..ebd6e68 100644 --- a/go/sqlmetrics/metrics.go +++ b/go/sqlmetrics/metrics.go @@ -11,7 +11,7 @@ var ( subsystem = "sql" defaultLabels = []string{ "per", proc.LabelHostname, "table", "dbname", "dbhost", "status", - proc.LabelProgram, proc.LabelTenant, proc.LabelCluster, + proc.LabelProgram, proc.LabelTenant, proc.LabelCluster, "statement", } sqlQueryDuration = prometheus.NewHistogramVec( diff --git a/go/sqlmetrics/mysql_test.go b/go/sqlmetrics/mysql_test.go new file mode 100644 index 0000000..73c920a --- /dev/null +++ b/go/sqlmetrics/mysql_test.go @@ -0,0 +1,233 @@ +package sqlmetrics + +import ( + "context" + "database/sql" + "log" + "net/http" + "os" + "testing" + + _ "github.com/go-sql-driver/mysql" + "github.com/last9/last9-cdk/go/proc" + "github.com/last9/last9-cdk/go/tests" + "github.com/prometheus/client_golang/prometheus/promhttp" + dto "github.com/prometheus/client_model/go" + "gotest.tools/assert" +) + +func getMySqlDSN() string { + dsn := os.Getenv("LAST9_MYSQL_DSN") + if dsn == "" { + panic("LAST9_MYSQL_DSN is not set") + } + + return dsn +} + +func getMySqlDB(driverName string) (*sql.DB, error) { + dsn := getMySqlDSN() + // NOTE: This is the Second change that you make. + // whatever the regsitered driver was, use :last9 to connect + // instead of . And that's it. + return sql.Open(driverName+":last9", dsn) +} + +func TestMySql(t *testing.T) { + log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + srv := tests.MakeServer(mux) + defer srv.Close() + + driverName := "mysql" + t.Run("register cannot return error", func(t *testing.T) { + // NOTE: This is the first change that you do. + // Declare the driver that you would want to use. + name, err := RegisterDriver(Options{Driver: driverName}) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "mysql:last9", name) + }) + + t.Run("connect to db", func(t *testing.T) { + dsn := os.Getenv("LAST9_MYSQL_DSN") + if dsn == "" { + t.Fatal("LAST9_MYSQL_DSN is not set") + } + + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + rows, err := db.Query("SELECT 'Hello'") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, err, nil) + + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatal(err) + } + assert.Equal(t, "Hello", name) + } + + assert.Equal(t, nil, rows.Close()) + }) + + t.Run("connect via proxy", func(t *testing.T) { + db, err := getMySqlDB(driverName) + + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + rows, err := db.Query("SELECT 'Hello'") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, err, nil) + + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatal(err) + } + assert.Equal(t, "Hello", name) + } + + assert.Equal(t, nil, rows.Close()) + }) + + t.Run("create and execute", func(t *testing.T) { + resetMetrics() + + ctx := context.Background() + db, err := getMySqlDB(driverName) + if err != nil { + t.Fatal(err) + } + + defer db.Close() + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + + if _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS pets (name VARCHAR(20) UNIQUE, owner VARCHAR(20), + species VARCHAR(20), sex CHAR(1), birth DATE, death DATE); + `); err != nil { + t.Fatal(err) + } + + // Here, the query is executed on the transaction instance, and + // not applied to the database yet + if _, err = tx.ExecContext(ctx, `INSERT IGNORE INTO pets (name, species) + VALUES ('Fido', 'dog'), ('Albert', 'cat'); + `); err != nil { + // Incase we find any error in the query execution, rollback + tx.Rollback() + return + } + + var catCount int + // Run a query to get a count of all cats + row := tx.QueryRow(`SELECT count(*) FROM pets WHERE species='cat';`) + // Store the count in the `catCount` variable + if err = row.Scan(&catCount); err != nil { + tx.Rollback() + t.Fatal(err) + return + } + + assert.Equal(t, 1, catCount) + + // Finally, if no errors are recieved from the queries, + // commit the transaction this applies the above changes to our database + if err = tx.Commit(); err != nil { + tx.Rollback() + t.Fatal(err) + } + + o, err := tests.GetMetrics(srv.URL) + if err != nil { + t.Fatal(err) + } + + req := o[expectedMetric] + + assert.Equal(t, req.GetType(), dto.MetricType_HISTOGRAM) + for _, m := range req.GetMetric() { + h := m.GetHistogram() + // log.Println("Labels : ", m.GetLabel()) + + assert.Equal(t, 1, int(h.GetSampleCount())) + assert.Equal(t, true, labelSetContains( + m.GetLabel(), map[string]string{ + "cluster": "", + "tenant": "", + "dbname": "last9", + "hostname": proc.GetHostname(), + "program": proc.GetProgamName(), + "status": "success", + "table": "", + }), + ) + } + }) + + t.Run("conn metrics", func(t *testing.T) { + dsn := getMySqlDSN() + labels, err := makeStatsLabels(driverName, dsn) + if err != nil { + t.Fatal(err) + } + + db, err := getMySqlDB(driverName) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + emitStats(db.Stats(), labels) + + o, err := tests.GetMetrics(srv.URL) + if err != nil { + t.Fatal(err) + } + + idle, ok := o["last9_sql_connections_idle"] + assert.Equal(t, true, ok) + assert.Equal(t, idle.GetType(), dto.MetricType_GAUGE) + + used, ok := o["last9_sql_connections_in_use"] + assert.Equal(t, true, ok) + assert.Equal(t, used.GetType(), dto.MetricType_GAUGE) + + max, ok := o["last9_sql_connections_max_open"] + assert.Equal(t, true, ok) + assert.Equal(t, max.GetType(), dto.MetricType_GAUGE) + + wait_duration, ok := o["last9_sql_connections_wait_duration_total"] + assert.Equal(t, true, ok) + assert.Equal(t, wait_duration.GetType(), dto.MetricType_COUNTER) + + wait_total, ok := o["last9_sql_connections_wait_total"] + assert.Equal(t, true, ok) + assert.Equal(t, wait_total.GetType(), dto.MetricType_COUNTER) + }) + +} diff --git a/go/sqlmetrics/pq_test.go b/go/sqlmetrics/pq_test.go index 1a3d86b..1be7ac1 100644 --- a/go/sqlmetrics/pq_test.go +++ b/go/sqlmetrics/pq_test.go @@ -18,9 +18,9 @@ import ( ) func getDSN() string { - dsn := os.Getenv("LAST9_SQL_DSN") + dsn := os.Getenv("LAST9_PQSQL_DSN") if dsn == "" { - panic("LAST9_SQL_DSN is not set") + panic("LAST9_PQSQL_DSN is not set") } return dsn @@ -52,21 +52,23 @@ func TestPq(t *testing.T) { srv := tests.MakeServer(mux) defer srv.Close() + driverName := "postgres" + t.Run("register cannot return error", func(t *testing.T) { // NOTE: This is the first change that you do. // Declare the driver that you would want to use. - name, err := RegisterDriver(Options{Driver: "postgres"}) + name, err := RegisterDriver(Options{Driver: driverName}) if err != nil { t.Fatal(err) } - assert.Equal(t, "postgres:last9", name) + assert.Equal(t, driverName+":last9", name) }) t.Run("connect to db", func(t *testing.T) { - dsn := os.Getenv("LAST9_SQL_DSN") + dsn := os.Getenv("LAST9_PQSQL_DSN") if dsn == "" { - t.Fatal("LAST9_SQL_DSN is not set") + t.Fatal("LAST9_PQSQL_DSN is not set") } db, err := sql.Open("postgres", dsn) @@ -122,7 +124,7 @@ func TestPq(t *testing.T) { t.Run("conn metrics", func(t *testing.T) { dsn := getDSN() - labels, err := makeStatsLabels(dsn) + labels, err := makeStatsLabels(driverName, dsn) if err != nil { t.Fatal(err) } diff --git a/go/sqlmetrics/sql_intercept.go b/go/sqlmetrics/sql_intercept.go index 6a42528..e8eeb86 100644 --- a/go/sqlmetrics/sql_intercept.go +++ b/go/sqlmetrics/sql_intercept.go @@ -123,7 +123,7 @@ func RegisterDriverWithLabelMaker(d Options, fn LabelMaker) (string, error) { c context.Context, ctx interface{}, conn *proxy.Conn, err error, ) error { dsn := ctx.(string) - storeConnInfo(conn, dsn) + storeConnInfo(conn, name, dsn) return nil }, @@ -145,7 +145,7 @@ func RegisterDriverWithLabelMaker(d Options, fn LabelMaker) (string, error) { dc := ctx.(*dbCtx) if err := emitDuration( - fn(dc.query).Merge( + fn(name, dc.query).Merge( info.LabelSet()), getQueryStatus(err), dc.start, ); err != nil { log.Printf("%+v", err) @@ -171,7 +171,7 @@ func RegisterDriverWithLabelMaker(d Options, fn LabelMaker) (string, error) { dc := ctx.(*dbCtx) if err := emitDuration( - fn(dc.query).Merge( + fn(name, dc.query).Merge( info.LabelSet()), getQueryStatus(err), dc.start, ); err != nil { log.Printf("%+v", err)