From e1135acb3b277d6c04e30c0886cb60a14f2173d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 22 Mar 2024 10:40:45 +0100 Subject: [PATCH 01/13] PMM-12551 Clickhouse v2. --- go.mod | 9 ++------- go.sum | 18 ++++-------------- qan-api2/db.go | 2 +- qan-api2/db_test.go | 2 +- qan-api2/services/analytics/filters_test.go | 2 +- 5 files changed, 9 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 98934ea3c8..131db7d002 100644 --- a/go.mod +++ b/go.mod @@ -13,12 +13,9 @@ replace gopkg.in/alecthomas/kingpin.v2 => github.com/Percona-Lab/kingpin v2.2.6- replace golang.org/x/crypto => github.com/percona-lab/crypto v0.0.0-20231108144114-756dfb24eaf2 -replace github.com/ClickHouse/clickhouse-go/151 => github.com/ClickHouse/clickhouse-go v1.5.1 // clickhouse-go/v2 cannot work with 1.5.1 which we need for QAN-API - require ( github.com/AlekSi/pointer v1.2.0 - github.com/ClickHouse/clickhouse-go/151 v0.0.0-00010101000000-000000000000 - github.com/ClickHouse/clickhouse-go/v2 v2.21.1 + github.com/ClickHouse/clickhouse-go/v2 v2.22.2 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/alecthomas/kong v0.9.0 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 @@ -78,7 +75,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 google.golang.org/grpc v1.62.0 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/reform.v1 v1.5.1 gopkg.in/yaml.v3 v3.0.1 @@ -119,7 +116,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/ClickHouse/ch-go v0.61.5 // indirect - github.com/ClickHouse/clickhouse-go v1.5.4 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 github.com/Microsoft/go-winio v0.6.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect @@ -128,7 +124,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charmbracelet/harmonica v0.2.0 // indirect - github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index b04e515a4b..5b7e171d63 100644 --- a/go.sum +++ b/go.sum @@ -50,12 +50,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= -github.com/ClickHouse/clickhouse-go v1.5.1 h1:I8zVFZTz80crCs0FFEBJooIxsPcV0xfthzK1YrkpJTc= -github.com/ClickHouse/clickhouse-go v1.5.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/ClickHouse/clickhouse-go/v2 v2.21.1 h1:x8wZEMOHDh4K8kLQBtGMeIIguejiaj8/bUiF2VzG6n4= -github.com/ClickHouse/clickhouse-go/v2 v2.21.1/go.mod h1:hTWNkV9mkQwiQ/df0rbN17VXF05UTResY4krnjbzVZA= +github.com/ClickHouse/clickhouse-go/v2 v2.22.2 h1:T1BljsIjj+3aQog80jKMTeF4EqAUG4P6TVcCvmakYAc= +github.com/ClickHouse/clickhouse-go/v2 v2.22.2/go.mod h1:tBhdF3f3RdP7sS59+oBAtTyhWpy0024ZxDMhgxra0QE= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -102,8 +100,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= -github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8= @@ -212,7 +208,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= @@ -347,7 +342,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -392,7 +386,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -416,7 +409,6 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= @@ -489,8 +481,6 @@ github.com/percona/promconfig v0.2.5 h1:f/HN/CbECQs7d9RIB6MKVkuXstsrsqEDxRvf6yig github.com/percona/promconfig v0.2.5/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg= github.com/pganalyze/pg_query_go/v2 v2.2.0 h1:OW+reH+ZY7jdEuPyuLGlf1m7dLbE+fDudKXhLs0Ttpk= github.com/pganalyze/pg_query_go/v2 v2.2.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -962,8 +952,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/qan-api2/db.go b/qan-api2/db.go index 060e137f70..44f55d76a7 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -22,7 +22,7 @@ import ( "net/url" "strings" - clickhouse "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver + clickhouse "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver "github.com/golang-migrate/migrate/v4" _ "github.com/golang-migrate/migrate/v4/database/clickhouse" // register golang-migrate driver "github.com/golang-migrate/migrate/v4/source/iofs" diff --git a/qan-api2/db_test.go b/qan-api2/db_test.go index 841803965e..e77a325963 100644 --- a/qan-api2/db_test.go +++ b/qan-api2/db_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - _ "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver + _ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver _ "github.com/golang-migrate/migrate/v4/database/clickhouse" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" diff --git a/qan-api2/services/analytics/filters_test.go b/qan-api2/services/analytics/filters_test.go index 31b4fb81f4..0c866b082c 100644 --- a/qan-api2/services/analytics/filters_test.go +++ b/qan-api2/services/analytics/filters_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - _ "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver + _ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver // TODO replace with 'google.golang.org/protobuf/encoding/protojson' since this one is deprecated. "github.com/golang/protobuf/ptypes/timestamp" "github.com/jmoiron/sqlx" From 77e9a3f7c6f314c074f1023111f4ba816235a536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 21 Mar 2024 14:35:18 +0100 Subject: [PATCH 02/13] PMM-12551 Fix for new connection string format. --- managed/services/telemetry/telemetry_test.go | 2 +- qan-api2/db_test.go | 8 ++++---- qan-api2/main.go | 2 +- qan-api2/services/analytics/filters_test.go | 2 +- qan-api2/services/analytics/profile_test.go | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/managed/services/telemetry/telemetry_test.go b/managed/services/telemetry/telemetry_test.go index 62bbcec44e..4de3244245 100644 --- a/managed/services/telemetry/telemetry_test.go +++ b/managed/services/telemetry/telemetry_test.go @@ -50,7 +50,7 @@ func TestRunTelemetryService(t *testing.T) { if ok { pgHostPort = pgHostPortFromEnv } - qanDSN := "tcp://localhost:9000?database=pmm" + qanDSN := "tcp://localhost:9000/pmm" qanDSNFromEnv, ok := os.LookupEnv(envQanDSN) if ok { qanDSN = qanDSNFromEnv diff --git a/qan-api2/db_test.go b/qan-api2/db_test.go index e77a325963..7adcaa506e 100644 --- a/qan-api2/db_test.go +++ b/qan-api2/db_test.go @@ -39,9 +39,9 @@ func setup() *sqlx.DB { } dsn, ok := os.LookupEnv("QANAPI_DSN_TEST") - dsn = strings.Replace(dsn, "?database=pmm_test", "?database=pmm_test_parts", 1) + dsn = strings.Replace(dsn, "/pmm_test", "/pmm_test_parts", 1) if !ok { - dsn = "clickhouse://127.0.0.1:19000?database=pmm_test_parts" + dsn = "clickhouse://127.0.0.1:19000/pmm_test_parts" } db, err := sqlx.Connect("clickhouse", dsn) if err != nil { @@ -113,9 +113,9 @@ func TestCreateDbIfNotExists(t *testing.T) { t.Run("connect to db that doesnt exist", func(t *testing.T) { dsn, ok := os.LookupEnv("QANAPI_DSN_TEST") - dsn = strings.Replace(dsn, "?database=pmm_test", "?database=pmm_created_db", 1) + dsn = strings.Replace(dsn, "/pmm_test", "/pmm_created_db", 1) if !ok { - dsn = "clickhouse://127.0.0.1:19000?database=pmm_created_db" + dsn = "clickhouse://127.0.0.1:19000/pmm_created_db" } db := createDB(dsn) diff --git a/qan-api2/main.go b/qan-api2/main.go index df5457964d..24e1e9af4b 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -61,7 +61,7 @@ import ( const ( shutdownTimeout = 3 * time.Second - defaultDsnF = "clickhouse://%s?database=%s&block_size=%s&pool_size=%s" + defaultDsnF = "clickhouse://%s/%s?block_size=%s&pool_size=%s" maxIdleConns = 5 maxOpenConns = 10 ) diff --git a/qan-api2/services/analytics/filters_test.go b/qan-api2/services/analytics/filters_test.go index 0c866b082c..fbb714f182 100644 --- a/qan-api2/services/analytics/filters_test.go +++ b/qan-api2/services/analytics/filters_test.go @@ -54,7 +54,7 @@ type testValuesUnmarshal struct { func TestService_GetFilters(t *testing.T) { dsn, ok := os.LookupEnv("QANAPI_DSN_TEST") if !ok { - dsn = "clickhouse://127.0.0.1:19000?database=pmm_test" + dsn = "clickhouse://127.0.0.1:19000/pmm_test" } db, err := sqlx.Connect("clickhouse", dsn) if err != nil { diff --git a/qan-api2/services/analytics/profile_test.go b/qan-api2/services/analytics/profile_test.go index 95e22f1f81..3db4b7cb55 100644 --- a/qan-api2/services/analytics/profile_test.go +++ b/qan-api2/services/analytics/profile_test.go @@ -38,7 +38,7 @@ import ( func setup() *sqlx.DB { dsn, ok := os.LookupEnv("QANAPI_DSN_TEST") if !ok { - dsn = "clickhouse://127.0.0.1:19000?database=pmm_test" + dsn = "clickhouse://127.0.0.1:19000/pmm_test" } db, err := sqlx.Connect("clickhouse", dsn) if err != nil { From 7cd9ca9e4f1b0b04607869eda3bb0571b77e551c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 21 Mar 2024 15:11:19 +0100 Subject: [PATCH 03/13] PMM-12551 Another fix for new connection string. --- qan-api2/db.go | 7 ++----- qan-api2/db_test.go | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index 44f55d76a7..e97e3a90aa 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -78,11 +78,8 @@ func createDB(dsn string) error { if err != nil { return err } - q := clickhouseURL.Query() - databaseName := q.Get("database") - q.Set("database", "default") - - clickhouseURL.RawQuery = q.Encode() + databaseName := strings.Replace(clickhouseURL.Path, "/", "", 1) + clickhouseURL.Path = "/default" defaultDB, err := sqlx.Connect("clickhouse", clickhouseURL.String()) if err != nil { diff --git a/qan-api2/db_test.go b/qan-api2/db_test.go index 7adcaa506e..812e5f2321 100644 --- a/qan-api2/db_test.go +++ b/qan-api2/db_test.go @@ -120,6 +120,6 @@ func TestCreateDbIfNotExists(t *testing.T) { db := createDB(dsn) - require.Equal(t, db, nil, "Check connection after we create database") + require.Equal(t, nil, db, "Check connection after we create database") }) } From 92449b1fbf870531d2946108b0f3666e32f909be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Thu, 21 Mar 2024 15:46:04 +0100 Subject: [PATCH 04/13] PMM-10684 Remove block size, pool size (not supported anymore). --- managed/CONTRIBUTING.md | 2 -- managed/services/supervisord/supervisord.go | 6 ------ qan-api2/main.go | 8 ++------ 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/managed/CONTRIBUTING.md b/managed/CONTRIBUTING.md index ddbafd1018..38dffe0ed0 100644 --- a/managed/CONTRIBUTING.md +++ b/managed/CONTRIBUTING.md @@ -57,8 +57,6 @@ go test -timeout=30s -p 1 ./... | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- | | PERCONA_TEST_PMM_CLICKHOUSE_ADDR | Sets Clickhouse address | 127.0.0.1:9000 | | PERCONA_TEST_PMM_CLICKHOUSE_DATABASE | Sets Clickhouse database | pmm | -| PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE | Sets Clickhouse connections pool size | none | -| PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE | Sets Clickhouse block size | none | | PERCONA_TEST_STARLARK_ALLOW_RECURSION | Allows recursive functions in checks scripts | false | | PERCONA_TEST_NICER_API | Enables nicer API with default/zero values in response. | false | | PERCONA_TEST_VERSION_SERVICE_URL | Sets versions service URL | https://check.percona.com/versions/v1 | diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index 2d94414418..5406b5e61d 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -425,8 +425,6 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin clickhouseDatabase := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_DATABASE", defaultClickhouseDatabase) clickhouseAddr := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_ADDR", defaultClickhouseAddr) clickhouseDataSourceAddr := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE_ADDR", defaultClickhouseDataSourceAddr) - clickhousePoolSize := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE", "") - clickhouseBlockSize := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE", "") clickhouseAddrPair := strings.SplitN(clickhouseAddr, ":", 2) vmSearchDisableCache := getValueFromENV("VM_search_disableCache", strconv.FormatBool(!settings.VictoriaMetrics.CacheEnabled)) vmSearchMaxQueryLen := getValueFromENV("VM_search_maxQueryLen", defaultVMSearchMaxQueryLen) @@ -457,8 +455,6 @@ func (s *Service) marshalConfig(tmpl *template.Template, settings *models.Settin "ClickhouseAddr": clickhouseAddr, "ClickhouseDataSourceAddr": clickhouseDataSourceAddr, "ClickhouseDatabase": clickhouseDatabase, - "ClickhousePoolSize": clickhousePoolSize, - "ClickhouseBlockSize": clickhouseBlockSize, "ClickhouseHost": clickhouseAddrPair[0], "ClickhousePort": clickhouseAddrPair[1], } @@ -727,8 +723,6 @@ command = environment = PERCONA_TEST_PMM_CLICKHOUSE_ADDR="{{ .ClickhouseAddr }}", PERCONA_TEST_PMM_CLICKHOUSE_DATABASE="{{ .ClickhouseDatabase }}", -{{ if .ClickhousePoolSize }} PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE={{ .ClickhousePoolSize }},{{- end}} -{{ if .ClickhouseBlockSize }} PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE={{ .ClickhouseBlockSize }}{{- end}} user = pmm autorestart = true autostart = true diff --git a/qan-api2/main.go b/qan-api2/main.go index 24e1e9af4b..234148ca5f 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -61,7 +61,7 @@ import ( const ( shutdownTimeout = 3 * time.Second - defaultDsnF = "clickhouse://%s/%s?block_size=%s&pool_size=%s" + defaultDsnF = "clickhouse://%s/%s" maxIdleConns = 5 maxOpenConns = 10 ) @@ -264,10 +264,6 @@ func main() { dsnF := kingpin.Flag("dsn", "ClickHouse database DSN. Can be override with database/host/port options").Default(defaultDsnF).String() clickHouseDatabaseF := kingpin.Flag("clickhouse-name", "Clickhouse database name").Default("pmm").Envar("PERCONA_TEST_PMM_CLICKHOUSE_DATABASE").String() clickhouseAddrF := kingpin.Flag("clickhouse-addr", "Clickhouse database address").Default("127.0.0.1:9000").Envar("PERCONA_TEST_PMM_CLICKHOUSE_ADDR").String() - clickhouseBlockSizeF := kingpin.Flag("clickhouse-block-size", "Number of rows that can be load from table in one cycle"). - Default("10000").Envar("PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE").String() - clickhousePoolSizeF := kingpin.Flag("clickhouse-pool-size", "Controls how much queries can be run simultaneously"). - Default("2").Envar("PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE").String() debugF := kingpin.Flag("debug", "Enable debug logging").Bool() traceF := kingpin.Flag("trace", "Enable trace logging (implies debug)").Bool() @@ -295,7 +291,7 @@ func main() { var dsn string if *dsnF == defaultDsnF { - dsn = fmt.Sprintf(defaultDsnF, *clickhouseAddrF, *clickHouseDatabaseF, *clickhouseBlockSizeF, *clickhousePoolSizeF) + dsn = fmt.Sprintf(defaultDsnF, *clickhouseAddrF, *clickHouseDatabaseF) } else { dsn = *dsnF } From 6e644ead5f49d3c5d18c036ea6088164b305cb21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 22 Mar 2024 08:18:38 +0100 Subject: [PATCH 05/13] PMM-10684 Supervisor fix. --- managed/testdata/supervisord.d/qan-api2.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/managed/testdata/supervisord.d/qan-api2.ini b/managed/testdata/supervisord.d/qan-api2.ini index 94124e5afb..29bf342e6d 100644 --- a/managed/testdata/supervisord.d/qan-api2.ini +++ b/managed/testdata/supervisord.d/qan-api2.ini @@ -8,8 +8,6 @@ command = environment = PERCONA_TEST_PMM_CLICKHOUSE_ADDR="127.0.0.1:9000", PERCONA_TEST_PMM_CLICKHOUSE_DATABASE="pmm", - - user = pmm autorestart = true autostart = true From 05f03fa6b05847bc71cd8e6d46d0f0ec4c343f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Fri, 22 Mar 2024 10:47:24 +0100 Subject: [PATCH 06/13] PMM-10684 Small change. --- qan-api2/db_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qan-api2/db_test.go b/qan-api2/db_test.go index 812e5f2321..9683f4cf5a 100644 --- a/qan-api2/db_test.go +++ b/qan-api2/db_test.go @@ -120,6 +120,6 @@ func TestCreateDbIfNotExists(t *testing.T) { db := createDB(dsn) - require.Equal(t, nil, db, "Check connection after we create database") + require.Nil(t, db, "Check connection after we create database") }) } From d7ab08353cd3cb0bf309dad87e3288dad07f6947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 11 Jun 2024 13:05:46 +0200 Subject: [PATCH 07/13] PMM-10684 QAN fix after merge. --- go.mod | 2 +- qan-api2/services/analytics/filters_test.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 80b0c1692d..129362306c 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/gogo/status v1.1.1 github.com/golang-migrate/migrate/v4 v4.17.0 + github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/grafana/grafana-api-golang-client v0.27.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 @@ -94,7 +95,6 @@ require ( github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/mock v1.4.4 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.0.0 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect diff --git a/qan-api2/services/analytics/filters_test.go b/qan-api2/services/analytics/filters_test.go index 3e35685e3f..3d10f080a1 100644 --- a/qan-api2/services/analytics/filters_test.go +++ b/qan-api2/services/analytics/filters_test.go @@ -24,7 +24,6 @@ import ( "time" _ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver - "github.com/golang/protobuf/ptypes/timestamp" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/timestamppb" From 705bae9e1cd5e154e97c7b8829ec9b040eda36da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 11 Jun 2024 13:08:26 +0200 Subject: [PATCH 08/13] PMM-10684 Remove outdated env variables. --- docker-compose.yml | 8 -------- docs/process/v2_to_v3_environment_variables.md | 2 -- managed/utils/envvars/parser.go | 2 +- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7317df4e0f..b9f29f11f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,8 +33,6 @@ services: # - PMM_DEV_TELEMETRY_RETRY_BACKOFF=10s # - PMM_CLICKHOUSE_ADDR=127.0.0.1:9000 # - PMM_CLICKHOUSE_DATABASE=pmm - # - PMM_CLICKHOUSE_BLOCK_SIZE=10000 - # - PMM_CLICKHOUSE_POOL_SIZE=2 # - PMM_DEBUG=1 # - PMM_DEV_ADVISOR_CHECKS_FILE=/srv/checks/local-checks.yml # - PMM_POSTGRES_ADDR=pg @@ -232,8 +230,6 @@ services: # - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1 - PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000 - PMM_CLICKHOUSE_DATABASE=pmm - - PMM_CLICKHOUSE_BLOCK_SIZE=10000 - - PMM_CLICKHOUSE_POOL_SIZE=2 - PMM_DISABLE_BUILTIN_CLICKHOUSE=1 - PMM_POSTGRES_ADDR=pg:5432 - PMM_POSTGRES_USERNAME=pmm-managed @@ -315,8 +311,6 @@ services: # - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1 - PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000 - PMM_CLICKHOUSE_DATABASE=pmm - - PMM_CLICKHOUSE_BLOCK_SIZE=10000 - - PMM_CLICKHOUSE_POOL_SIZE=2 - PMM_DISABLE_BUILTIN_CLICKHOUSE=1 - PMM_POSTGRES_ADDR=pg:5432 - PMM_POSTGRES_USERNAME=pmm-managed @@ -397,8 +391,6 @@ services: # - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1 - PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000 - PMM_CLICKHOUSE_DATABASE=pmm - - PMM_CLICKHOUSE_BLOCK_SIZE=10000 - - PMM_CLICKHOUSE_POOL_SIZE=2 - PMM_DISABLE_BUILTIN_CLICKHOUSE=1 - PMM_POSTGRES_ADDR=pg:5432 - PMM_POSTGRES_USERNAME=pmm-managed diff --git a/docs/process/v2_to_v3_environment_variables.md b/docs/process/v2_to_v3_environment_variables.md index f059f978da..43edf20f45 100644 --- a/docs/process/v2_to_v3_environment_variables.md +++ b/docs/process/v2_to_v3_environment_variables.md @@ -26,11 +26,9 @@ Below is a list of affected variables and their new names. | `PERCONA_TEST_CHECKS_PUBLIC_KEY` | | Removed in PMM v3, use `PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY` | | `PERCONA_TEST_NICER_API` | `PMM_NICER_API` | | | `PERCONA_TEST_PMM_CLICKHOUSE_ADDR` | `PMM_CLICKHOUSE_ADDR` | | -| `PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE` | `PMM_CLICKHOUSE_BLOCK_SIZE` | | | `PERCONA_TEST_PMM_CLICKHOUSE_DATABASE` | `PMM_CLICKHOUSE_DATABASE` | | | `PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE` | `PMM_CLICKHOUSE_DATASOURCE` | | | `PERCONA_TEST_PMM_CLICKHOUSE_HOST` | `PMM_CLICKHOUSE_HOST` | | -| `PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE` | `PMM_CLICKHOUSE_POOL_SIZE` | | | `PERCONA_TEST_PMM_CLICKHOUSE_PORT` | `PMM_CLICKHOUSE_PORT` | | | `PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE` | `PMM_DISABLE_BUILTIN_CLICKHOUSE` | | | `PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES` | `PMM_DISABLE_BUILTIN_POSTGRES` | | diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index 35adf8fa2c..7868117eed 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -98,7 +98,7 @@ func ParseEnvVars(envs []string) (*models.ChangeSettingsParams, []error, []strin case "PMM_DEV_VERSION_SERVICE_URL": // skip pmm-managed environment variables that are already handled by kingpin continue - case "PMM_CLICKHOUSE_DATABASE", "PMM_CLICKHOUSE_ADDR", "PMM_CLICKHOUSE_BLOCK_SIZE", "PMM_CLICKHOUSE_POOL_SIZE": + case "PMM_CLICKHOUSE_DATABASE", "PMM_CLICKHOUSE_ADDR": // skip env variables for external clickhouse continue case "PMM_DISABLE_UPDATES": From bcfe01b8d19487151c3b7a90c1a787e805dc6f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 11 Jun 2024 13:16:11 +0200 Subject: [PATCH 09/13] PMM-10684 Tidy. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 129362306c..80b0c1692d 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,6 @@ require ( github.com/go-sql-driver/mysql v1.7.1 github.com/gogo/status v1.1.1 github.com/golang-migrate/migrate/v4 v4.17.0 - github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/grafana/grafana-api-golang-client v0.27.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 @@ -95,6 +94,7 @@ require ( github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/mock v1.4.4 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.0.0 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect From fa38077c04ee1f700ed4828973adeffea1ca73de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Tue, 11 Jun 2024 14:11:05 +0200 Subject: [PATCH 10/13] PMM-10684 Fix. --- managed/services/supervisord/supervisord.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/managed/services/supervisord/supervisord.go b/managed/services/supervisord/supervisord.go index e0983ba989..49a8963d9c 100644 --- a/managed/services/supervisord/supervisord.go +++ b/managed/services/supervisord/supervisord.go @@ -723,6 +723,8 @@ command = environment = PMM_CLICKHOUSE_ADDR="{{ .ClickhouseAddr }}", PMM_CLICKHOUSE_DATABASE="{{ .ClickhouseDatabase }}", + + user = pmm autorestart = true autostart = true From cbda0a194e0de239296e1f3b7594828d7242622b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 17 Jun 2024 11:07:07 +0200 Subject: [PATCH 11/13] PMM-10684 Fix for plan. --- qan-api2/models/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qan-api2/models/metrics.go b/qan-api2/models/metrics.go index 6db8380acf..23d10b06fd 100644 --- a/qan-api2/models/metrics.go +++ b/qan-api2/models/metrics.go @@ -825,7 +825,7 @@ func (m *Metrics) SelectQueryPlan(ctx context.Context, queryID string) (*qanpb.Q defer cancel() var res qanpb.QueryPlanReply - err := m.db.GetContext(queryCtx, &res, planByQueryID, []interface{}{queryID}) //nolint:asasalint + err := m.db.GetContext(queryCtx, &res, planByQueryID, queryID) //nolint:asasalint if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, fmt.Errorf("QueryxContext error:%v", err) //nolint:errorlint } From 4e6d768d233943460285f6d559255f1fa3eae0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 17 Jun 2024 11:33:21 +0200 Subject: [PATCH 12/13] PMM-10684 Lint. --- qan-api2/models/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qan-api2/models/metrics.go b/qan-api2/models/metrics.go index 23d10b06fd..9633bf65c1 100644 --- a/qan-api2/models/metrics.go +++ b/qan-api2/models/metrics.go @@ -825,7 +825,7 @@ func (m *Metrics) SelectQueryPlan(ctx context.Context, queryID string) (*qanpb.Q defer cancel() var res qanpb.QueryPlanReply - err := m.db.GetContext(queryCtx, &res, planByQueryID, queryID) //nolint:asasalint + err := m.db.GetContext(queryCtx, &res, planByQueryID, queryID) if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, fmt.Errorf("QueryxContext error:%v", err) //nolint:errorlint } From b74cdc9dcbb77a539b24019086f5b6b9408323f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= Date: Mon, 17 Jun 2024 12:25:54 +0200 Subject: [PATCH 13/13] PMM-10684 Env var description. --- docs/process/v2_to_v3_environment_variables.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/process/v2_to_v3_environment_variables.md b/docs/process/v2_to_v3_environment_variables.md index 43edf20f45..91021c6931 100644 --- a/docs/process/v2_to_v3_environment_variables.md +++ b/docs/process/v2_to_v3_environment_variables.md @@ -26,9 +26,11 @@ Below is a list of affected variables and their new names. | `PERCONA_TEST_CHECKS_PUBLIC_KEY` | | Removed in PMM v3, use `PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY` | | `PERCONA_TEST_NICER_API` | `PMM_NICER_API` | | | `PERCONA_TEST_PMM_CLICKHOUSE_ADDR` | `PMM_CLICKHOUSE_ADDR` | | +| `PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE` | | Removed in PMM v3, because of new clickhouse version. | | `PERCONA_TEST_PMM_CLICKHOUSE_DATABASE` | `PMM_CLICKHOUSE_DATABASE` | | | `PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE` | `PMM_CLICKHOUSE_DATASOURCE` | | | `PERCONA_TEST_PMM_CLICKHOUSE_HOST` | `PMM_CLICKHOUSE_HOST` | | +| `PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE` | | Removed in PMM v3, because of new clickhouse version. | | `PERCONA_TEST_PMM_CLICKHOUSE_PORT` | `PMM_CLICKHOUSE_PORT` | | | `PERCONA_TEST_PMM_DISABLE_BUILTIN_CLICKHOUSE` | `PMM_DISABLE_BUILTIN_CLICKHOUSE` | | | `PERCONA_TEST_PMM_DISABLE_BUILTIN_POSTGRES` | `PMM_DISABLE_BUILTIN_POSTGRES` | |