Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PMM-10684 Clickhouse v2. #1542

Merged
merged 15 commits into from
Jun 18, 2024
Merged

PMM-10684 Clickhouse v2. #1542

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
e1135ac
PMM-12551 Clickhouse v2.
JiriCtvrtka Mar 22, 2024
77e9a3f
PMM-12551 Fix for new connection string format.
JiriCtvrtka Mar 21, 2024
7cd9ca9
PMM-12551 Another fix for new connection string.
JiriCtvrtka Mar 21, 2024
92449b1
PMM-10684 Remove block size, pool size (not supported anymore).
JiriCtvrtka Mar 21, 2024
6e644ea
PMM-10684 Supervisor fix.
JiriCtvrtka Mar 22, 2024
05f03fa
PMM-10684 Small change.
JiriCtvrtka Mar 22, 2024
0ee113b
Merge branch 'v3' into PMM-10684-clickhouse-v2
JiriCtvrtka Jun 11, 2024
d7ab083
PMM-10684 QAN fix after merge.
JiriCtvrtka Jun 11, 2024
705bae9
PMM-10684 Remove outdated env variables.
JiriCtvrtka Jun 11, 2024
bcfe01b
PMM-10684 Tidy.
JiriCtvrtka Jun 11, 2024
fa38077
PMM-10684 Fix.
JiriCtvrtka Jun 11, 2024
cbda0a1
PMM-10684 Fix for plan.
JiriCtvrtka Jun 17, 2024
4e6d768
PMM-10684 Lint.
JiriCtvrtka Jun 17, 2024
b74cdc9
PMM-10684 Env var description.
JiriCtvrtka Jun 17, 2024
3a02a10
Merge branch 'v3' into PMM-10684-clickhouse-v2
JiriCtvrtka Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,9 @@ replace gopkg.in/alecthomas/kingpin.v2 => github.com/Percona-Lab/kingpin v2.2.6-

replace golang.org/x/crypto => github.com/percona-lab/crypto v0.0.0-20231108144114-756dfb24eaf2

replace github.com/ClickHouse/clickhouse-go/151 => github.com/ClickHouse/clickhouse-go v1.5.1 // clickhouse-go/v2 cannot work with 1.5.1 which we need for QAN-API

require (
github.com/AlekSi/pointer v1.2.0
github.com/ClickHouse/clickhouse-go/151 v0.0.0-00010101000000-000000000000
github.com/ClickHouse/clickhouse-go/v2 v2.21.1
github.com/ClickHouse/clickhouse-go/v2 v2.22.2
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/alecthomas/kong v0.9.0
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
Expand Down Expand Up @@ -78,7 +75,7 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80
google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80
google.golang.org/grpc v1.62.0
google.golang.org/protobuf v1.32.0
google.golang.org/protobuf v1.33.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/reform.v1 v1.5.1
gopkg.in/yaml.v3 v3.0.1
Expand Down Expand Up @@ -119,7 +116,6 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
github.com/ClickHouse/ch-go v0.61.5 // indirect
github.com/ClickHouse/clickhouse-go v1.5.4 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
Expand All @@ -128,7 +124,6 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/charmbracelet/harmonica v0.2.0 // indirect
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
Expand Down
18 changes: 4 additions & 14 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
github.com/ClickHouse/clickhouse-go v1.5.1 h1:I8zVFZTz80crCs0FFEBJooIxsPcV0xfthzK1YrkpJTc=
github.com/ClickHouse/clickhouse-go v1.5.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=
github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/ClickHouse/clickhouse-go/v2 v2.21.1 h1:x8wZEMOHDh4K8kLQBtGMeIIguejiaj8/bUiF2VzG6n4=
github.com/ClickHouse/clickhouse-go/v2 v2.21.1/go.mod h1:hTWNkV9mkQwiQ/df0rbN17VXF05UTResY4krnjbzVZA=
github.com/ClickHouse/clickhouse-go/v2 v2.22.2 h1:T1BljsIjj+3aQog80jKMTeF4EqAUG4P6TVcCvmakYAc=
github.com/ClickHouse/clickhouse-go/v2 v2.22.2/go.mod h1:tBhdF3f3RdP7sS59+oBAtTyhWpy0024ZxDMhgxra0QE=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
Expand Down Expand Up @@ -102,8 +100,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8=
Expand Down Expand Up @@ -212,7 +208,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=
github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
Expand Down Expand Up @@ -347,7 +342,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
Expand Down Expand Up @@ -392,7 +386,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
Expand All @@ -416,7 +409,6 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
Expand Down Expand Up @@ -489,8 +481,6 @@ github.com/percona/promconfig v0.2.5 h1:f/HN/CbECQs7d9RIB6MKVkuXstsrsqEDxRvf6yig
github.com/percona/promconfig v0.2.5/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
github.com/pganalyze/pg_query_go/v2 v2.2.0 h1:OW+reH+ZY7jdEuPyuLGlf1m7dLbE+fDudKXhLs0Ttpk=
github.com/pganalyze/pg_query_go/v2 v2.2.0/go.mod h1:XAxmVqz1tEGqizcQ3YSdN90vCOHBWjJi8URL1er5+cA=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
Expand Down Expand Up @@ -962,8 +952,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
2 changes: 0 additions & 2 deletions managed/CONTRIBUTING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,6 @@ go test -timeout=30s -p 1 ./...
| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- |
| PERCONA_TEST_PMM_CLICKHOUSE_ADDR | Sets Clickhouse address | 127.0.0.1:9000 |
| PERCONA_TEST_PMM_CLICKHOUSE_DATABASE | Sets Clickhouse database | pmm |
| PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE | Sets Clickhouse connections pool size | none |
| PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE | Sets Clickhouse block size | none |
| PERCONA_TEST_STARLARK_ALLOW_RECURSION | Allows recursive functions in checks scripts | false |
| PERCONA_TEST_NICER_API | Enables nicer API with default/zero values in response. | false |
| PERCONA_TEST_VERSION_SERVICE_URL | Sets versions service URL | https://check.percona.com/versions/v1 |
Expand Down
6 changes: 0 additions & 6 deletions managed/services/supervisord/supervisord.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,8 +425,6 @@
clickhouseDatabase := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_DATABASE", defaultClickhouseDatabase)
clickhouseAddr := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_ADDR", defaultClickhouseAddr)
clickhouseDataSourceAddr := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_DATASOURCE_ADDR", defaultClickhouseDataSourceAddr)
clickhousePoolSize := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE", "")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did they remove these two params from the client?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, I see the commit message - it's no longer supported )

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In v2 pool size and block size changed. It is also recommended to set them agains exact DB.

See: https://clickhouse.com/docs/en/integrations/go
MaxIdleConns should be alternative for pool size
BlockBufferSize for block size

But our block size default was 10 000 and BlockBufferSize did not accept such number, only lower ones, like 100.

time="2024-03-22T07:35:19.278+00:00" level=info msg="DSN: clickhouse://127.0.0.1:9000/pmm?block_buffer_size=10000&max_idle_conns=2" component=main stdlog: Connection: strconv.ParseUint: parsing "10000": value out of range

So seems it is slightly different. MaxIdleConns should behave same like pool size, but since both of them are with prefix "PERCONA_TEST" I decided to remove them.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should the PR be against v3 branch?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My fault. Thanks for notice that.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't we treat variables prefixed with PERCONA_TEST_ as non-public, internal ones?

I'm referring to

pmm/managed/CONTRIBUTING.md

Line 54 in 70d5afd

## Available test environment variables:

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean they should be used in non-prod environments, i.e. test environments.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right, seems like we can merge it to PMM 2 as well

Copy link
Contributor Author

@JiriCtvrtka JiriCtvrtka Mar 22, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@BupycHuk So should I change it to v2 (ticket and PR)? Thanks.

If yes, I will keep this one to merge it into v3 and for v2 I will create another PR and update ticket.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's do it for v3 now, later we can backport to v2

clickhouseBlockSize := getValueFromENV("PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE", "")
clickhouseAddrPair := strings.SplitN(clickhouseAddr, ":", 2)
vmSearchDisableCache := getValueFromENV("VM_search_disableCache", strconv.FormatBool(!settings.VictoriaMetrics.CacheEnabled))
vmSearchMaxQueryLen := getValueFromENV("VM_search_maxQueryLen", defaultVMSearchMaxQueryLen)
Expand Down Expand Up @@ -457,8 +455,6 @@
"ClickhouseAddr": clickhouseAddr,
"ClickhouseDataSourceAddr": clickhouseDataSourceAddr,
"ClickhouseDatabase": clickhouseDatabase,
"ClickhousePoolSize": clickhousePoolSize,
"ClickhouseBlockSize": clickhouseBlockSize,
"ClickhouseHost": clickhouseAddrPair[0],
"ClickhousePort": clickhouseAddrPair[1],
}
Expand Down Expand Up @@ -534,7 +530,7 @@
// Returns true if configuration was changed.
func (s *Service) saveConfigAndReload(name string, cfg []byte) (bool, error) {
// read existing content
path := filepath.Join(s.configDir, name+".ini")

Check failure on line 533 in managed/services/supervisord/supervisord.go

View workflow job for this annotation

GitHub Actions / Checks 

string `.ini` has 3 occurrences, make it a constant (goconst)
oldCfg, err := os.ReadFile(path) //nolint:gosec
if errors.Is(err, fs.ErrNotExist) {
err = nil
Expand Down Expand Up @@ -727,8 +723,6 @@
environment =
PERCONA_TEST_PMM_CLICKHOUSE_ADDR="{{ .ClickhouseAddr }}",
PERCONA_TEST_PMM_CLICKHOUSE_DATABASE="{{ .ClickhouseDatabase }}",
{{ if .ClickhousePoolSize }} PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE={{ .ClickhousePoolSize }},{{- end}}
{{ if .ClickhouseBlockSize }} PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE={{ .ClickhouseBlockSize }}{{- end}}
user = pmm
autorestart = true
autostart = true
Expand Down
2 changes: 1 addition & 1 deletion managed/services/telemetry/telemetry_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestRunTelemetryService(t *testing.T) {
if ok {
pgHostPort = pgHostPortFromEnv
}
qanDSN := "tcp://localhost:9000?database=pmm"
qanDSN := "tcp://localhost:9000/pmm"
qanDSNFromEnv, ok := os.LookupEnv(envQanDSN)
if ok {
qanDSN = qanDSNFromEnv
Expand Down
2 changes: 0 additions & 2 deletions managed/testdata/supervisord.d/qan-api2.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ command =
environment =
PERCONA_TEST_PMM_CLICKHOUSE_ADDR="127.0.0.1:9000",
PERCONA_TEST_PMM_CLICKHOUSE_DATABASE="pmm",


user = pmm
autorestart = true
autostart = true
Expand Down
9 changes: 3 additions & 6 deletions qan-api2/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
"net/url"
"strings"

clickhouse "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver
clickhouse "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver
"github.com/golang-migrate/migrate/v4"
_ "github.com/golang-migrate/migrate/v4/database/clickhouse" // register golang-migrate driver
"github.com/golang-migrate/migrate/v4/source/iofs"
Expand Down Expand Up @@ -78,11 +78,8 @@ func createDB(dsn string) error {
if err != nil {
return err
}
q := clickhouseURL.Query()
databaseName := q.Get("database")
q.Set("database", "default")

clickhouseURL.RawQuery = q.Encode()
databaseName := strings.Replace(clickhouseURL.Path, "/", "", 1)
clickhouseURL.Path = "/default"

defaultDB, err := sqlx.Connect("clickhouse", clickhouseURL.String())
if err != nil {
Expand Down
12 changes: 6 additions & 6 deletions qan-api2/db_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"testing"
"time"

_ "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver
_ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver
_ "github.com/golang-migrate/migrate/v4/database/clickhouse"
"github.com/jmoiron/sqlx"
"github.com/stretchr/testify/assert"
Expand All @@ -39,9 +39,9 @@ func setup() *sqlx.DB {
}

dsn, ok := os.LookupEnv("QANAPI_DSN_TEST")
dsn = strings.Replace(dsn, "?database=pmm_test", "?database=pmm_test_parts", 1)
dsn = strings.Replace(dsn, "/pmm_test", "/pmm_test_parts", 1)
if !ok {
dsn = "clickhouse://127.0.0.1:19000?database=pmm_test_parts"
dsn = "clickhouse://127.0.0.1:19000/pmm_test_parts"
}
db, err := sqlx.Connect("clickhouse", dsn)
if err != nil {
Expand Down Expand Up @@ -113,13 +113,13 @@ func TestCreateDbIfNotExists(t *testing.T) {
t.Run("connect to db that doesnt exist", func(t *testing.T) {
dsn, ok := os.LookupEnv("QANAPI_DSN_TEST")

dsn = strings.Replace(dsn, "?database=pmm_test", "?database=pmm_created_db", 1)
dsn = strings.Replace(dsn, "/pmm_test", "/pmm_created_db", 1)
if !ok {
dsn = "clickhouse://127.0.0.1:19000?database=pmm_created_db"
dsn = "clickhouse://127.0.0.1:19000/pmm_created_db"
}

db := createDB(dsn)

require.Equal(t, db, nil, "Check connection after we create database")
require.Nil(t, db, "Check connection after we create database")
})
}
8 changes: 2 additions & 6 deletions qan-api2/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ import (

const (
shutdownTimeout = 3 * time.Second
defaultDsnF = "clickhouse://%s?database=%s&block_size=%s&pool_size=%s"
defaultDsnF = "clickhouse://%s/%s"
maxIdleConns = 5
maxOpenConns = 10
)
Expand Down Expand Up @@ -264,10 +264,6 @@ func main() {
dsnF := kingpin.Flag("dsn", "ClickHouse database DSN. Can be override with database/host/port options").Default(defaultDsnF).String()
clickHouseDatabaseF := kingpin.Flag("clickhouse-name", "Clickhouse database name").Default("pmm").Envar("PERCONA_TEST_PMM_CLICKHOUSE_DATABASE").String()
clickhouseAddrF := kingpin.Flag("clickhouse-addr", "Clickhouse database address").Default("127.0.0.1:9000").Envar("PERCONA_TEST_PMM_CLICKHOUSE_ADDR").String()
clickhouseBlockSizeF := kingpin.Flag("clickhouse-block-size", "Number of rows that can be load from table in one cycle").
Default("10000").Envar("PERCONA_TEST_PMM_CLICKHOUSE_BLOCK_SIZE").String()
clickhousePoolSizeF := kingpin.Flag("clickhouse-pool-size", "Controls how much queries can be run simultaneously").
Default("2").Envar("PERCONA_TEST_PMM_CLICKHOUSE_POOL_SIZE").String()

debugF := kingpin.Flag("debug", "Enable debug logging").Bool()
traceF := kingpin.Flag("trace", "Enable trace logging (implies debug)").Bool()
Expand Down Expand Up @@ -295,7 +291,7 @@ func main() {

var dsn string
if *dsnF == defaultDsnF {
dsn = fmt.Sprintf(defaultDsnF, *clickhouseAddrF, *clickHouseDatabaseF, *clickhouseBlockSizeF, *clickhousePoolSizeF)
dsn = fmt.Sprintf(defaultDsnF, *clickhouseAddrF, *clickHouseDatabaseF)
} else {
dsn = *dsnF
}
Expand Down
4 changes: 2 additions & 2 deletions qan-api2/services/analytics/filters_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"testing"
"time"

_ "github.com/ClickHouse/clickhouse-go/151" // register database/sql driver
_ "github.com/ClickHouse/clickhouse-go/v2" // register database/sql driver
// TODO replace with 'google.golang.org/protobuf/encoding/protojson' since this one is deprecated.
"github.com/golang/protobuf/ptypes/timestamp"
"github.com/jmoiron/sqlx"
Expand Down Expand Up @@ -54,7 +54,7 @@ type testValuesUnmarshal struct {
func TestService_GetFilters(t *testing.T) {
dsn, ok := os.LookupEnv("QANAPI_DSN_TEST")
if !ok {
dsn = "clickhouse://127.0.0.1:19000?database=pmm_test"
dsn = "clickhouse://127.0.0.1:19000/pmm_test"
}
db, err := sqlx.Connect("clickhouse", dsn)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion qan-api2/services/analytics/profile_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import (
func setup() *sqlx.DB {
dsn, ok := os.LookupEnv("QANAPI_DSN_TEST")
if !ok {
dsn = "clickhouse://127.0.0.1:19000?database=pmm_test"
dsn = "clickhouse://127.0.0.1:19000/pmm_test"
}
db, err := sqlx.Connect("clickhouse", dsn)
if err != nil {
Expand Down
Loading