Skip to content

Commit

Permalink
Merge branch 'v3' into PMM-13129-encryption
Browse files Browse the repository at this point in the history
  • Loading branch information
JiriCtvrtka authored Sep 12, 2024
2 parents 182b3d7 + 5561f58 commit 8a805fe
Show file tree
Hide file tree
Showing 27 changed files with 2,489 additions and 672 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ jobs:
- name: Update API compatibility descriptors
run: |
# log if descriptors changed, useful for "update descriptors" PRs
make descriptors
make -C api descriptors
git diff --text
- name: Run check-license
Expand Down
135 changes: 7 additions & 128 deletions Makefile.include
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
# vim: ts=8:sw=8:ft=make:noai:noet
SWAGGER_UI_VERSION?=latest


.PHONY: default init release gen test clean all
default: help
Expand All @@ -25,8 +24,8 @@ release: ## Build release versions of all components
make -C qan-api2 release

gen: clean ## Generate files
make gen-api
make clean_swagger
make -C api gen
make -C api clean-swagger

make -C agent gen
make -C admin gen
Expand All @@ -38,129 +37,13 @@ gen: clean ## Generate files
make format ## TODO: One formatting run is not enough, figure out why.
go install -v ./...

clean: ## Remove generated files
make -C api clean

gen-mocks:
find . -name mock_*.go -delete
./bin/mockery --config .mockery.yaml

gen-api: ## Generate PMM API
# generated by descriptors target
bin/buf breaking --against descriptor.bin api

bin/buf generate -v api

SPECS="\
api/agentlocal/v1 \
api/server/v1 \
api/user/v1 \
api/inventory/v1 \
api/management/v1 \
api/management/v1/service \
api/actions/v1 \
api/advisors/v1 \
api/alerting/v1 \
api/backup/v1 \
api/dump/v1beta1 \
api/accesscontrol/v1beta1 \
api/qan/v1 \
api/platform/v1"; \
for API in $$SPECS; do \
set -x ; \
bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
bin/swagger validate $$API/json/$$(basename $$API).json ; \
bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
--additional-initialism=aws \
--additional-initialism=db \
--additional-initialism=ok \
--additional-initialism=pmm \
--additional-initialism=psmdb \
--additional-initialism=pxc \
--additional-initialism=pt \
--additional-initialism=qan \
--additional-initialism=rds \
--additional-initialism=sql \
--additional-initialism=ha ; \
done

# generate public API spec, omit agentlocal and inventory (always private),
# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
bin/swagger mixin --output=api/swagger/swagger.json \
api/swagger/header.json \
api/server/v1/json/v1.json \
api/user/v1/json/v1.json \
api/inventory/v1/json/v1.json \
api/management/v1/json/v1.json \
api/actions/v1/json/v1.json \
api/alerting/v1/json/v1.json \
api/advisors/v1/json/v1.json \
api/backup/v1/json/v1.json \
api/qan/v1/json/v1.json \
api/platform/v1/json/v1.json
bin/swagger validate api/swagger/swagger.json

bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json

# generate API spec with all PMM Server APIs (omit agentlocal)
bin/swagger mixin --output=api/swagger/swagger-dev.json \
api/swagger/header-dev.json \
api/server/v1/json/v1.json \
api/user/v1/json/v1.json \
api/inventory/v1/json/v1.json \
api/management/v1/json/v1.json \
api/actions/v1/json/v1.json \
api/alerting/v1/json/v1.json \
api/advisors/v1/json/v1.json \
api/backup/v1/json/v1.json \
api/dump/v1beta1/json/v1beta1.json \
api/accesscontrol/v1beta1/json/v1beta1.json \
api/qan/v1/json/v1.json \
api/platform/v1/json/v1.json

bin/swagger validate api/swagger/swagger-dev.json

bin/swagger-order --output=api/swagger/swagger-dev.json api/swagger/swagger-dev.json

clean_swagger:
find api -name '*.swagger.json' -print -delete


ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
get_swagger_version:
override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
else
get_swagger_version:
endif

update-swagger: get_swagger_version ## Update Swagger UI in api/swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;api/swagger/;g'
rm -f api/swagger/.npmrc
sed -i 's/url: ".*"/url: "\/swagger.json",\n validatorUrl: "none"/' api/swagger/swagger-initializer.js

clean: clean_swagger ## Remove generated files
find api -name '*.pb.go' -print -delete
find api -name '*.pb.gw.go' -print -delete
find api -name '*.validate.go' -print -delete

SPECS="\
api/agentlocal/v1 \
api/server/v1 \
api/user/v1 \
api/inventory/v1 \
api/management/v1 \
api/actions/v1 \
api/alerting/v1 \
api/advisors/v1 \
api/backup/v1 \
api/dump/v1beta1 \
api/accesscontrol/v1beta1 \
api/qan/v1 \
api/platform/v1"; \
for API in $$SPECS; do \
rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
done
rm -f api/swagger/swagger.json api/swagger/swagger-dev.json

test-common: ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed)
go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update)

Expand All @@ -182,14 +65,10 @@ check-all: check-license check ## Run golangci linter to check for changes ag
FILES = $(shell find . -type f -name '*.go')

format: ## Format source code
make -C api format
bin/gofumpt -l -w $(FILES)
bin/goimports -local github.com/percona/pmm -l -w $(FILES)
bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
bin/buf format api -w

serve: ## Serve API documentation with nginx
nginx -p . -c api/nginx/nginx.conf

descriptors: ## Update API compatibility descriptors
#./prototool break descriptor-set . -o api/api.descriptor
bin/buf build -o descriptor.bin --as-file-descriptor-set api
36 changes: 36 additions & 0 deletions api-tests/server/updates_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,15 @@ func TestCheckUpdates(t *testing.T) {
assert.NotEmpty(t, res.Payload.Installed.FullVersion)

if res.Payload.UpdateAvailable {
require.NotEmpty(t, res.Payload.Latest)
assert.True(t, strings.HasPrefix(res.Payload.Latest.Version, "2.") || strings.HasPrefix(res.Payload.Installed.Version, "3."),
"latest.version = %q should have '2.' or '3.' prefix", res.Payload.Latest.Version)
require.NotEmpty(t, res.Payload.Latest.Timestamp)
ts = time.Time(res.Payload.Latest.Timestamp)
hour, min, _ = ts.Clock()
assert.Zero(t, hour, "latest.timestamp should contain only date")
assert.Zero(t, min, "latest.timestamp should contain only date")

assert.NotEmpty(t, res.Payload.Latest.Tag)
require.NotEmpty(t, res.Payload.Latest.Timestamp)
ts = time.Time(res.Payload.Latest.Timestamp)
Expand All @@ -81,6 +90,7 @@ func TestCheckUpdates(t *testing.T) {
assert.NotEqual(t, res.Payload.Installed.FullVersion, res.Payload.Latest.Version)
assert.NotEqual(t, res.Payload.Installed.Timestamp, res.Payload.Latest.Timestamp)
assert.True(t, strings.HasPrefix(res.Payload.LatestNewsURL, "https://per.co.na/pmm/2."), "latest_news_url = %q", res.Payload.LatestNewsURL)
assert.True(t, strings.HasPrefix(res.Payload.Latest.ReleaseNotesURL, "https://per.co.na/pmm/2."), "latest_news_url = %q", res.Payload.Latest.ReleaseNotesURL)
}
assert.NotEmpty(t, res.Payload.LastCheck)

Expand Down Expand Up @@ -110,6 +120,32 @@ func TestCheckUpdates(t *testing.T) {
})
}

func TestListUpdates(t *testing.T) {
const fast, slow = 5 * time.Second, 60 * time.Second

if !pmmapitests.RunUpdateTest {
t.Skip("skipping PMM Server check update test")
}

version, err := serverClient.Default.ServerService.Version(server.NewVersionParamsWithTimeout(fast))
require.NoError(t, err)
if version.Payload.Server == nil || version.Payload.Server.Version == "" {
t.Skip("skipping test in developer's environment")
}

params := &server.ListChangeLogsParams{
Context: pmmapitests.Context,
}
params.SetTimeout(slow)
res, err := serverClient.Default.ServerService.ListChangeLogs(params)
require.NoError(t, err)

if len(res.Payload.Updates) > 0 {
assert.True(t, strings.HasPrefix(res.Payload.Updates[0].Version, "3."),
"installed.version = %q should have '3.' prefix", res.Payload.Updates[0].Version)
}
}

func TestUpdate(t *testing.T) {
// do not run this test in parallel with other tests

Expand Down
127 changes: 127 additions & 0 deletions api/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
# vim: ts=8:sw=8:ft=make:noai:noet
SWAGGER_UI_VERSION?=latest

gen: ## Generate PMM API
# generated by descriptors target
../bin/buf breaking --against descriptor.bin .

../bin/buf generate -v .

SPECS="\
agentlocal/v1 \
server/v1 \
user/v1 \
inventory/v1 \
management/v1 \
management/v1/service \
actions/v1 \
advisors/v1 \
alerting/v1 \
backup/v1 \
dump/v1beta1 \
accesscontrol/v1beta1 \
qan/v1 \
platform/v1"; \
for API in $$SPECS; do \
set -x ; \
../bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
../bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
../bin/swagger validate $$API/json/$$(basename $$API).json ; \
../bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
--additional-initialism=aws \
--additional-initialism=db \
--additional-initialism=ok \
--additional-initialism=pmm \
--additional-initialism=psmdb \
--additional-initialism=pxc \
--additional-initialism=pt \
--additional-initialism=qan \
--additional-initialism=rds \
--additional-initialism=sql \
--additional-initialism=ha ; \
done

# generate public API spec, omit agentlocal and inventory (always private),
# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
../bin/swagger mixin --output=swagger/swagger.json \
swagger/header.json \
server/v1/json/v1.json \
user/v1/json/v1.json \
inventory/v1/json/v1.json \
management/v1/json/v1.json \
actions/v1/json/v1.json \
alerting/v1/json/v1.json \
advisors/v1/json/v1.json \
backup/v1/json/v1.json \
qan/v1/json/v1.json \
platform/v1/json/v1.json
../bin/swagger validate swagger/swagger.json

../bin/swagger-order --output=swagger/swagger.json swagger/swagger.json

# generate API spec with all PMM Server APIs (omit agentlocal)
../bin/swagger mixin --output=swagger/swagger-dev.json \
swagger/header-dev.json \
server/v1/json/v1.json \
user/v1/json/v1.json \
inventory/v1/json/v1.json \
management/v1/json/v1.json \
actions/v1/json/v1.json \
alerting/v1/json/v1.json \
advisors/v1/json/v1.json \
backup/v1/json/v1.json \
dump/v1beta1/json/v1beta1.json \
accesscontrol/v1beta1/json/v1beta1.json \
qan/v1/json/v1.json \
platform/v1/json/v1.json

../bin/swagger validate swagger/swagger-dev.json

../bin/swagger-order --output=swagger/swagger-dev.json swagger/swagger-dev.json

format: ## Format API definitions
../bin/buf format . -w

clean-swagger:
find . -name '*.swagger.json' -print -delete


ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
get_swagger_version:
override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
else
get_swagger_version:
endif

update-swagger: get_swagger_version ## Update Swagger UI in swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;swagger/;g'
rm -f swagger/.npmrc
sed -i 's/url: ".*"/url: "\/swagger.json",\n validatorUrl: "none"/' swagger/swagger-initializer.js

clean: clean-swagger ## Remove generated files
find . -name '*.pb.go' -print -delete
find . -name '*.pb.gw.go' -print -delete
find . -name '*.validate.go' -print -delete

SPECS="\
agentlocal/v1 \
server/v1 \
user/v1 \
inventory/v1 \
management/v1 \
actions/v1 \
alerting/v1 \
advisors/v1 \
backup/v1 \
dump/v1beta1 \
accesscontrol/v1beta1 \
qan/v1 \
platform/v1"; \
for API in $$SPECS; do \
rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
done
rm -f swagger/swagger.json swagger/swagger-dev.json

descriptors: ## Update API compatibility descriptors
../bin/buf build -o descriptor.bin --as-file-descriptor-set .
34 changes: 34 additions & 0 deletions api/buf.gen.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
version: v2
managed:
enabled: true
disable:
- file_option: go_package
module: buf.build/googleapis/googleapis
- file_option: go_package
module: buf.build/grpc-ecosystem/grpc-gateway
- file_option: go_package
module: buf.build/envoyproxy/protoc-gen-validate
override:
- file_option: go_package_prefix
value: github.com/percona/pmm/api
plugins:
- local: ../bin/protoc-gen-go
out: .
opt: paths=source_relative
- local: ../bin/protoc-gen-go-grpc
out: .
opt: paths=source_relative
- local: ../bin/protoc-gen-grpc-gateway
out: .
opt: paths=source_relative
- local: ../bin/protoc-gen-validate
out: .
opt:
- paths=source_relative
- lang=go
- local: ../bin/protoc-gen-openapiv2
out: .
opt:
- simple_operation_ids=true
- json_names_for_fields=false
- proto3_optional_nullable=true
Loading

0 comments on commit 8a805fe

Please sign in to comment.