Merge branch 'v3' into PMM-11231-pmm-updates-ui

.github/workflows/clean.yml

@@ -15,8 +15,6 @@ jobs:
         go:
           - version: 1.22.x
             may-fail: false
-          - version: tip
-            may-fail: true
 
     continue-on-error: ${{ matrix.go.may-fail }}
     runs-on: ubuntu-22.04

.github/workflows/devcontainer.yml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   devcontainer:
-    name: Build
+    name: Devcontainer
     runs-on: ubuntu-22.04
     timeout-minutes: 15
     permissions:
@@ -51,9 +51,10 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and push to registries
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
-          file: ./devcontainer.Dockerfile
+          file: ./.devcontainer/Dockerfile
+          context: .
           push: true
           tags: |
             ${{ env.GH_DEVCONTAINER_IMAGE }}

.github/workflows/main.yml

@@ -93,15 +93,14 @@ jobs:
       - name: Run go-consistent
         env:
           COMMAND: 'bin/go-consistent -pedantic -exclude "tests" ./...'
-          REDIRECT: "| bin/reviewdog -f=go-consistent -reporter=github-pr-review -fail-on-error=true"
           REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.ROBOT_TOKEN || secrets.GITHUB_TOKEN }}
         run: |
           if out=$( ${{ env.COMMAND }} ); exit_code=$?; [ $exit_code -ne 0 ]; then
             if [ $exit_code -gt 1 ] || ${{ github.event.pull_request == null }}; then
               echo "$out"
               exit $exit_code
             else
-              echo "$out" ${{ env.REDIRECT }}
+              echo "$out" | bin/reviewdog -f=go-consistent -reporter=github-pr-review -fail-on-error=true
             fi
           else
             echo "$out"
@@ -113,10 +112,8 @@ jobs:
       - name: Run debug commands on failure
         if: ${{ failure() }}
         run: |
-          env
-          go version
-          go env
-          pwd
+          env | sort
+          go env | sort
           git status
 
   merge-gatekeeper:

.github/workflows/managed.yml

@@ -22,82 +22,56 @@ on:
 
 jobs:
   test:
-    name: Tests
+    name: Managed tests
     runs-on: ubuntu-22.04
     timeout-minutes: 30
+    permissions:
+      packages: write    
 
     env:
       PMM_SERVER_IMAGE: perconalab/pmm-server:3-dev-latest
+      PMM_CACHE_IMAGE: ghcr.io/percona/pmm:dev-cache
       AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
       AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
       PMM_DEV_OAUTH_CLIENT_ID: ${{ secrets.OAUTH_PMM_CLIENT_ID }}
       PMM_DEV_OAUTH_CLIENT_SECRET: ${{ secrets.OAUTH_PMM_CLIENT_SECRET }}
-      DEVCONTAINER_CACHE_ENABLED: false
+      BUILD_CACHE: ${{ github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'v3' }} # change to `main` once v3 goes GA
 
     steps:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Enable Go build cache
-        if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/go-build
-          key: ${{ runner.os }}-go-build-${{ github.ref }}-${{ hashFiles('**') }}
-          restore-keys: |
-            ${{ runner.os }}-go-build-${{ github.ref }}-
-            ${{ runner.os }}-go-build-
-
-      - name: Enable Go modules cache
-        if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
-        uses: actions/cache@v4
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }}
-          restore-keys: ${{ runner.os }}-go-modules-
-
-      - name: Download tools
-        if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
-        run: |
-          pushd tools && go mod download -x
-          popd        && go mod download -x
-
       - name: Launch PMM Server (see docker-compose.yml)
         run: |
-          # Note: launching the container with --wait fails for an unknown reason.
-          # A temporary workaround is to run it manually. To be reverted once the issue is resolved.
-          # make env-compose-up # the container workdir is /root/go/src/github.com/percona/pmm
-          docker compose --profile pmm up -d
-          sleep 100s
+          make env-compose-up # the container workdir is /root/go/src/github.com/percona/pmm
           docker logs pmm-server
-      - name: Restore Go build cache
-        if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
-        continue-on-error: true
-        run: docker cp ~/.cache/go-build pmm-server:/root/.cache/go-build
-
-      - name: Restore Go modules cache
-        if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
-        continue-on-error: true
-        run: docker cp ~/go/pkg/mod pmm-server:/root/go/pkg/mod
 
       - name: Mark the root directory of pmm as safe
         run: docker exec -i pmm-server git config --global --add safe.directory /root/go/src/github.com/percona/pmm
 
-      - name: Update binaries
+      - name: Pull the cache image and inject cache to PMM Server
+        continue-on-error: true
         run: |
-          docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy
+          if docker pull --platform linux/amd64 ${{ env.PMM_CACHE_IMAGE }}; then
+            docker run --rm --volumes-from pmm-server ${{ env.PMM_CACHE_IMAGE }} sh -c "cp -rf /mod-cache/* /root/go/pkg/mod; cp -rf /build-cache/* /root/.cache/go-build"
+            docker exec -t pmm-server du -sh /root/.cache/go-build
+            docker exec -t pmm-server du -sh /root/go/pkg/mod
+          fi
+
+      - name: Remove the cache image
+        continue-on-error: true
+        run: docker image rm ${{ env.PMM_CACHE_IMAGE }}
+
+      - name: Rebuild the binaries
+        run: docker exec -i pmm-server make run-managed-ci run-agent run-vmproxy run-qan
 
       - name: Check the status of components
-        run: docker exec -t pmm-server supervisorctl status || true
+        continue-on-error: true
+        run: docker exec -t pmm-server supervisorctl status
 
       - name: Run tests
         run: docker exec -i pmm-server make -C managed test-cover
 
-      - name: Run PMM server update test
-        run: |
-          echo "PMM Server update test will be refactored once we have a new update mechanism."
-          # docker exec -i pmm-server make -C managed test-update
-
       - name: Upload coverage results
         uses: codecov/codecov-action@v4
         with:
@@ -107,18 +81,47 @@ jobs:
           fail_ci_if_error: false
           token: ${{ secrets.CODECOV_TOKEN }}
 
-      - name: Cache
-        if: ${{ fromJSON(env.DEVCONTAINER_CACHE_ENABLED) }}
+      - name: Run PMM Server update test
+        run: |
+          echo "PMM Server update test will be refactored once we have a new update mechanism."
+          # docker exec -i pmm-server make -C managed test-update
+
+      - name: Save PMM Go module and build cache as an image
+        if: ${{ fromJSON(env.BUILD_CACHE) }}
         run: |
-          docker exec pmm-server go clean -testcache
-          docker exec pmm-server find ./managed -type d -name fuzzdata -exec rm -r {} +
-          rm -fr ~/.cache/go-build
-          mkdir -p ~/.cache
-          docker cp pmm-server:/root/.cache/go-build ~/.cache/go-build
-
-      - name: Run debug commands on failure
-        if: ${{ failure() }}
+          df -h | grep -A 1 Filesystem
+          # We need to free up some space, see more https://github.com/actions/runner-images/issues/2840
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          df -h | grep -A 1 Filesystem
+          docker run --name pmm-cache --volumes-from pmm-server busybox sh -c "mkdir /mod-cache /build-cache; cp -r /root/go/pkg/mod/* /mod-cache; cp -r /root/.cache/go-build/* /build-cache"
+          docker commit pmm-cache ${{ env.PMM_CACHE_IMAGE }}
+          docker rm -v pmm-cache
+
+      - name: Login to ghcr.io registry
+        if: ${{ fromJSON(env.BUILD_CACHE) }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push the image to ghcr.io
+        if: ${{ fromJSON(env.BUILD_CACHE) }}
+        run: docker push ${{ env.PMM_CACHE_IMAGE }}
+
+      - name: Run debug commands
+        if: ${{ always() }}
         run: |
           env | sort
           go env | sort
           git status
+          docker exec pmm-server env | sort
+          docker exec pmm-server go env | sort
+          docker exec pmm-server supervisorctl status || true
+          services=$(docker exec pmm-server supervisorctl status | awk '{print $1}')
+          while IFS= read -r service; do
+            echo "Logs for $service:"
+            docker exec pmm-server supervisorctl tail $service
+          done <<< "$services"

Makefile

@@ -8,7 +8,7 @@ endif
 
 env-up: 							## Start devcontainer
 	COMPOSE_PROFILES=$(PROFILES) \
-	docker compose up -d
+	docker compose up -d --wait --wait-timeout 100
 
 env-up-rebuild: env-update-image	## Rebuild and start devcontainer. Useful for custom $PMM_SERVER_IMAGE
 	COMPOSE_PROFILES=$(PROFILES) \

Makefile.devcontainer

@@ -11,6 +11,9 @@ release-dev-agent:                ## Build pmm-agent
 release-vmproxy:                  ## Build vmproxy
 	make -C vmproxy release
 
+release-qan:                      ## Build QAN
+	make -C qan-api2 release
+
 # used by host Makefile
 _bash:
 	/bin/bash
@@ -38,6 +41,12 @@ run-vmproxy: release-vmproxy
 	cp $(PMM_RELEASE_PATH)/vmproxy /usr/sbin/vmproxy
 	supervisorctl start vmproxy
 
+run-qan: release-qan
+	supervisorctl stop qan-api2
+	cp $(PMM_RELEASE_PATH)/qan-api2 /usr/sbin/percona-qan-api2
+	echo -n > /srv/logs/qan-api2.log
+	supervisorctl start qan-api2
+
 run-all: run-agent run-managed       ## Run pmm-managed and pmm-agent
 
 run:                                 ## Deprecated

agent/agents/postgres/parser/parser.go

@@ -21,7 +21,7 @@ import (
 	"sort"
 	"strings"
 
-	pgquery "github.com/pganalyze/pg_query_go/v2"
+	pgquery "github.com/pganalyze/pg_query_go/v5"
 	"github.com/pkg/errors"
 )
 

agent/agents/postgres/parser/testdata/query019.json

@@ -1,3 +1,8 @@
 {
-  "error": "error on parsing sql query: subquery in FROM must have an alias"
+  "tables": [
+    "credit_card_wo_complaints",
+    "without_complaints"
+  ],
+  "error": ""
 }
+

agent/agents/postgres/pgstatmonitor/stat_monitor_cache.go

@@ -21,7 +21,7 @@ import (
 	"time"
 
 	"github.com/AlekSi/pointer"
-	pgquery "github.com/pganalyze/pg_query_go/v2"
+	pgquery "github.com/pganalyze/pg_query_go/v5"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/reform.v1"

agent/agents/postgres/pgstatstatements/pgstatstatements_test.go

@@ -398,10 +398,12 @@ func TestPGStatStatementsQAN(t *testing.T) {
 		require.Len(t, buckets, 1)
 
 		var fingerprint string
+		tables := []string{tableName}
+
 		switch engineVersion {
 		case "9.4", "9.5", "9.6":
 			fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES (?, ?, ?, ?)`, tableName)
-
+			tables = []string{}
 		default:
 			fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName)
 		}
@@ -412,7 +414,7 @@ func TestPGStatStatementsQAN(t *testing.T) {
 				Queryid:             actual.Common.Queryid,
 				Fingerprint:         fingerprint,
 				Database:            "pmm-agent",
-				Tables:              []string{tableName},
+				Tables:              tables,
 				Comments:            map[string]string{"controller": "test"},
 				Username:            "pmm-agent",
 				AgentId:             "agent_id",

api-tests/helpers.go

@@ -145,6 +145,7 @@ func UnregisterNodes(t TestingT, nodeIDs ...string) {
 		params := &node.UnregisterNodeParams{
 			Body: node.UnregisterNodeBody{
 				NodeID: nodeID,
+				Force:  true,
 			},
 			Context: context.Background(),
 		}

api-tests/management/nodes_test.go

@@ -150,7 +150,6 @@ func TestNodeRegister(t *testing.T) {
 			})
 			assert.NotEmpty(t, nodeID)
 			assert.NotEmpty(t, pmmAgentID)
-			pmmapitests.UnregisterNodes(t, nodeID)
 
 			body := node.RegisterNodeBody{
 				NodeName:   nodeName + "_new",

api-tests/server/updates_test.go

@@ -67,20 +67,20 @@ func TestCheckUpdates(t *testing.T) {
 	require.NotEmpty(t, res.Payload.Latest)
 	assert.True(t, strings.HasPrefix(res.Payload.Latest.Version, "2."),
 		"latest.version = %q should have '2.' prefix", res.Payload.Latest.Version)
-	assert.NotEmpty(t, res.Payload.Latest.FullVersion)
 	require.NotEmpty(t, res.Payload.Latest.Timestamp)
 	ts = time.Time(res.Payload.Latest.Timestamp)
 	hour, min, _ = ts.Clock()
 	assert.Zero(t, hour, "latest.timestamp should contain only date")
 	assert.Zero(t, min, "latest.timestamp should contain only date")
 
 	if res.Payload.UpdateAvailable {
-		assert.NotEqual(t, res.Payload.Installed.FullVersion, res.Payload.Latest.FullVersion)
+		assert.NotEmpty(t, res.Payload.Latest.Tag)
+		assert.NotEqual(t, res.Payload.Installed.FullVersion, res.Payload.Latest.Version)
 		assert.NotEqual(t, res.Payload.Installed.Timestamp, res.Payload.Latest.Timestamp)
 		assert.True(t, strings.HasPrefix(res.Payload.LatestNewsURL, "https://per.co.na/pmm/2."), "latest_news_url = %q", res.Payload.LatestNewsURL)
 	} else {
-		assert.Equal(t, res.Payload.Installed.FullVersion, res.Payload.Latest.FullVersion)
 		assert.Equal(t, res.Payload.Installed.Timestamp, res.Payload.Latest.Timestamp)
+		assert.Empty(t, res.Payload.Installed.FullVersion, res.Payload.Latest.Version)
 		assert.Empty(t, res.Payload.LatestNewsURL, "latest_news_url should be empty")
 	}
 	assert.NotEmpty(t, res.Payload.LastCheck)
@@ -108,7 +108,7 @@ func TestCheckUpdates(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, res.Payload.Installed, resForce.Payload.Installed)
-		assert.Equal(t, resForce.Payload.Installed.FullVersion != resForce.Payload.Latest.FullVersion, resForce.Payload.UpdateAvailable)
+		assert.Equal(t, resForce.Payload.Latest.Tag != "", resForce.Payload.UpdateAvailable)
 		assert.NotEqual(t, res.Payload.LastCheck, resForce.Payload.LastCheck)
 	})
 }