From b8a3eed8e17380ce15b1889cc7719c5cbaa955df Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Mon, 23 Jan 2023 16:09:29 +0700 Subject: [PATCH 01/11] Switch from Travis to GitHub Actions --- .github/workflows/test.yml | 41 ++++++++++++++++++++++++++++++++ .travis.yml | 48 -------------------------------------- README.md | 2 +- 3 files changed, 42 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..2387ac9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,41 @@ +name: Test + +on: + push: + branches: + - "**" + pull_request: + branches: + - main + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + Test: + strategy: + matrix: + pg_branch: [master, REL_11_STABLE, REL_12_STABLE, REL_13_STABLE, REL_14_STABLE] + test_case: [tap, all] + mode: [none, legacy, paranoia] + exclude: + - test_case: tap + mode: paranoia + - test_case: all + mode: none + - test_case: all + mode: legacy + continue-on-error: true + runs-on: + - ubuntu-latest + env: + PG_BRANCH: ${{ matrix.pg_branch }} + TEST_CASE: ${{ matrix.test_case }} + steps: + - uses: actions/checkout@v2 + - name: Build the Docker image + run: | + ./make_dockerfile.sh + docker-compose build + - name: Test + run: docker-compose run $(bash <(curl -s https://codecov.io/env)) tests + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b6bd63a..0000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -os: linux - -dist: bionic - -language: c - -services: - - docker - -install: - - ./make_dockerfile.sh - - docker-compose build - -script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - -notifications: - email: - on_success: change - on_failure: always - -# keep in sync with codecov.yml number of builds -env: - - PG_BRANCH=master TEST_CASE=tap - - PG_BRANCH=master TEST_CASE=tap MODE=legacy -# - PG_BRANCH=master TEST_CASE=all - - PG_BRANCH=master TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_14_STABLE TEST_CASE=tap - - PG_BRANCH=REL_14_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_14_STABLE TEST_CASE=all - - PG_BRANCH=REL_14_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_13_STABLE TEST_CASE=tap - - PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_13_STABLE TEST_CASE=all - - PG_BRANCH=REL_13_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_12_STABLE TEST_CASE=tap - - PG_BRANCH=REL_12_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_12_STABLE TEST_CASE=all - - PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_11_STABLE TEST_CASE=tap - - PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_11_STABLE TEST_CASE=all - - PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia - -jobs: - allow_failures: - - if: env(PG_BRANCH) = master - diff --git a/README.md b/README.md index ece43af..02fd2f6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.com/postgrespro/ptrack.svg?branch=master)](https://travis-ci.com/postgrespro/ptrack) +![example workflow](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg) [![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) From 7bb1c9dc5ee6c15f601db88aff401cfc2a061d23 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 24 Jan 2023 11:14:30 +0700 Subject: [PATCH 02/11] Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02fd2f6..79af797 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![example workflow](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg) +![test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg) [![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) From 3a416565debc58e096b24de54cbb4d3871108213 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 24 Jan 2023 11:23:33 +0700 Subject: [PATCH 03/11] Continue on error only for experimental jobs --- .github/workflows/test.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2387ac9..1bf8fbe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,23 @@ jobs: Test: strategy: matrix: - pg_branch: [master, REL_11_STABLE, REL_12_STABLE, REL_13_STABLE, REL_14_STABLE] + pg_branch: [REL_11_STABLE, REL_12_STABLE, REL_13_STABLE, REL_14_STABLE] test_case: [tap, all] mode: [none, legacy, paranoia] + experimental: [false] + include: + - pg_branch: master + test_case: tap + mode: none + experimental: true + - pg_branch: master + test_case: tap + mode: legacy + experimental: true + - pg_branch: master + test_case: all + mode: paranoia + experimental: true exclude: - test_case: tap mode: paranoia @@ -24,7 +38,7 @@ jobs: mode: none - test_case: all mode: legacy - continue-on-error: true + continue-on-error: ${{ matrix.experimental }} runs-on: - ubuntu-latest env: From 1577a6d7e3833d3a175d701d18a16de3dc927966 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Tue, 24 Jan 2023 15:28:14 +0700 Subject: [PATCH 04/11] Don't terminate other parallel jobs if some fail --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1bf8fbe..e4aeedb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,6 +38,7 @@ jobs: mode: none - test_case: all mode: legacy + fail-fast: false continue-on-error: ${{ matrix.experimental }} runs-on: - ubuntu-latest From e854bf1f36748e4a24dd746c7b043d4d281309d6 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Thu, 26 Jan 2023 11:21:23 +0700 Subject: [PATCH 05/11] Link the status badge to the workflow results --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 79af797..f9eff7d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg) +[![Test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg)](https://github.com/postgrespro/ptrack/actions/workflows/test.yml) [![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) From ad21a7129b264416f05a5ffb726bf6655a021049 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 18 Aug 2023 09:27:25 +0700 Subject: [PATCH 06/11] Refactor the test workflow --- .github/workflows/test.yml | 116 +++++++++++++++++++++++---------- Dockerfile.in | 25 -------- Makefile | 43 ++++++++++++- README.md | 66 ++++++++++++++++--- codecov.yml | 5 +- docker-compose.yml | 17 ----- make_dockerfile.sh | 33 ---------- run_tests.sh | 127 ------------------------------------- 8 files changed, 183 insertions(+), 249 deletions(-) delete mode 100644 Dockerfile.in delete mode 100644 docker-compose.yml delete mode 100755 make_dockerfile.sh delete mode 100755 run_tests.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4aeedb..14756f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,43 +14,89 @@ jobs: Test: strategy: matrix: - pg_branch: [REL_11_STABLE, REL_12_STABLE, REL_13_STABLE, REL_14_STABLE] - test_case: [tap, all] - mode: [none, legacy, paranoia] - experimental: [false] - include: - - pg_branch: master - test_case: tap - mode: none - experimental: true - - pg_branch: master - test_case: tap - mode: legacy - experimental: true - - pg_branch: master - test_case: all - mode: paranoia - experimental: true + # pg_version: [15] + pg_version: [11, 12, 13, 14, 15] + os: [ubuntu-22.04] + # tests: [tap] + tests: [tap, python] + # test_mode: [normal, legacy, paranoia] + test_mode: [normal, paranoia] exclude: - - test_case: tap - mode: paranoia - - test_case: all - mode: none - - test_case: all - mode: legacy + - tests: tap + test_mode: paranoia + - tests: python + test_mode: normal + - tests: python + test_mode: legacy fail-fast: false - continue-on-error: ${{ matrix.experimental }} - runs-on: - - ubuntu-latest - env: - PG_BRANCH: ${{ matrix.pg_branch }} - TEST_CASE: ${{ matrix.test_case }} + name: ${{ format('Ptrack ({0}, PostgreSQL {1}, {2} tests, {3} mode)', matrix.os, matrix.pg_version, matrix.tests, matrix.test_mode) }} + container: + image: ${{ format('ghcr.io/postgres-dev/{0}:1.0', matrix.os) }} + env: + PG_BRANCH: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} + PREFIX: /home/dev/pgsql + LD_LIBRARY_PATH: /home/dev/pgsql/lib + PGDATA: /home/dev/data + TEST_MODE: ${{ matrix.test_mode }} + options: --privileged steps: - - uses: actions/checkout@v2 - - name: Build the Docker image + - name: Get Postgres sources + uses: actions/checkout@v3 + with: + repository: postgres/postgres + ref: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} + path: postgres + - name: Get Ptrack sources + uses: actions/checkout@v3 + with: + path: ptrack + - name: Get Pg_probackup sources + uses: actions/checkout@v3 + with: + repository: postgrespro/pg_probackup + path: pg_probackup + - name: Apply ptrack patches + run: make patch top_builddir=../postgres + working-directory: ptrack + - name: Install Postgres run: | - ./make_dockerfile.sh - docker-compose build + if [ $TEST_MODE = legacy ]; then + ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet + else + ./configure --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests + fi && + make -sj $(nproc) install && + make -C contrib -sj $(nproc) install && + echo $PREFIX/bin >> $GITHUB_PATH + working-directory: postgres + - name: Install Ptrack + run: make install USE_PGXS=1 PG_CPPFLAGS=-coverage SHLIB_LINK=-coverage + working-directory: ptrack + - name: Install Pg_probackup + run: make install-pg-probackup USE_PGXS=1 top_srcdir=../postgres + working-directory: ptrack + shell: bash {0} + - name: Install additional packages + run: | + apt update && + apt install -y python3-pip python3-six python3-pytest python3-pytest-xdist curl && + pip3 install --no-input testgres + # All steps have been so far executed by root but ptrack tests run from an + # unprivileged user so change some permissions + - name: Adjust the permissions of ptrack test folders + run: | + mkdir pg_probackup/tests/tmp_dirs + chown -R "dev:" pg_probackup ptrack - name: Test - run: docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - + run: make test-${{ matrix.tests }} USE_PGXS=1 + working-directory: ptrack + shell: runuser dev {0} + - name: Collect coverage results + run: make coverage + working-directory: ptrack + shell: runuser dev {0} + - name: Upload coverage results to Codecov + uses: codecov/codecov-action@v3 + with: + working-directory: ptrack + runs-on: ubuntu-latest diff --git a/Dockerfile.in b/Dockerfile.in deleted file mode 100644 index c2b0ffd..0000000 --- a/Dockerfile.in +++ /dev/null @@ -1,25 +0,0 @@ -FROM ololobus/postgres-dev:stretch - -USER root -RUN apt-get update -RUN apt-get -yq install python python-pip python-virtualenv - -# Environment -ENV PG_BRANCH=${PG_BRANCH} -ENV LANG=C.UTF-8 PGHOME=/testdir/pgbin -ENV MODE=${MODE} TEST_CASE=${TEST_CASE} TEST_REPEATS=${TEST_REPEATS} - -# Make directories -RUN mkdir -p /testdir - -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - -COPY . /testdir/ptrack -WORKDIR /testdir - -# Grant privileges -RUN chown -R postgres:postgres /testdir - -USER postgres -ENTRYPOINT /run.sh diff --git a/Makefile b/Makefile index e3d25a4..ce68344 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ + # contrib/ptrack/Makefile MODULE_big = ptrack @@ -20,8 +21,48 @@ PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else -subdir = contrib/ptrack top_builddir = ../.. +# Makefile.global is a build artifact and initially may not be available +ifneq ($(wildcard $(top_builddir)/src/Makefile.global), ) include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif +endif + +# Assuming make is started in the ptrack directory +patch: + @cd $(top_builddir) && \ + echo Applying the ptrack patch... && \ + git apply --verbose --3way $(CURDIR)/patches/${PG_BRANCH}-ptrack-core.diff +ifeq ($(MODE), paranoia) + @echo Applying turn-off-hint-bits.diff for the paranoia mode... && \ + git apply --verbose --3way $(CURDIR)/patches/turn-off-hint-bits.diff +endif + +TEST_MODE ?= normal +test-tap: +ifeq ($(TEST_MODE), legacy) + setarch x86_64 --addr-no-randomize $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) +else + $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) +endif + +pg_probackup_dir ?= $(CURDIR)/../pg_probackup +# Pg_probackup's Makefile uses top_srcdir when building via PGXS so set it when calling this target +# At the moment building pg_probackup with multiple threads may run some jobs too early and end with an error so do not set the -j option +install-pg-probackup: + $(MAKE) -C $(pg_probackup_dir) install USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) top_srcdir=$(top_srcdir) + +PYTEST_PROCESSES ?= $(shell nproc) +test-python: + cd $(pg_probackup_dir); \ + env="PG_PROBACKUP_PTRACK=ON PG_CONFIG=$(PG_CONFIG)"; \ + if [ "$(TEST_MODE)" = normal ]; then \ + env="$$env PG_PROBACKUP_TEST_BASIC=ON"; \ + elif [ "$(TEST_MODE)" = paranoia ]; then \ + env="$$env PG_PROBACKUP_PARANOIA=ON"; \ + fi; \ + env $$env python3 -m pytest -svv -n $(PYTEST_PROCESSES)$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py + +coverage: + gcov *.c *.h diff --git a/README.md b/README.md index f9eff7d..f1deb82 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg)](https://github.com/postgrespro/ptrack/actions/workflows/test.yml) -[![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) +[![Codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) # ptrack @@ -12,7 +12,7 @@ It is designed to allow false positives (i.e. block/page is marked in the `ptrac Currently, `ptrack` codebase is split between small PostgreSQL core patch and extension. All public SQL API methods and main engine are placed in the `ptrack` extension, while the core patch contains only certain hooks and modifies binary utilities to ignore `ptrack.map.*` files. -This extension is compatible with PostgreSQL [11](https://github.com/postgrespro/ptrack/blob/master/patches/REL_11_STABLE-ptrack-core.diff), [12](https://github.com/postgrespro/ptrack/blob/master/patches/REL_12_STABLE-ptrack-core.diff), [13](https://github.com/postgrespro/ptrack/blob/master/patches/REL_13_STABLE-ptrack-core.diff), [14](https://github.com/postgrespro/ptrack/blob/master/patches/REL_14_STABLE-ptrack-core.diff). +This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff), [16](patches/REL_16_STABLE-ptrack-core.diff). ## Installation @@ -162,20 +162,66 @@ Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare) ### Tests -Everything is tested automatically with [travis-ci.com](https://travis-ci.com/postgrespro/ptrack) and [codecov.io](https://codecov.io/gh/postgrespro/ptrack), but you can also run tests locally via `Docker`: +All changes of the source code in this repository are checked by CI - see commit statuses and the project status badge. You can also run tests locally by executing a few Makefile targets. + +#### Prerequisites + +To run Python tests inbstall the following packages: + +OS packages: + - python3-pip + - python3-six + - python3-pytest + - python3-pytest-xdist + +PIP packages: + - testgres + +For example, for Ubuntu: + +```sh +apt update +apt install python3-pip python3-six python3-pytest python3-pytest-xdist +pip3 install testgres +``` + +#### Testing ```sh -export PG_BRANCH=REL_14_STABLE -export TEST_CASE=all -export MODE=paranoia +export PG_BRANCH=REL_15_STABLE +export PREFIX=/path/to/pgsql +export PATH=$PREFIX/bin:$PATH + +cd /path/to/ptrack +make patch top_builddir=/path/to/postgres + +cd /path/to/postgres +./configure --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests +make -sj `nproc` install +make -C contrib -sj `nproc` install + +export USE_PGXS=1 -./make_dockerfile.sh +cd /path/to/ptrack +make install +make install-pg-probackup top_srcdir=/path/to/postgres +make test-tap +make test-python +``` + +You can use a public Docker image which already has the necessary build environment (but not the testing prerequisites): -docker-compose build -docker-compose run tests +```sh +docker run -e USER_ID=`id -u` -it -v $PWD:/work --name=ptrack ghcr.io/postgres-dev/ubuntu-22.04:1.0 +dev@a033797d2f73:~$ ``` -Available test modes (`MODE`) are `basic` (default) and `paranoia` (per-block checksum comparison of `PGDATA` content before and after backup-restore process). Available test cases (`TEST_CASE`) are `tap` (minimalistic PostgreSQL [tap test](https://github.com/postgrespro/ptrack/blob/master/t/001_basic.pl)), `all` or any specific [pg_probackup test](https://github.com/postgrespro/pg_probackup/blob/master/tests/ptrack.py), e.g. `test_ptrack_simple`. +You can control how tests are executed by using the following environment (or `make`) variables: + +| Variable | Possible values | Required | Default value | Description | +| - | - | - | - | - | +| TESTS | A Pytest filter expression | No | Not set (run all Python tests) | A filter to include only selected tests into the run. See the Pytest `-k` option for more information. This variable is only applicable to `test-python` for the tests located in [tests](https://github.com/postgrespro/pg_probackup/tree/master/tests). | +| TEST_MODE | normal, legacy, paranoia | No | normal | The "legacy" mode runs tests in an environment similar to a 32-bit Windows system. This mode is only applicable to `test-tap`. The "paranoia" mode compares the checksums of each block of the database catalog (PGDATA) contents before making a backuo and after the restoration. This mode is only applicable to `test-python`.| ### TODO diff --git a/codecov.yml b/codecov.yml index fe3b308..00b744e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,9 @@ codecov: notify: - after_n_builds: 12 # keep in sync with .travis.yml number of builds + # must be equal to the total number of parallel jobs in a CI pipeline + # (Postgres versions x test types x test modes x OSes minus excluded + # combinations) + after_n_builds: 10 # datapagemap.c/.h are copied from Postgres, so let's remove it # from report. Otherwise, we would have to remove some currently diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index fc65455..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.7" -services: - tests: - build: - context: . - - cap_add: - - SYS_PTRACE - - security_opt: - - seccomp=unconfined - - # don't work - #sysctls: - # kernel.yama.ptrace_scope: 0 - privileged: true - diff --git a/make_dockerfile.sh b/make_dockerfile.sh deleted file mode 100755 index 409a5b9..0000000 --- a/make_dockerfile.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env sh - -if [ -z ${PG_BRANCH+x} ]; then - echo PG_BRANCH is not set! - exit 1 -fi - -if [ -z ${MODE+x} ]; then - MODE=basic -else - echo MODE=${MODE} -fi - -if [ -z ${TEST_CASE+x} ]; then - TEST_CASE=all -else - echo TEST_CASE=${TEST_CASE} -fi - -if [ -z ${TEST_REPEATS+x} ]; then - TEST_REPEATS=1 -else - echo TEST_REPEATS=${TEST_REPEATS} -fi - -echo PG_BRANCH=${PG_BRANCH} - -sed \ - -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ - -e 's/${MODE}/'${MODE}/g \ - -e 's/${TEST_CASE}/'${TEST_CASE}/g \ - -e 's/${TEST_REPEATS}/'${TEST_REPEATS}/g \ -Dockerfile.in > Dockerfile diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index 1b4a693..0000000 --- a/run_tests.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright (c) 2019-2021, Postgres Professional -# - -PTRACK_SRC=${PWD}/ptrack -PG_SRC=${PWD}/postgres -PBK_SRC=${PWD}/pg_probackup -status=0 - -######################################################### -# Clone Postgres -echo "############### Getting Postgres sources" -git clone https://github.com/postgres/postgres.git --depth=1 --branch=${PG_BRANCH} ${PG_SRC} - -# Clone pg_probackup -echo "############### Getting pg_probackup sources" -git clone https://github.com/postgrespro/pg_probackup.git --depth=1 --branch=master ${PBK_SRC} - -######################################################### -# Compile and install Postgres -cd ${PG_SRC} # Go to postgres dir - -echo "############### Applying ptrack patch" -git apply --verbose --3way ${PTRACK_SRC}/patches/${PG_BRANCH}-ptrack-core.diff - -if [ "${MODE}" = "paranoia" ]; then - echo "############### Paranoia mode: applying turn-off-hint-bits.diff" - git apply --verbose --3way ${PTRACK_SRC}/patches/turn-off-hint-bits.diff -fi - -echo "############### Compiling Postgres" -if [ "${TEST_CASE}" = "tap" ] && [ "${MODE}" = "legacy" ]; then - ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet -else - ./configure --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet -fi -make --quiet --jobs=$(nproc) install -make --quiet --jobs=$(nproc) --directory=contrib/ install - -# Override default Postgres instance -export PATH=${PGHOME}/bin:${PATH} -export LD_LIBRARY_PATH=${PGHOME}/lib -export PG_CONFIG=$(which pg_config) - -# Show pg_config path (just in case) -echo "############### pg_config path" -which pg_config - -# Show pg_config just in case -echo "############### pg_config" -pg_config - -######################################################### -# Build and install ptrack extension -echo "############### Compiling and installing ptrack extension" -cp --recursive ${PTRACK_SRC} ${PG_SRC}/contrib/ptrack -make USE_PGXS=1 --directory=${PG_SRC}/contrib/ptrack/ clean -make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" --directory=${PG_SRC}/contrib/ptrack/ install - -if [ "${TEST_CASE}" = "tap" ]; then - - # Run tap tests - echo "############### Running tap tests" - if [ "${MODE}" = "legacy" ]; then - # There is a known issue with attaching shared memory segment using the same - # address each time, when EXEC_BACKEND mechanism is turned on. It happens due - # to the ASLR address space randomization, so we are trying to attach a segment - # to the already occupied location. That way we simply turning off ASLR here. - # - # Postgres comment: https://github.com/postgres/postgres/blob/5cbfce562f7cd2aab0cdc4694ce298ec3567930e/src/backend/postmaster/postmaster.c#L4929 - setarch x86_64 --addr-no-randomize make --directory=${PG_SRC}/contrib/ptrack check || status=$? - else - make --directory=${PG_SRC}/contrib/ptrack check || status=$? - fi - -else - # Set kernel params (used for debugging -- probackup tests) - echo "############### setting kernel params" - sudo sh -c 'echo 0 > /proc/sys/kernel/yama/ptrace_scope' - - # Build and install pg_probackup - echo "############### Compiling and installing pg_probackup" - cd ${PBK_SRC} # Go to pg_probackup dir - make USE_PGXS=1 top_srcdir=${PG_SRC} install - - # Setup python environment - echo "############### Setting up python env" - virtualenv --python=/usr/bin/python3 pyenv - source pyenv/bin/activate - pip install testgres==1.8.2 - - echo "############### Testing" - export PG_PROBACKUP_PTRACK=ON - if [ "${MODE}" = "basic" ]; then - export PG_PROBACKUP_TEST_BASIC=ON - elif [ "${MODE}" = "paranoia" ]; then - export PG_PROBACKUP_PARANOIA=ON - fi - - if [ "${TEST_CASE}" = "all" ]; then - # Run all pg_probackup ptrack tests - PBK_TEST_CASE=tests.ptrack - else - PBK_TEST_CASE=tests.ptrack.PtrackTest.${TEST_CASE} - fi - for i in `seq ${TEST_REPEATS}`; do - python3 -m unittest -v ${PBK_TEST_CASE} || status=$? - done - - # Exit virtualenv - deactivate -fi - -######################################################### -# codecov -echo "############### Codecov" -cd ${PTRACK_SRC} -# Generate *.gcov files -gcov ${PG_SRC}/contrib/ptrack/*.c ${PG_SRC}/contrib/ptrack/*.h - -# Send coverage stats to Codecov -bash <(curl -s https://codecov.io/bash) - -# Something went wrong, exit with code 1 -if [ ${status} -ne 0 ]; then exit 1; fi From 182e96a5093bb7e06f6181e445b27911f0bacd33 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 18 Aug 2023 09:52:23 +0700 Subject: [PATCH 07/11] [no ci] Remove the mentioning of the 16th version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1deb82..b4854d3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It is designed to allow false positives (i.e. block/page is marked in the `ptrac Currently, `ptrack` codebase is split between small PostgreSQL core patch and extension. All public SQL API methods and main engine are placed in the `ptrack` extension, while the core patch contains only certain hooks and modifies binary utilities to ignore `ptrack.map.*` files. -This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff), [16](patches/REL_16_STABLE-ptrack-core.diff). +This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff). ## Installation From a7f17a0befd60057924f74ecb1b4803748d75001 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 18 Aug 2023 14:50:29 +0700 Subject: [PATCH 08/11] Install Postgres from the ptrack Makefile --- .github/workflows/test.yml | 16 ++++------------ Makefile | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14756f0..f05773a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,9 +34,7 @@ jobs: image: ${{ format('ghcr.io/postgres-dev/{0}:1.0', matrix.os) }} env: PG_BRANCH: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} - PREFIX: /home/dev/pgsql - LD_LIBRARY_PATH: /home/dev/pgsql/lib - PGDATA: /home/dev/data + PGDATA: $HOME/data TEST_MODE: ${{ matrix.test_mode }} options: --privileged steps: @@ -60,15 +58,9 @@ jobs: working-directory: ptrack - name: Install Postgres run: | - if [ $TEST_MODE = legacy ]; then - ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet - else - ./configure --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests - fi && - make -sj $(nproc) install && - make -C contrib -sj $(nproc) install && - echo $PREFIX/bin >> $GITHUB_PATH - working-directory: postgres + make install-postgres top_builddir=$GITHUB_WORKSPACE/postgres prefix=$HOME/pgsql && + echo $HOME/pgsql/bin >> $GITHUB_PATH + working-directory: ptrack - name: Install Ptrack run: make install USE_PGXS=1 PG_CPPFLAGS=-coverage SHLIB_LINK=-coverage working-directory: ptrack diff --git a/Makefile b/Makefile index ce68344..33b06b0 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,24 @@ ifeq ($(MODE), paranoia) git apply --verbose --3way $(CURDIR)/patches/turn-off-hint-bits.diff endif +nproc := $(shell nproc) +prefix ?= $(abspath $(top_builddir)/pgsql) TEST_MODE ?= normal +# Postgres Makefile skips some targets depending on the MAKELEVEL variable so +# reset it when calling install targets as if they are started directly from the +# command line +install-postgres: + @cd $(top_builddir) && \ + if [ "$(TEST_MODE)" = legacy ]; then \ + ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet; \ + else \ + ./configure --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests; \ + fi && \ + $(MAKE) -sj $(nproc) install MAKELEVEL=0 && \ + $(MAKE) -sj $(nproc) -C contrib/ install MAKELEVEL=0 + +# Now when Postgres is built call all remainig targets with USE_PGXS=1 + test-tap: ifeq ($(TEST_MODE), legacy) setarch x86_64 --addr-no-randomize $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) From daae1656e0c4c3bca0ca9b3092d2ba9dc8811948 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Tue, 22 Aug 2023 15:34:06 +0700 Subject: [PATCH 09/11] Run Python tests in parallel only if pytest-xdist is available --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 33b06b0..9f3d62f 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test-python: elif [ "$(TEST_MODE)" = paranoia ]; then \ env="$$env PG_PROBACKUP_PARANOIA=ON"; \ fi; \ - env $$env python3 -m pytest -svv -n $(PYTEST_PROCESSES)$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py + env $$env python3 -m pytest -svv$(if $(shell python3 -m pytest --help | grep '\-n '), -n $(PYTEST_PROCESSES))$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py coverage: gcov *.c *.h From 2eaf70d1a5188ea219a52fcd92eddeac02a13f91 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Wed, 23 Aug 2023 09:03:12 +0700 Subject: [PATCH 10/11] Use the default pg_config location --- Makefile | 20 +++++------ README.md | 106 +++++++++++++++++++++++++++++++----------------------- 2 files changed, 71 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 9f3d62f..499067a 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,9 @@ TAP_TESTS = 1 # with Mkvcbuild.pm on PGv15+ PG_LIBS_INTERNAL += $(libpq_pgport) -ifdef USE_PGXS PG_CONFIG ?= pg_config + +ifdef USE_PGXS PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else @@ -33,14 +34,14 @@ endif patch: @cd $(top_builddir) && \ echo Applying the ptrack patch... && \ - git apply --verbose --3way $(CURDIR)/patches/${PG_BRANCH}-ptrack-core.diff + git apply --3way -v $(CURDIR)/patches/${PG_BRANCH}-ptrack-core.diff ifeq ($(MODE), paranoia) @echo Applying turn-off-hint-bits.diff for the paranoia mode... && \ - git apply --verbose --3way $(CURDIR)/patches/turn-off-hint-bits.diff + git apply --3way -v $(CURDIR)/patches/turn-off-hint-bits.diff endif -nproc := $(shell nproc) -prefix ?= $(abspath $(top_builddir)/pgsql) +NPROC ?= $(shell nproc) +prefix := $(abspath $(top_builddir)/pgsql) TEST_MODE ?= normal # Postgres Makefile skips some targets depending on the MAKELEVEL variable so # reset it when calling install targets as if they are started directly from the @@ -52,8 +53,8 @@ install-postgres: else \ ./configure --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests; \ fi && \ - $(MAKE) -sj $(nproc) install MAKELEVEL=0 && \ - $(MAKE) -sj $(nproc) -C contrib/ install MAKELEVEL=0 + $(MAKE) -sj $(NPROC) install MAKELEVEL=0 && \ + $(MAKE) -sj $(NPROC) -C contrib/ install MAKELEVEL=0 # Now when Postgres is built call all remainig targets with USE_PGXS=1 @@ -64,13 +65,12 @@ else $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) endif -pg_probackup_dir ?= $(CURDIR)/../pg_probackup +pg_probackup_dir = ../pg_probackup # Pg_probackup's Makefile uses top_srcdir when building via PGXS so set it when calling this target # At the moment building pg_probackup with multiple threads may run some jobs too early and end with an error so do not set the -j option install-pg-probackup: $(MAKE) -C $(pg_probackup_dir) install USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) top_srcdir=$(top_srcdir) -PYTEST_PROCESSES ?= $(shell nproc) test-python: cd $(pg_probackup_dir); \ env="PG_PROBACKUP_PTRACK=ON PG_CONFIG=$(PG_CONFIG)"; \ @@ -79,7 +79,7 @@ test-python: elif [ "$(TEST_MODE)" = paranoia ]; then \ env="$$env PG_PROBACKUP_PARANOIA=ON"; \ fi; \ - env $$env python3 -m pytest -svv$(if $(shell python3 -m pytest --help | grep '\-n '), -n $(PYTEST_PROCESSES))$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py + env $$env python3 -m pytest -svv$(if $(shell python3 -m pytest --help | grep '\-n '), -n $(NPROC))$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py coverage: gcov *.c *.h diff --git a/README.md b/README.md index b4854d3..92d922b 100644 --- a/README.md +++ b/README.md @@ -16,40 +16,62 @@ This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-c ## Installation -1) Get latest `ptrack` sources: +1) Specify the PostgreSQL branch to work with: ```shell -git clone https://github.com/postgrespro/ptrack.git +export PG_BRANCH=REL_15_STABLE +``` + +2) Get the latest PostgreSQL sources: + +```shell +git clone https://github.com/postgres/postgres.git -b $PG_BRANCH +``` + +3) Get the latest `ptrack` sources: + +```shell +git clone https://github.com/postgrespro/ptrack.git postgres/contrib/ptrack ``` -2) Get latest PostgreSQL sources: +4) Change to the `ptrack` directory: ```shell -git clone https://github.com/postgres/postgres.git -b REL_14_STABLE && cd postgres +cd postgres/contrib/ptrack ``` -3) Apply PostgreSQL core patch: +5) Apply the PostgreSQL core patch: ```shell -git apply -3 ../ptrack/patches/REL_14_STABLE-ptrack-core.diff +make patch ``` -4) Compile and install PostgreSQL +6) Compile and install PostgreSQL: -5) Set `ptrack.map_size` (in MB) +```shell +make install-postgres prefix=$PWD/pgsql # or some other prefix of your choice +``` + +7) Add the newly created binaries to the PATH: ```shell -echo "shared_preload_libraries = 'ptrack'" >> postgres_data/postgresql.conf -echo "ptrack.map_size = 64" >> postgres_data/postgresql.conf +export PATH=$PWD/pgsql/bin:$PATH ``` -6) Compile and install `ptrack` extension +8) Compile and install `ptrack`: ```shell -USE_PGXS=1 make -C /path/to/ptrack/ install +make install USE_PGXS=1 ``` -7) Run PostgreSQL and create `ptrack` extension +9) Set `ptrack.map_size` (in MB): + +```shell +echo "shared_preload_libraries = 'ptrack'" >> /postgresql.conf +echo "ptrack.map_size = 64" >> /postgresql.conf +``` + +10) Run PostgreSQL and create the `ptrack` extension: ```sql postgres=# CREATE EXTENSION ptrack; @@ -158,13 +180,13 @@ To gather the whole changeset of modified blocks in `ptrack_get_pagemapset()` we ## Contribution -Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare), [fill up issues](https://github.com/postgrespro/ptrack/issues/new), or just reach one of us directly (e.g. <[Alexey Kondratov](mailto:a.kondratov@postgrespro.ru?subject=[GitHub]%20Ptrack), [@ololobus](https://github.com/ololobus)>) if you are interested in `ptrack`. +Feel free to [send a pull request](https://github.com/postgrespro/ptrack/compare), [create an issue](https://github.com/postgrespro/ptrack/issues/new) or [reach us by e-mail](mailto:team-wd40@lists.postgrespro.ru??subject=[GitHub]%20Ptrack) if you are interested in `ptrack`. -### Tests +## Tests All changes of the source code in this repository are checked by CI - see commit statuses and the project status badge. You can also run tests locally by executing a few Makefile targets. -#### Prerequisites +### Prerequisites To run Python tests inbstall the following packages: @@ -179,49 +201,43 @@ PIP packages: For example, for Ubuntu: -```sh -apt update -apt install python3-pip python3-six python3-pytest python3-pytest-xdist -pip3 install testgres +```shell +sudo apt update +sudo apt install python3-pip python3-six python3-pytest python3-pytest-xdist +sudo pip3 install testgres ``` -#### Testing - -```sh -export PG_BRANCH=REL_15_STABLE -export PREFIX=/path/to/pgsql -export PATH=$PREFIX/bin:$PATH - -cd /path/to/ptrack -make patch top_builddir=/path/to/postgres - -cd /path/to/postgres -./configure --prefix=$PREFIX --enable-debug --enable-cassert --enable-depend --enable-tap-tests -make -sj `nproc` install -make -C contrib -sj `nproc` install +### Testing -export USE_PGXS=1 - -cd /path/to/ptrack -make install -make install-pg-probackup top_srcdir=/path/to/postgres -make test-tap +Install PostgreSQL and ptrack as described in [Installation](#installation), install the testing prerequisites, then do (assuming the current directory is `ptrack`): +```shell +git clone https://github.com/postgrespro/pg_probackup.git ../pg_probackup # clone the repository into postgres/contrib/pg_probackup +# remember to export PATH=/path/to/pgsql/bin:$PATH +make install-pg-probackup USE_PGXS=1 top_srcdir=../.. +make test-tap USE_PGXS=1 make test-python ``` +If `pg_probackup` is not located in `postgres/contrib` then additionally specify the path to the `pg_probackup` directory when building `pg_probackup`: +```shell +make install-pg-probackup USE_PGXS=1 top_srcdir=/path/to/postgres pg_probackup_dir=/path/to/pg_probackup +``` + You can use a public Docker image which already has the necessary build environment (but not the testing prerequisites): -```sh +```shell docker run -e USER_ID=`id -u` -it -v $PWD:/work --name=ptrack ghcr.io/postgres-dev/ubuntu-22.04:1.0 dev@a033797d2f73:~$ ``` -You can control how tests are executed by using the following environment (or `make`) variables: +## Environment variables -| Variable | Possible values | Required | Default value | Description | -| - | - | - | - | - | +| Variable | Possible values | Required | Default value | Description | +| - | - | - | - | - | +| NPROC | An integer greater than 0 | No | Output of `nproc` | The number of threads used for building and running tests | +| PG_CONFIG | File path | No | pg_config (from the PATH) | The path to the `pg_config` binary | | TESTS | A Pytest filter expression | No | Not set (run all Python tests) | A filter to include only selected tests into the run. See the Pytest `-k` option for more information. This variable is only applicable to `test-python` for the tests located in [tests](https://github.com/postgrespro/pg_probackup/tree/master/tests). | -| TEST_MODE | normal, legacy, paranoia | No | normal | The "legacy" mode runs tests in an environment similar to a 32-bit Windows system. This mode is only applicable to `test-tap`. The "paranoia" mode compares the checksums of each block of the database catalog (PGDATA) contents before making a backuo and after the restoration. This mode is only applicable to `test-python`.| +| TEST_MODE | normal, legacy, paranoia | No | normal | The "legacy" mode runs tests in an environment similar to a 32-bit Windows system. This mode is only applicable to `test-tap`. The "paranoia" mode compares the checksums of each block of the database catalog (PGDATA) contents before making a backup and after the restoration. This mode is only applicable to `test-python`.| ### TODO From 44a437a0cae8b15bc02f147f6e21e6c0ea3b698a Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 28 Aug 2023 18:34:44 +0700 Subject: [PATCH 11/11] Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 92d922b..8d257ef 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ All changes of the source code in this repository are checked by CI - see commit ### Prerequisites -To run Python tests inbstall the following packages: +To run Python tests install the following packages: OS packages: - python3-pip