From 056c5d67473bdcd35aa74ec1cb8290e192d9c632 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Thu, 19 Oct 2023 11:52:51 +0300 Subject: [PATCH 1/3] Update the update section in Readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ece43af..4a3e4e3 100644 --- a/README.md +++ b/README.md @@ -102,13 +102,13 @@ postgres=# SELECT * FROM ptrack_get_change_stat('0/285C8C8'); ## Upgrading -Usually, you have to only install new version of `ptrack` and do `ALTER EXTENSION 'ptrack' UPDATE;`. However, some specific actions may be required as well: +Usually, you have to only install new version of `ptrack` and do `ALTER EXTENSION ptrack UPDATE;`. However, some specific actions may be required as well: #### Upgrading from 2.0.0 to 2.1.*: * Put `shared_preload_libraries = 'ptrack'` into `postgresql.conf`. * Rename `ptrack_map_size` to `ptrack.map_size`. -* Do `ALTER EXTENSION 'ptrack' UPDATE;`. +* Do `ALTER EXTENSION ptrack UPDATE;`. * Restart your server. #### Upgrading from 2.1.* to 2.2.*: @@ -121,14 +121,14 @@ Since version 2.2 we use a different algorithm for tracking changed pages. Thus, * Update ptrack binaries * Remove global/ptrack.map.mmap if it exist in server data directory * Start server -* Do `ALTER EXTENSION 'ptrack' UPDATE;`. +* Do `ALTER EXTENSION ptrack UPDATE;`. #### Upgrading from 2.3.* to 2.4.*: * Stop your server * Update ptrack binaries * Start server -* Do `ALTER EXTENSION 'ptrack' UPDATE;`. +* Do `ALTER EXTENSION ptrack UPDATE;`. ## Limitations From 2c4157e2bf540456067aff9113f4b73ee5be40cb Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Fri, 20 Oct 2023 14:10:47 +0300 Subject: [PATCH 2/3] Update ce from master --- .github/workflows/test.yml | 94 +++++++++++++++++++++++++++ .travis.yml | 48 -------------- AUTHORS.md | 22 +++++++ Dockerfile.in | 25 -------- Makefile | 62 +++++++++++++++++- README.md | 116 +++++++++++++++++++++++++-------- codecov.yml | 5 +- docker-compose.yml | 17 ----- make_dockerfile.sh | 33 ---------- run_tests.sh | 127 ------------------------------------- 10 files changed, 269 insertions(+), 280 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml create mode 100644 AUTHORS.md delete mode 100644 Dockerfile.in delete mode 100644 docker-compose.yml delete mode 100755 make_dockerfile.sh delete mode 100755 run_tests.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f05773a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,94 @@ +name: Test + +on: + push: + branches: + - "**" + pull_request: + branches: + - main + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + Test: + strategy: + matrix: + # pg_version: [15] + pg_version: [11, 12, 13, 14, 15] + os: [ubuntu-22.04] + # tests: [tap] + tests: [tap, python] + # test_mode: [normal, legacy, paranoia] + test_mode: [normal, paranoia] + exclude: + - tests: tap + test_mode: paranoia + - tests: python + test_mode: normal + - tests: python + test_mode: legacy + fail-fast: false + name: ${{ format('Ptrack ({0}, PostgreSQL {1}, {2} tests, {3} mode)', matrix.os, matrix.pg_version, matrix.tests, matrix.test_mode) }} + container: + image: ${{ format('ghcr.io/postgres-dev/{0}:1.0', matrix.os) }} + env: + PG_BRANCH: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} + PGDATA: $HOME/data + TEST_MODE: ${{ matrix.test_mode }} + options: --privileged + steps: + - name: Get Postgres sources + uses: actions/checkout@v3 + with: + repository: postgres/postgres + ref: ${{ format('REL_{0}_STABLE', matrix.pg_version) }} + path: postgres + - name: Get Ptrack sources + uses: actions/checkout@v3 + with: + path: ptrack + - name: Get Pg_probackup sources + uses: actions/checkout@v3 + with: + repository: postgrespro/pg_probackup + path: pg_probackup + - name: Apply ptrack patches + run: make patch top_builddir=../postgres + working-directory: ptrack + - name: Install Postgres + run: | + make install-postgres top_builddir=$GITHUB_WORKSPACE/postgres prefix=$HOME/pgsql && + echo $HOME/pgsql/bin >> $GITHUB_PATH + working-directory: ptrack + - name: Install Ptrack + run: make install USE_PGXS=1 PG_CPPFLAGS=-coverage SHLIB_LINK=-coverage + working-directory: ptrack + - name: Install Pg_probackup + run: make install-pg-probackup USE_PGXS=1 top_srcdir=../postgres + working-directory: ptrack + shell: bash {0} + - name: Install additional packages + run: | + apt update && + apt install -y python3-pip python3-six python3-pytest python3-pytest-xdist curl && + pip3 install --no-input testgres + # All steps have been so far executed by root but ptrack tests run from an + # unprivileged user so change some permissions + - name: Adjust the permissions of ptrack test folders + run: | + mkdir pg_probackup/tests/tmp_dirs + chown -R "dev:" pg_probackup ptrack + - name: Test + run: make test-${{ matrix.tests }} USE_PGXS=1 + working-directory: ptrack + shell: runuser dev {0} + - name: Collect coverage results + run: make coverage + working-directory: ptrack + shell: runuser dev {0} + - name: Upload coverage results to Codecov + uses: codecov/codecov-action@v3 + with: + working-directory: ptrack + runs-on: ubuntu-latest diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b6bd63a..0000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -os: linux - -dist: bionic - -language: c - -services: - - docker - -install: - - ./make_dockerfile.sh - - docker-compose build - -script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - -notifications: - email: - on_success: change - on_failure: always - -# keep in sync with codecov.yml number of builds -env: - - PG_BRANCH=master TEST_CASE=tap - - PG_BRANCH=master TEST_CASE=tap MODE=legacy -# - PG_BRANCH=master TEST_CASE=all - - PG_BRANCH=master TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_14_STABLE TEST_CASE=tap - - PG_BRANCH=REL_14_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_14_STABLE TEST_CASE=all - - PG_BRANCH=REL_14_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_13_STABLE TEST_CASE=tap - - PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_13_STABLE TEST_CASE=all - - PG_BRANCH=REL_13_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_12_STABLE TEST_CASE=tap - - PG_BRANCH=REL_12_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_12_STABLE TEST_CASE=all - - PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia - - PG_BRANCH=REL_11_STABLE TEST_CASE=tap - - PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy -# - PG_BRANCH=REL_11_STABLE TEST_CASE=all - - PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia - -jobs: - allow_failures: - - if: env(PG_BRANCH) = master - diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..ed4d0eb --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,22 @@ +# Authors + +This list is sorted by the number of commits per contributor in _descending_ order. + +Avatar|Contributor|Contributions +:-:|---|:-: +@ololobus|[@ololobus](https://github.com/ololobus)|62 +@funny-falcon|[@funny-falcon](https://github.com/funny-falcon)|15 +@alubennikova|[@alubennikova](https://github.com/alubennikova)|9 +@kulaginm|[@kulaginm](https://github.com/kulaginm)|5 +@daniel-95|[@daniel-95](https://github.com/daniel-95)|4 +@ziva777|[@ziva777](https://github.com/ziva777)|2 +@vegebird|[@vegebird](https://github.com/vegebird)|2 +@kovdb75|[@kovdb75](https://github.com/kovdb75)|1 +@MarinaPolyakova|[@MarinaPolyakova](https://github.com/MarinaPolyakova)|1 +@rzharkov|[@rzharkov](https://github.com/rzharkov)|1 +@vbwagner|[@vbwagner](https://github.com/vbwagner)|1 +@waaeer|[@waaeer](https://github.com/waaeer)|1 + +--- + +Auto-generated by [gaocegege/maintainer](https://github.com/maintainer-org/maintainer) on 2023-08-03. diff --git a/Dockerfile.in b/Dockerfile.in deleted file mode 100644 index c2b0ffd..0000000 --- a/Dockerfile.in +++ /dev/null @@ -1,25 +0,0 @@ -FROM ololobus/postgres-dev:stretch - -USER root -RUN apt-get update -RUN apt-get -yq install python python-pip python-virtualenv - -# Environment -ENV PG_BRANCH=${PG_BRANCH} -ENV LANG=C.UTF-8 PGHOME=/testdir/pgbin -ENV MODE=${MODE} TEST_CASE=${TEST_CASE} TEST_REPEATS=${TEST_REPEATS} - -# Make directories -RUN mkdir -p /testdir - -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - -COPY . /testdir/ptrack -WORKDIR /testdir - -# Grant privileges -RUN chown -R postgres:postgres /testdir - -USER postgres -ENTRYPOINT /run.sh diff --git a/Makefile b/Makefile index e3d25a4..499067a 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ + # contrib/ptrack/Makefile MODULE_big = ptrack @@ -15,13 +16,70 @@ TAP_TESTS = 1 # with Mkvcbuild.pm on PGv15+ PG_LIBS_INTERNAL += $(libpq_pgport) -ifdef USE_PGXS PG_CONFIG ?= pg_config + +ifdef USE_PGXS PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else -subdir = contrib/ptrack top_builddir = ../.. +# Makefile.global is a build artifact and initially may not be available +ifneq ($(wildcard $(top_builddir)/src/Makefile.global), ) include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif +endif + +# Assuming make is started in the ptrack directory +patch: + @cd $(top_builddir) && \ + echo Applying the ptrack patch... && \ + git apply --3way -v $(CURDIR)/patches/${PG_BRANCH}-ptrack-core.diff +ifeq ($(MODE), paranoia) + @echo Applying turn-off-hint-bits.diff for the paranoia mode... && \ + git apply --3way -v $(CURDIR)/patches/turn-off-hint-bits.diff +endif + +NPROC ?= $(shell nproc) +prefix := $(abspath $(top_builddir)/pgsql) +TEST_MODE ?= normal +# Postgres Makefile skips some targets depending on the MAKELEVEL variable so +# reset it when calling install targets as if they are started directly from the +# command line +install-postgres: + @cd $(top_builddir) && \ + if [ "$(TEST_MODE)" = legacy ]; then \ + ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet; \ + else \ + ./configure --prefix=$(prefix) --enable-debug --enable-cassert --enable-depend --enable-tap-tests; \ + fi && \ + $(MAKE) -sj $(NPROC) install MAKELEVEL=0 && \ + $(MAKE) -sj $(NPROC) -C contrib/ install MAKELEVEL=0 + +# Now when Postgres is built call all remainig targets with USE_PGXS=1 + +test-tap: +ifeq ($(TEST_MODE), legacy) + setarch x86_64 --addr-no-randomize $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) +else + $(MAKE) installcheck USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) +endif + +pg_probackup_dir = ../pg_probackup +# Pg_probackup's Makefile uses top_srcdir when building via PGXS so set it when calling this target +# At the moment building pg_probackup with multiple threads may run some jobs too early and end with an error so do not set the -j option +install-pg-probackup: + $(MAKE) -C $(pg_probackup_dir) install USE_PGXS=$(USE_PGXS) PG_CONFIG=$(PG_CONFIG) top_srcdir=$(top_srcdir) + +test-python: + cd $(pg_probackup_dir); \ + env="PG_PROBACKUP_PTRACK=ON PG_CONFIG=$(PG_CONFIG)"; \ + if [ "$(TEST_MODE)" = normal ]; then \ + env="$$env PG_PROBACKUP_TEST_BASIC=ON"; \ + elif [ "$(TEST_MODE)" = paranoia ]; then \ + env="$$env PG_PROBACKUP_PARANOIA=ON"; \ + fi; \ + env $$env python3 -m pytest -svv$(if $(shell python3 -m pytest --help | grep '\-n '), -n $(NPROC))$(if $(TESTS), -k '$(TESTS)') tests/ptrack_test.py + +coverage: + gcov *.c *.h diff --git a/README.md b/README.md index 4a3e4e3..1dd4a94 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Build Status](https://travis-ci.com/postgrespro/ptrack.svg?branch=master)](https://travis-ci.com/postgrespro/ptrack) -[![codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) +[![Test](https://github.com/postgrespro/ptrack/actions/workflows/test.yml/badge.svg)](https://github.com/postgrespro/ptrack/actions/workflows/test.yml) +[![Codecov](https://codecov.io/gh/postgrespro/ptrack/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/ptrack) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/ptrack?include_prereleases)](https://github.com/postgrespro/ptrack/releases/latest) # ptrack @@ -12,44 +12,66 @@ It is designed to allow false positives (i.e. block/page is marked in the `ptrac Currently, `ptrack` codebase is split between small PostgreSQL core patch and extension. All public SQL API methods and main engine are placed in the `ptrack` extension, while the core patch contains only certain hooks and modifies binary utilities to ignore `ptrack.map.*` files. -This extension is compatible with PostgreSQL [11](https://github.com/postgrespro/ptrack/blob/master/patches/REL_11_STABLE-ptrack-core.diff), [12](https://github.com/postgrespro/ptrack/blob/master/patches/REL_12_STABLE-ptrack-core.diff), [13](https://github.com/postgrespro/ptrack/blob/master/patches/REL_13_STABLE-ptrack-core.diff), [14](https://github.com/postgrespro/ptrack/blob/master/patches/REL_14_STABLE-ptrack-core.diff). +This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff). ## Installation -1) Get latest `ptrack` sources: +1) Specify the PostgreSQL branch to work with: ```shell -git clone https://github.com/postgrespro/ptrack.git +export PG_BRANCH=REL_15_STABLE ``` -2) Get latest PostgreSQL sources: +2) Get the latest PostgreSQL sources: ```shell -git clone https://github.com/postgres/postgres.git -b REL_14_STABLE && cd postgres +git clone https://github.com/postgres/postgres.git -b $PG_BRANCH ``` -3) Apply PostgreSQL core patch: +3) Get the latest `ptrack` sources: ```shell -git apply -3 ../ptrack/patches/REL_14_STABLE-ptrack-core.diff +git clone https://github.com/postgrespro/ptrack.git postgres/contrib/ptrack ``` -4) Compile and install PostgreSQL +4) Change to the `ptrack` directory: -5) Set `ptrack.map_size` (in MB) +```shell +cd postgres/contrib/ptrack +``` + +5) Apply the PostgreSQL core patch: + +```shell +make patch +``` + +6) Compile and install PostgreSQL: + +```shell +make install-postgres prefix=$PWD/pgsql # or some other prefix of your choice +``` + +7) Add the newly created binaries to the PATH: + +```shell +export PATH=$PWD/pgsql/bin:$PATH +``` + +8) Compile and install `ptrack`: ```shell -echo "shared_preload_libraries = 'ptrack'" >> postgres_data/postgresql.conf -echo "ptrack.map_size = 64" >> postgres_data/postgresql.conf +make install USE_PGXS=1 ``` -6) Compile and install `ptrack` extension +9) Set `ptrack.map_size` (in MB): ```shell -USE_PGXS=1 make -C /path/to/ptrack/ install +echo "shared_preload_libraries = 'ptrack'" >> /postgresql.conf +echo "ptrack.map_size = 64" >> /postgresql.conf ``` -7) Run PostgreSQL and create `ptrack` extension +10) Run PostgreSQL and create the `ptrack` extension: ```sql postgres=# CREATE EXTENSION ptrack; @@ -158,24 +180,64 @@ To gather the whole changeset of modified blocks in `ptrack_get_pagemapset()` we ## Contribution -Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare), [fill up issues](https://github.com/postgrespro/ptrack/issues/new), or just reach one of us directly (e.g. <[Alexey Kondratov](mailto:a.kondratov@postgrespro.ru?subject=[GitHub]%20Ptrack), [@ololobus](https://github.com/ololobus)>) if you are interested in `ptrack`. +Feel free to [send a pull request](https://github.com/postgrespro/ptrack/compare), [create an issue](https://github.com/postgrespro/ptrack/issues/new) or [reach us by e-mail](mailto:team-wd40@lists.postgrespro.ru??subject=[GitHub]%20Ptrack) if you are interested in `ptrack`. + +## Tests -### Tests +All changes of the source code in this repository are checked by CI - see commit statuses and the project status badge. You can also run tests locally by executing a few Makefile targets. -Everything is tested automatically with [travis-ci.com](https://travis-ci.com/postgrespro/ptrack) and [codecov.io](https://codecov.io/gh/postgrespro/ptrack), but you can also run tests locally via `Docker`: +### Prerequisites -```sh -export PG_BRANCH=REL_14_STABLE -export TEST_CASE=all -export MODE=paranoia +To run Python tests install the following packages: -./make_dockerfile.sh +OS packages: + - python3-pip + - python3-six + - python3-pytest + - python3-pytest-xdist -docker-compose build -docker-compose run tests +PIP packages: + - testgres + +For example, for Ubuntu: + +```shell +sudo apt update +sudo apt install python3-pip python3-six python3-pytest python3-pytest-xdist +sudo pip3 install testgres +``` + +### Testing + +Install PostgreSQL and ptrack as described in [Installation](#installation), install the testing prerequisites, then do (assuming the current directory is `ptrack`): +```shell +git clone https://github.com/postgrespro/pg_probackup.git ../pg_probackup # clone the repository into postgres/contrib/pg_probackup +# remember to export PATH=/path/to/pgsql/bin:$PATH +make install-pg-probackup USE_PGXS=1 top_srcdir=../.. +make test-tap USE_PGXS=1 +make test-python ``` -Available test modes (`MODE`) are `basic` (default) and `paranoia` (per-block checksum comparison of `PGDATA` content before and after backup-restore process). Available test cases (`TEST_CASE`) are `tap` (minimalistic PostgreSQL [tap test](https://github.com/postgrespro/ptrack/blob/master/t/001_basic.pl)), `all` or any specific [pg_probackup test](https://github.com/postgrespro/pg_probackup/blob/master/tests/ptrack.py), e.g. `test_ptrack_simple`. +If `pg_probackup` is not located in `postgres/contrib` then additionally specify the path to the `pg_probackup` directory when building `pg_probackup`: +```shell +make install-pg-probackup USE_PGXS=1 top_srcdir=/path/to/postgres pg_probackup_dir=/path/to/pg_probackup +``` + +You can use a public Docker image which already has the necessary build environment (but not the testing prerequisites): + +```shell +docker run -e USER_ID=`id -u` -it -v $PWD:/work --name=ptrack ghcr.io/postgres-dev/ubuntu-22.04:1.0 +dev@a033797d2f73:~$ +``` + +## Environment variables + +| Variable | Possible values | Required | Default value | Description | +| - | - | - | - | - | +| NPROC | An integer greater than 0 | No | Output of `nproc` | The number of threads used for building and running tests | +| PG_CONFIG | File path | No | pg_config (from the PATH) | The path to the `pg_config` binary | +| TESTS | A Pytest filter expression | No | Not set (run all Python tests) | A filter to include only selected tests into the run. See the Pytest `-k` option for more information. This variable is only applicable to `test-python` for the tests located in [tests](https://github.com/postgrespro/pg_probackup/tree/master/tests). | +| TEST_MODE | normal, legacy, paranoia | No | normal | The "legacy" mode runs tests in an environment similar to a 32-bit Windows system. This mode is only applicable to `test-tap`. The "paranoia" mode compares the checksums of each block of the database catalog (PGDATA) contents before making a backup and after the restoration. This mode is only applicable to `test-python`.| ### TODO diff --git a/codecov.yml b/codecov.yml index fe3b308..00b744e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,6 +1,9 @@ codecov: notify: - after_n_builds: 12 # keep in sync with .travis.yml number of builds + # must be equal to the total number of parallel jobs in a CI pipeline + # (Postgres versions x test types x test modes x OSes minus excluded + # combinations) + after_n_builds: 10 # datapagemap.c/.h are copied from Postgres, so let's remove it # from report. Otherwise, we would have to remove some currently diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index fc65455..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.7" -services: - tests: - build: - context: . - - cap_add: - - SYS_PTRACE - - security_opt: - - seccomp=unconfined - - # don't work - #sysctls: - # kernel.yama.ptrace_scope: 0 - privileged: true - diff --git a/make_dockerfile.sh b/make_dockerfile.sh deleted file mode 100755 index 409a5b9..0000000 --- a/make_dockerfile.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env sh - -if [ -z ${PG_BRANCH+x} ]; then - echo PG_BRANCH is not set! - exit 1 -fi - -if [ -z ${MODE+x} ]; then - MODE=basic -else - echo MODE=${MODE} -fi - -if [ -z ${TEST_CASE+x} ]; then - TEST_CASE=all -else - echo TEST_CASE=${TEST_CASE} -fi - -if [ -z ${TEST_REPEATS+x} ]; then - TEST_REPEATS=1 -else - echo TEST_REPEATS=${TEST_REPEATS} -fi - -echo PG_BRANCH=${PG_BRANCH} - -sed \ - -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ - -e 's/${MODE}/'${MODE}/g \ - -e 's/${TEST_CASE}/'${TEST_CASE}/g \ - -e 's/${TEST_REPEATS}/'${TEST_REPEATS}/g \ -Dockerfile.in > Dockerfile diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index 1b4a693..0000000 --- a/run_tests.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright (c) 2019-2021, Postgres Professional -# - -PTRACK_SRC=${PWD}/ptrack -PG_SRC=${PWD}/postgres -PBK_SRC=${PWD}/pg_probackup -status=0 - -######################################################### -# Clone Postgres -echo "############### Getting Postgres sources" -git clone https://github.com/postgres/postgres.git --depth=1 --branch=${PG_BRANCH} ${PG_SRC} - -# Clone pg_probackup -echo "############### Getting pg_probackup sources" -git clone https://github.com/postgrespro/pg_probackup.git --depth=1 --branch=master ${PBK_SRC} - -######################################################### -# Compile and install Postgres -cd ${PG_SRC} # Go to postgres dir - -echo "############### Applying ptrack patch" -git apply --verbose --3way ${PTRACK_SRC}/patches/${PG_BRANCH}-ptrack-core.diff - -if [ "${MODE}" = "paranoia" ]; then - echo "############### Paranoia mode: applying turn-off-hint-bits.diff" - git apply --verbose --3way ${PTRACK_SRC}/patches/turn-off-hint-bits.diff -fi - -echo "############### Compiling Postgres" -if [ "${TEST_CASE}" = "tap" ] && [ "${MODE}" = "legacy" ]; then - ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet -else - ./configure --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet -fi -make --quiet --jobs=$(nproc) install -make --quiet --jobs=$(nproc) --directory=contrib/ install - -# Override default Postgres instance -export PATH=${PGHOME}/bin:${PATH} -export LD_LIBRARY_PATH=${PGHOME}/lib -export PG_CONFIG=$(which pg_config) - -# Show pg_config path (just in case) -echo "############### pg_config path" -which pg_config - -# Show pg_config just in case -echo "############### pg_config" -pg_config - -######################################################### -# Build and install ptrack extension -echo "############### Compiling and installing ptrack extension" -cp --recursive ${PTRACK_SRC} ${PG_SRC}/contrib/ptrack -make USE_PGXS=1 --directory=${PG_SRC}/contrib/ptrack/ clean -make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" --directory=${PG_SRC}/contrib/ptrack/ install - -if [ "${TEST_CASE}" = "tap" ]; then - - # Run tap tests - echo "############### Running tap tests" - if [ "${MODE}" = "legacy" ]; then - # There is a known issue with attaching shared memory segment using the same - # address each time, when EXEC_BACKEND mechanism is turned on. It happens due - # to the ASLR address space randomization, so we are trying to attach a segment - # to the already occupied location. That way we simply turning off ASLR here. - # - # Postgres comment: https://github.com/postgres/postgres/blob/5cbfce562f7cd2aab0cdc4694ce298ec3567930e/src/backend/postmaster/postmaster.c#L4929 - setarch x86_64 --addr-no-randomize make --directory=${PG_SRC}/contrib/ptrack check || status=$? - else - make --directory=${PG_SRC}/contrib/ptrack check || status=$? - fi - -else - # Set kernel params (used for debugging -- probackup tests) - echo "############### setting kernel params" - sudo sh -c 'echo 0 > /proc/sys/kernel/yama/ptrace_scope' - - # Build and install pg_probackup - echo "############### Compiling and installing pg_probackup" - cd ${PBK_SRC} # Go to pg_probackup dir - make USE_PGXS=1 top_srcdir=${PG_SRC} install - - # Setup python environment - echo "############### Setting up python env" - virtualenv --python=/usr/bin/python3 pyenv - source pyenv/bin/activate - pip install testgres==1.8.2 - - echo "############### Testing" - export PG_PROBACKUP_PTRACK=ON - if [ "${MODE}" = "basic" ]; then - export PG_PROBACKUP_TEST_BASIC=ON - elif [ "${MODE}" = "paranoia" ]; then - export PG_PROBACKUP_PARANOIA=ON - fi - - if [ "${TEST_CASE}" = "all" ]; then - # Run all pg_probackup ptrack tests - PBK_TEST_CASE=tests.ptrack - else - PBK_TEST_CASE=tests.ptrack.PtrackTest.${TEST_CASE} - fi - for i in `seq ${TEST_REPEATS}`; do - python3 -m unittest -v ${PBK_TEST_CASE} || status=$? - done - - # Exit virtualenv - deactivate -fi - -######################################################### -# codecov -echo "############### Codecov" -cd ${PTRACK_SRC} -# Generate *.gcov files -gcov ${PG_SRC}/contrib/ptrack/*.c ${PG_SRC}/contrib/ptrack/*.h - -# Send coverage stats to Codecov -bash <(curl -s https://codecov.io/bash) - -# Something went wrong, exit with code 1 -if [ ${status} -ne 0 ]; then exit 1; fi From 68ca7176a6da82d71dc3a878f51a70aebc1629b9 Mon Sep 17 00:00:00 2001 From: Daria Lepikhova Date: Thu, 2 Nov 2023 16:45:17 +0700 Subject: [PATCH 3/3] PGPRO-8844: Add subdir into Makefile for pgpro CI tests --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 499067a..9d4b451 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ ifdef USE_PGXS PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else +subdir = contrib/ptrack top_builddir = ../.. # Makefile.global is a build artifact and initially may not be available ifneq ($(wildcard $(top_builddir)/src/Makefile.global), )