From 15815ccb4da257f18a5a994f20f2d250d2ab283a Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 08:26:11 +0300 Subject: [PATCH 01/16] Adds postgres 15 support for docker --- .../templates/docker/docker-pkgvars.tmpl | 3 -- .../docker/latest/latest.tmpl.dockerfile | 4 +-- .../postgres-13/postgres-13.tmpl.dockerfile | 4 +-- .../postgres-14.tmpl.dockerfile} | 6 ++-- packaging_automation/update_docker.py | 32 +++++++++---------- 5 files changed, 23 insertions(+), 26 deletions(-) delete mode 100644 packaging_automation/templates/docker/docker-pkgvars.tmpl rename packaging_automation/templates/docker/{postgres-12/postgres-12.tmpl.dockerfile => postgres-14/postgres-14.tmpl.dockerfile} (91%) diff --git a/packaging_automation/templates/docker/docker-pkgvars.tmpl b/packaging_automation/templates/docker/docker-pkgvars.tmpl deleted file mode 100644 index f88b7747..00000000 --- a/packaging_automation/templates/docker/docker-pkgvars.tmpl +++ /dev/null @@ -1,3 +0,0 @@ -# This file is auto generated from it's template, -# see citusdata/tools/packaging_automation/templates/docker/docker-pkgvars.tmpl. -latest_postgres_version={{postgres_version}} \ No newline at end of file diff --git a/packaging_automation/templates/docker/latest/latest.tmpl.dockerfile b/packaging_automation/templates/docker/latest/latest.tmpl.dockerfile index 7ff41965..cc25c459 100644 --- a/packaging_automation/templates/docker/latest/latest.tmpl.dockerfile +++ b/packaging_automation/templates/docker/latest/latest.tmpl.dockerfile @@ -20,8 +20,8 @@ RUN apt-get update \ curl \ && curl -s https://install.citusdata.com/community/deb.sh | bash \ && apt-get install -y postgresql-$PG_MAJOR-citus-{{project_minor_version}}=$CITUS_VERSION \ - postgresql-$PG_MAJOR-hll=2.16.citus-1 \ - postgresql-$PG_MAJOR-topn=2.4.0 \ + postgresql-$PG_MAJOR-hll=2.17.citus-1 \ + postgresql-$PG_MAJOR-topn=2.5.0 \ && apt-get purge -y --auto-remove curl \ && rm -rf /var/lib/apt/lists/* diff --git a/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile b/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile index 02111fcb..d46b8592 100644 --- a/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile +++ b/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile @@ -20,8 +20,8 @@ RUN apt-get update \ curl \ && curl -s https://install.citusdata.com/community/deb.sh | bash \ && apt-get install -y postgresql-$PG_MAJOR-citus-{{project_minor_version}}=$CITUS_VERSION \ - postgresql-$PG_MAJOR-hll=2.16.citus-1 \ - postgresql-$PG_MAJOR-topn=2.4.0 \ + postgresql-$PG_MAJOR-hll=2.17.citus-1 \ + postgresql-$PG_MAJOR-topn=2.5.0 \ && apt-get purge -y --auto-remove curl \ && rm -rf /var/lib/apt/lists/* diff --git a/packaging_automation/templates/docker/postgres-12/postgres-12.tmpl.dockerfile b/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile similarity index 91% rename from packaging_automation/templates/docker/postgres-12/postgres-12.tmpl.dockerfile rename to packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile index 027620f1..d46b8592 100644 --- a/packaging_automation/templates/docker/postgres-12/postgres-12.tmpl.dockerfile +++ b/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile @@ -1,5 +1,5 @@ # This file is auto generated from it's template, -# see citusdata/tools/packaging_automation/templates/docker/postgres-12/postgres-12.tmpl.dockerfile. +# see citusdata/tools/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile. FROM postgres:{{postgres_version}} ARG VERSION={{project_version}} LABEL maintainer="Citus Data https://citusdata.com" \ @@ -20,8 +20,8 @@ RUN apt-get update \ curl \ && curl -s https://install.citusdata.com/community/deb.sh | bash \ && apt-get install -y postgresql-$PG_MAJOR-citus-{{project_minor_version}}=$CITUS_VERSION \ - postgresql-$PG_MAJOR-hll=2.16.citus-1 \ - postgresql-$PG_MAJOR-topn=2.4.0 \ + postgresql-$PG_MAJOR-hll=2.17.citus-1 \ + postgresql-$PG_MAJOR-topn=2.5.0 \ && apt-get purge -y --auto-remove curl \ && rm -rf /var/lib/apt/lists/* diff --git a/packaging_automation/update_docker.py b/packaging_automation/update_docker.py index ef7e2c3d..3bfc0d79 100644 --- a/packaging_automation/update_docker.py +++ b/packaging_automation/update_docker.py @@ -21,21 +21,21 @@ class SupportedDockerImages(Enum): latest = 1 docker_compose = 2 alpine = 3 - postgres12 = 4 - postgres13 = 5 + postgres13 = 4 + postgres14 = 5 docker_templates = {SupportedDockerImages.latest: "latest/latest.tmpl.dockerfile", SupportedDockerImages.docker_compose: "latest/docker-compose.tmpl.yml", SupportedDockerImages.alpine: "alpine/alpine.tmpl.dockerfile", - SupportedDockerImages.postgres12: "postgres-12/postgres-12.tmpl.dockerfile", - SupportedDockerImages.postgres13: "postgres-13/postgres-13.tmpl.dockerfile"} + SupportedDockerImages.postgres13: "postgres-12/postgres-13.tmpl.dockerfile", + SupportedDockerImages.postgres14: "postgres-13/postgres-14.tmpl.dockerfile"} docker_outputs = {SupportedDockerImages.latest: "Dockerfile", SupportedDockerImages.docker_compose: "docker-compose.yml", SupportedDockerImages.alpine: "alpine/Dockerfile", - SupportedDockerImages.postgres12: "postgres-12/Dockerfile", - SupportedDockerImages.postgres13: "postgres-13/Dockerfile"} + SupportedDockerImages.postgres13: "postgres-13/Dockerfile", + SupportedDockerImages.postgres14: "postgres-14/Dockerfile"} BASE_PATH = pathlib2.Path(__file__).parent.absolute() @@ -68,24 +68,24 @@ def update_docker_file_alpine(project_version: str, template_path: str, exec_pat write_to_file(content, dest_file_name) -def update_docker_file_for_postgres12(project_version: str, template_path: str, exec_path: str, postgres_version: str): +def update_docker_file_for_postgres13(project_version: str, template_path: str, exec_path: str, postgres_version: str): minor_version = get_minor_project_version_for_docker(project_version) debian_project_version = project_version.replace("_", "-") content = process_template_file_with_minor(debian_project_version, template_path, - docker_templates[SupportedDockerImages.postgres12], minor_version, + docker_templates[SupportedDockerImages.postgres13], minor_version, postgres_version) - dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.postgres12]}" + dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.postgres13]}" create_directory_if_not_exists(dest_file_name) write_to_file(content, dest_file_name) -def update_docker_file_for_postgres13(project_version: str, template_path: str, exec_path: str, postgres_version: str): +def update_docker_file_for_postgres14(project_version: str, template_path: str, exec_path: str, postgres_version: str): minor_version = get_minor_project_version_for_docker(project_version) debian_project_version = project_version.replace("_", "-") content = process_template_file_with_minor(debian_project_version, template_path, - docker_templates[SupportedDockerImages.postgres13], minor_version, + docker_templates[SupportedDockerImages.postgres14], minor_version, postgres_version) - dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.postgres13]}" + dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.postgres14]}" create_directory_if_not_exists(dest_file_name) write_to_file(content, dest_file_name) @@ -127,22 +127,22 @@ def update_all_docker_files(project_version: str, exec_path: str): template_path = f"{BASE_PATH}/templates/docker" pkgvars_file = f"{exec_path}/pkgvars" - postgres_14_version, postgres_13_version, postgres_12_version = read_postgres_versions(pkgvars_file) + postgres_15_version, postgres_14_version, postgres_13_version = read_postgres_versions(pkgvars_file) - latest_postgres_version = postgres_14_version + latest_postgres_version = postgres_15_version update_docker_file_for_latest_postgres(project_version, template_path, exec_path, latest_postgres_version) update_regular_docker_compose_file(project_version, template_path, exec_path) update_docker_file_alpine(project_version, template_path, exec_path, latest_postgres_version) - update_docker_file_for_postgres12(project_version, template_path, exec_path, postgres_12_version) update_docker_file_for_postgres13(project_version, template_path, exec_path, postgres_13_version) + update_docker_file_for_postgres14(project_version, template_path, exec_path, postgres_14_version) update_changelog(project_version, exec_path, latest_postgres_version) def read_postgres_versions(pkgvars_file: str) -> Tuple[str, str, str]: if os.path.exists(pkgvars_file): config = dotenv_values(pkgvars_file) - return config["postgres_14_version"], config["postgres_13_version"], config["postgres_12_version"] + return config["postgres_15_version"], config["postgres_14_version"], config["postgres_13_version"] return "14.1", "13.5", "12.9" From e957f63b8842a759b95202b3f78e60fe43c7d430 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 10:20:54 +0300 Subject: [PATCH 02/16] Updates publish_docker --- packaging_automation/publish_docker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index cdc211bc..c0119f19 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -28,8 +28,8 @@ class DockerImageType(Enum): latest = 1 alpine = 2 nightly = 3 - postgres_12 = 4 - postgres_13 = 5 + postgres_13 = 4 + postgres_14 = 5 class ManualTriggerType(Enum): @@ -48,9 +48,9 @@ class ScheduleType(Enum): DockerImageType.latest: {"file-name": "Dockerfile", "docker-tag": "latest", "schedule-type": ScheduleType.regular}, DockerImageType.alpine: {"file-name": "alpine/Dockerfile", "docker-tag": "alpine", "schedule-type": ScheduleType.regular}, - DockerImageType.postgres_12: {"file-name": "postgres-12/Dockerfile", "docker-tag": "pg12", + DockerImageType.postgres_13: {"file-name": "postgres-13/Dockerfile", "docker-tag": "pg12", "schedule-type": ScheduleType.regular}, - DockerImageType.postgres_13: {"file-name": "postgres-13/Dockerfile", "docker-tag": "pg13", + DockerImageType.postgres_14: {"file-name": "postgres-14/Dockerfile", "docker-tag": "pg13", "schedule-type": ScheduleType.regular}, DockerImageType.nightly: {"file-name": "nightly/Dockerfile", "docker-tag": "nightly", "schedule-type": ScheduleType.nightly}} From 58b16f6739f69c9ad9d43075c17720e45401ce92 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 10:39:47 +0300 Subject: [PATCH 03/16] Updates update_docker --- packaging_automation/update_docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/update_docker.py b/packaging_automation/update_docker.py index 3bfc0d79..94b7c059 100644 --- a/packaging_automation/update_docker.py +++ b/packaging_automation/update_docker.py @@ -28,7 +28,7 @@ class SupportedDockerImages(Enum): docker_templates = {SupportedDockerImages.latest: "latest/latest.tmpl.dockerfile", SupportedDockerImages.docker_compose: "latest/docker-compose.tmpl.yml", SupportedDockerImages.alpine: "alpine/alpine.tmpl.dockerfile", - SupportedDockerImages.postgres13: "postgres-12/postgres-13.tmpl.dockerfile", + SupportedDockerImages.postgres13: "postgres-13/postgres-13.tmpl.dockerfile", SupportedDockerImages.postgres14: "postgres-13/postgres-14.tmpl.dockerfile"} docker_outputs = {SupportedDockerImages.latest: "Dockerfile", From 408d5ec1baa808c168a2554959ea48af4403cbe8 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 10:40:41 +0300 Subject: [PATCH 04/16] Updates update_docker --- packaging_automation/update_docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/update_docker.py b/packaging_automation/update_docker.py index 94b7c059..385f36c5 100644 --- a/packaging_automation/update_docker.py +++ b/packaging_automation/update_docker.py @@ -29,7 +29,7 @@ class SupportedDockerImages(Enum): SupportedDockerImages.docker_compose: "latest/docker-compose.tmpl.yml", SupportedDockerImages.alpine: "alpine/alpine.tmpl.dockerfile", SupportedDockerImages.postgres13: "postgres-13/postgres-13.tmpl.dockerfile", - SupportedDockerImages.postgres14: "postgres-13/postgres-14.tmpl.dockerfile"} + SupportedDockerImages.postgres14: "postgres-14/postgres-14.tmpl.dockerfile"} docker_outputs = {SupportedDockerImages.latest: "Dockerfile", SupportedDockerImages.docker_compose: "docker-compose.yml", From 1bc6066c749db93a59706f4b0aa5f7648ab11695 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 10:56:43 +0300 Subject: [PATCH 05/16] Updates alpine template --- .../templates/docker/alpine/alpine.tmpl.dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging_automation/templates/docker/alpine/alpine.tmpl.dockerfile b/packaging_automation/templates/docker/alpine/alpine.tmpl.dockerfile index dc305db3..f1d89cf8 100644 --- a/packaging_automation/templates/docker/alpine/alpine.tmpl.dockerfile +++ b/packaging_automation/templates/docker/alpine/alpine.tmpl.dockerfile @@ -24,6 +24,8 @@ RUN apk add --no-cache \ llvm \ lz4-dev \ zstd-dev \ + libxslt-dev \ + libxml2-dev \ icu-dev && \ apk add --no-cache libcurl && \ curl -sfLO "https://github.com/citusdata/citus/archive/v${VERSION}.tar.gz" && \ From c06e88d33d5e9a51cbf15aa30671a76d6ad2ff28 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 11:21:37 +0300 Subject: [PATCH 06/16] Fixes unit test --- packaging_automation/tests/test_publish_docker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging_automation/tests/test_publish_docker.py b/packaging_automation/tests/test_publish_docker.py index 820b8bcf..757f36f1 100644 --- a/packaging_automation/tests/test_publish_docker.py +++ b/packaging_automation/tests/test_publish_docker.py @@ -45,8 +45,8 @@ def test_get_image_tag(): image_name = get_image_tag(remove_prefix(TAG_NAME, "v"), DockerImageType.latest) assert image_name == "10.0.3" - image_name = get_image_tag(remove_prefix(TAG_NAME, "v"), DockerImageType.postgres_12) - assert image_name == "10.0.3-pg12" + image_name = get_image_tag(remove_prefix(TAG_NAME, "v"), DockerImageType.postgres_13) + assert image_name == "10.0.3-pg13" def test_publish_main_docker_images(): From 6a7c7318ab0e8b5b4ecbc86f7c3daf109f11116b Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 11:23:22 +0300 Subject: [PATCH 07/16] Fixes unit test --- .../tests/files/verify/expected_alpine_10.0.3.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging_automation/tests/files/verify/expected_alpine_10.0.3.txt b/packaging_automation/tests/files/verify/expected_alpine_10.0.3.txt index 5c9829ce..a518032c 100644 --- a/packaging_automation/tests/files/verify/expected_alpine_10.0.3.txt +++ b/packaging_automation/tests/files/verify/expected_alpine_10.0.3.txt @@ -24,6 +24,8 @@ RUN apk add --no-cache \ llvm \ lz4-dev \ zstd-dev \ + libxslt-dev \ + libxml2-dev \ icu-dev && \ apk add --no-cache libcurl && \ curl -sfLO "https://github.com/citusdata/citus/archive/v${VERSION}.tar.gz" && \ From 971119b49f26b5deb7b27d255220de99865158d1 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 11:32:03 +0300 Subject: [PATCH 08/16] Fixes unit tests --- packaging_automation/publish_docker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging_automation/publish_docker.py b/packaging_automation/publish_docker.py index c0119f19..9711844f 100644 --- a/packaging_automation/publish_docker.py +++ b/packaging_automation/publish_docker.py @@ -48,9 +48,9 @@ class ScheduleType(Enum): DockerImageType.latest: {"file-name": "Dockerfile", "docker-tag": "latest", "schedule-type": ScheduleType.regular}, DockerImageType.alpine: {"file-name": "alpine/Dockerfile", "docker-tag": "alpine", "schedule-type": ScheduleType.regular}, - DockerImageType.postgres_13: {"file-name": "postgres-13/Dockerfile", "docker-tag": "pg12", + DockerImageType.postgres_13: {"file-name": "postgres-13/Dockerfile", "docker-tag": "pg13", "schedule-type": ScheduleType.regular}, - DockerImageType.postgres_14: {"file-name": "postgres-14/Dockerfile", "docker-tag": "pg13", + DockerImageType.postgres_14: {"file-name": "postgres-14/Dockerfile", "docker-tag": "pg14", "schedule-type": ScheduleType.regular}, DockerImageType.nightly: {"file-name": "nightly/Dockerfile", "docker-tag": "nightly", "schedule-type": ScheduleType.nightly}} From fdfa31b14ccbcc5d71bc77914c69424ab7a8693a Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 11:43:06 +0300 Subject: [PATCH 09/16] Fixes unit tests --- packaging_automation/tests/test_update_docker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 319f2f25..04f4c599 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -7,7 +7,7 @@ DEFAULT_UNICODE_ERROR_HANDLER) from dotenv import dotenv_values from ..update_docker import (update_docker_file_for_latest_postgres, update_regular_docker_compose_file, - update_docker_file_alpine, update_docker_file_for_postgres12, + update_docker_file_alpine, update_docker_file_for_postgres14, update_docker_file_for_postgres13, update_changelog) BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2]) @@ -69,8 +69,8 @@ def test_update_docker_file_alpine(): assert len(lines) == 56 -def test_update_docker_file_for_postgres12(): - update_docker_file_for_postgres12(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_12_VERSION) +def test_update_docker_file_for_postgres14(): + update_docker_file_for_postgres14(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_12_VERSION) with open(f"{TEST_BASE_PATH}/postgres-12/Dockerfile", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: content = reader.read() From 84c76b565897d7bdb99b39850dc6163b815f83b6 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 11:47:48 +0300 Subject: [PATCH 10/16] Fixes unit tests --- packaging_automation/tests/test_update_docker.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index 04f4c599..a028c42a 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -13,9 +13,10 @@ BASE_PATH = os.getenv("BASE_PATH", default=pathlib2.Path(__file__).parents[2]) TEST_BASE_PATH = f"{BASE_PATH}/docker" PROJECT_VERSION = "10.0.3" -POSTGRES_14_VERSION = "14.1" -POSTGRES_13_VERSION = "13.5" -POSTGRES_12_VERSION = "12.9" + +POSTGRES_14_VERSION = "14.5" +POSTGRES_13_VERSION = "13.8" + PROJECT_NAME = "citus" version_details = get_version_details(PROJECT_VERSION) TEMPLATE_PATH = f"{BASE_PATH}/packaging_automation/templates/docker" @@ -70,12 +71,12 @@ def test_update_docker_file_alpine(): def test_update_docker_file_for_postgres14(): - update_docker_file_for_postgres14(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_12_VERSION) - with open(f"{TEST_BASE_PATH}/postgres-12/Dockerfile", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, + update_docker_file_for_postgres14(PROJECT_VERSION, TEMPLATE_PATH, TEST_BASE_PATH, POSTGRES_14_VERSION) + with open(f"{TEST_BASE_PATH}/postgres-14/Dockerfile", "r", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING, errors=DEFAULT_UNICODE_ERROR_HANDLER) as reader: content = reader.read() lines = content.splitlines() - assert lines[2].strip() == f"FROM postgres:{POSTGRES_12_VERSION}" + assert lines[2].strip() == f"FROM postgres:{POSTGRES_14_VERSION}" assert lines[3].strip() == f"ARG VERSION={PROJECT_VERSION}" assert f"postgresql-$PG_MAJOR-{PROJECT_NAME}-" \ f"{version_details['major']}.{version_details['minor']}=$CITUS_VERSION" in lines[21] From aa3473e8c92c30c368a11c46c1c4053b7955bc2a Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 11:49:16 +0300 Subject: [PATCH 11/16] Fixes unit tests --- packaging_automation/tests/test_citus_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 5ca82fde..a7f8d3a6 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -94,7 +94,7 @@ def test_build_packages(): platform=PLATFORM) assert os.path.exists(postgres_version_file_path) config = dotenv_values(postgres_version_file_path) - assert config["release_versions"] == "12,13,14" + assert config["release_versions"] == "13,14" assert config["nightly_versions"] == "13,14" From 4c1799c91a47d678697a5f136a0c7a57d3ec84e9 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 12:43:20 +0300 Subject: [PATCH 12/16] Fixes unit tests --- packaging_automation/tests/test_citus_package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index a7f8d3a6..c911f80a 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -48,7 +48,7 @@ def get_required_package_count(input_files_dir: str, platform: str): release_versions, _ = get_postgres_versions(platform=platform, input_files_dir=input_files_dir) - print(f"get_required_package_count: Relase versions:{release_versions}:{single_postgres_package_counts[platform]}") + print(f"get_required_package_count: Release versions:{release_versions}:{single_postgres_package_counts[platform]}") return len(release_versions) * single_postgres_package_counts[platform] @@ -99,7 +99,7 @@ def test_build_packages(): def test_get_required_package_count(): - assert get_required_package_count(input_files_dir=PACKAGING_EXEC_FOLDER, platform="el/8") == 9 + assert get_required_package_count(input_files_dir=PACKAGING_EXEC_FOLDER, platform="el/8") == 6 def test_decode_os_packages(): From 90eb8357b3ee8fcd594c2d8b804e8e1326de012e Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 12:46:48 +0300 Subject: [PATCH 13/16] Fixes unit tests --- packaging_automation/tests/test_update_docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/tests/test_update_docker.py b/packaging_automation/tests/test_update_docker.py index a028c42a..3c0db52c 100644 --- a/packaging_automation/tests/test_update_docker.py +++ b/packaging_automation/tests/test_update_docker.py @@ -67,7 +67,7 @@ def test_update_docker_file_alpine(): lines = content.splitlines() assert lines[2].strip() == f"FROM postgres:{POSTGRES_14_VERSION}-alpine" assert lines[3].strip() == f"ARG VERSION={PROJECT_VERSION}" - assert len(lines) == 56 + assert len(lines) == 58 def test_update_docker_file_for_postgres14(): From 43e0053b322c1c6625895d1833a485ed6a549384 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 13:42:23 +0300 Subject: [PATCH 14/16] Updates tesmplates and fixes unit tests --- .../citus-package-all-platforms-beta-test.yml | 53 ------------------- .../docker/latest/latest.tmpl.dockerfile | 2 +- .../postgres-13/postgres-13.tmpl.dockerfile | 2 +- .../postgres-14/postgres-14.tmpl.dockerfile | 2 +- .../tests/test_citus_package.py | 4 +- 5 files changed, 5 insertions(+), 58 deletions(-) delete mode 100644 .github/workflows/citus-package-all-platforms-beta-test.yml diff --git a/.github/workflows/citus-package-all-platforms-beta-test.yml b/.github/workflows/citus-package-all-platforms-beta-test.yml deleted file mode 100644 index cd0b9e9d..00000000 --- a/.github/workflows/citus-package-all-platforms-beta-test.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Citus package all platforms beta tests - -env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} - MICROSOFT_EMAIL: gindibay@microsoft.com - USER_NAME: Gurkan Indibay - MAIN_BRANCH: all-citus - PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} - PACKAGING_BRANCH_NAME: all-citus-unit-tests-beta - -on: - push: - branches: - - "**" - - workflow_dispatch: - -jobs: - unit_test_execution: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - el/7 - - el/8 - - ol/7 - - ol/8 - - debian/stretch - - debian/buster - - debian/bullseye - - ubuntu/bionic - - ubuntu/focal - env: - PLATFORM: ${{ matrix.platform }} - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources - - - name: Define git credentails - run: git config --global user.email "${MICROSOFT_EMAIL}"&& git config --global user.name "${USER_NAME}" - - - name: Install python requirements - run: python -m pip install -r packaging_automation/requirements.txt - - - name: Citus package tests - run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s diff --git a/packaging_automation/templates/docker/latest/latest.tmpl.dockerfile b/packaging_automation/templates/docker/latest/latest.tmpl.dockerfile index cc25c459..0e70b981 100644 --- a/packaging_automation/templates/docker/latest/latest.tmpl.dockerfile +++ b/packaging_automation/templates/docker/latest/latest.tmpl.dockerfile @@ -21,7 +21,7 @@ RUN apt-get update \ && curl -s https://install.citusdata.com/community/deb.sh | bash \ && apt-get install -y postgresql-$PG_MAJOR-citus-{{project_minor_version}}=$CITUS_VERSION \ postgresql-$PG_MAJOR-hll=2.17.citus-1 \ - postgresql-$PG_MAJOR-topn=2.5.0 \ + postgresql-$PG_MAJOR-topn=2.5.0.citus-1 \ && apt-get purge -y --auto-remove curl \ && rm -rf /var/lib/apt/lists/* diff --git a/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile b/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile index d46b8592..d5b985fb 100644 --- a/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile +++ b/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile @@ -21,7 +21,7 @@ RUN apt-get update \ && curl -s https://install.citusdata.com/community/deb.sh | bash \ && apt-get install -y postgresql-$PG_MAJOR-citus-{{project_minor_version}}=$CITUS_VERSION \ postgresql-$PG_MAJOR-hll=2.17.citus-1 \ - postgresql-$PG_MAJOR-topn=2.5.0 \ + postgresql-$PG_MAJOR-topn=2.5.0.citus-1 \ && apt-get purge -y --auto-remove curl \ && rm -rf /var/lib/apt/lists/* diff --git a/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile b/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile index d46b8592..d5b985fb 100644 --- a/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile +++ b/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile @@ -21,7 +21,7 @@ RUN apt-get update \ && curl -s https://install.citusdata.com/community/deb.sh | bash \ && apt-get install -y postgresql-$PG_MAJOR-citus-{{project_minor_version}}=$CITUS_VERSION \ postgresql-$PG_MAJOR-hll=2.17.citus-1 \ - postgresql-$PG_MAJOR-topn=2.5.0 \ + postgresql-$PG_MAJOR-topn=2.5.0.citus-1 \ && apt-get purge -y --auto-remove curl \ && rm -rf /var/lib/apt/lists/* diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index c911f80a..4baf174a 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -94,12 +94,12 @@ def test_build_packages(): platform=PLATFORM) assert os.path.exists(postgres_version_file_path) config = dotenv_values(postgres_version_file_path) - assert config["release_versions"] == "13,14" + assert config["release_versions"] == "12,13,14" assert config["nightly_versions"] == "13,14" def test_get_required_package_count(): - assert get_required_package_count(input_files_dir=PACKAGING_EXEC_FOLDER, platform="el/8") == 6 + assert get_required_package_count(input_files_dir=PACKAGING_EXEC_FOLDER, platform="el/8") == 9 def test_decode_os_packages(): From 9121e44c7b467a7c3c92dd3fb02329db6125f1e9 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 14:32:35 +0300 Subject: [PATCH 15/16] Fixes coment issue on template --- .../templates/docker/postgres-14/postgres-14.tmpl.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile b/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile index d5b985fb..bb7cb609 100644 --- a/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile +++ b/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile @@ -1,5 +1,5 @@ # This file is auto generated from it's template, -# see citusdata/tools/packaging_automation/templates/docker/postgres-13/postgres-13.tmpl.dockerfile. +# see citusdata/tools/packaging_automation/templates/docker/postgres-14/postgres-14.tmpl.dockerfile. FROM postgres:{{postgres_version}} ARG VERSION={{project_version}} LABEL maintainer="Citus Data https://citusdata.com" \ From 559f11a9c83b9870db60554d99be0b2aacc57215 Mon Sep 17 00:00:00 2001 From: gindibay Date: Mon, 17 Oct 2022 15:57:02 +0300 Subject: [PATCH 16/16] Rollbacks beta unit tests --- .../citus-package-all-platforms-beta-test.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/citus-package-all-platforms-beta-test.yml diff --git a/.github/workflows/citus-package-all-platforms-beta-test.yml b/.github/workflows/citus-package-all-platforms-beta-test.yml new file mode 100644 index 00000000..cd0b9e9d --- /dev/null +++ b/.github/workflows/citus-package-all-platforms-beta-test.yml @@ -0,0 +1,53 @@ +name: Citus package all platforms beta tests + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} + MICROSOFT_EMAIL: gindibay@microsoft.com + USER_NAME: Gurkan Indibay + MAIN_BRANCH: all-citus + PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} + PACKAGING_BRANCH_NAME: all-citus-unit-tests-beta + +on: + push: + branches: + - "**" + + workflow_dispatch: + +jobs: + unit_test_execution: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - el/7 + - el/8 + - ol/7 + - ol/8 + - debian/stretch + - debian/buster + - debian/bullseye + - ubuntu/bionic + - ubuntu/focal + env: + PLATFORM: ${{ matrix.platform }} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources + + - name: Define git credentails + run: git config --global user.email "${MICROSOFT_EMAIL}"&& git config --global user.name "${USER_NAME}" + + - name: Install python requirements + run: python -m pip install -r packaging_automation/requirements.txt + + - name: Citus package tests + run: python -m pytest -q packaging_automation/tests/test_citus_package.py -s