From d923693c8978e2d2aa70388df96797ffb80c04d7 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Mon, 7 Oct 2024 17:48:29 +0300 Subject: [PATCH] add airbyte rock --- .licenserc.yaml | 1 + airbyte_rock/local-files/pod-sweeper.sh | 69 ++++++++++ airbyte_rock/patches/db-jooq.patch | 31 +++++ airbyte_rock/rockcraft.yaml | 165 ++++++++++++++++++++++++ charmcraft.yaml | 7 - src/charm.py | 15 +-- tests/integration/conftest.py | 17 ++- tests/unit/test_charm.py | 2 +- 8 files changed, 282 insertions(+), 25 deletions(-) create mode 100644 airbyte_rock/local-files/pod-sweeper.sh create mode 100644 airbyte_rock/patches/db-jooq.patch create mode 100644 airbyte_rock/rockcraft.yaml diff --git a/.licenserc.yaml b/.licenserc.yaml index 5aa32de..ca9d0ed 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -34,4 +34,5 @@ header: - 'LICENSE' - 'trivy.yaml' - 'lib/**' + - '**/*.patch' comment: on-failure diff --git a/airbyte_rock/local-files/pod-sweeper.sh b/airbyte_rock/local-files/pod-sweeper.sh new file mode 100644 index 0000000..d5e0d0b --- /dev/null +++ b/airbyte_rock/local-files/pod-sweeper.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +# https://github.com/airbytehq/airbyte-platform/blob/main/charts/airbyte-pod-sweeper/templates/configmap.yaml +# TODO(kelkawi-a): Move this to Airbyte ROCK + +get_job_pods() { + # echo "Running kubectl command to get job pods..." + kubectl -n "${JOB_KUBE_NAMESPACE}" -L airbyte -l airbyte=job-pod \ + get pods \ + -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' +} + +delete_pod() { + printf "From status '%s' since '%s', " "$2" "$3" + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n "${JOB_KUBE_NAMESPACE}" delete pod +} + +while : +do + echo "Starting pod sweeper cycle:" + + if [ -n "${RUNNING_TTL_MINUTES}" ]; then + # Time window for running pods + RUNNING_DATE_STR=$(date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins) + RUNNING_DATE=$(date -d "${RUNNING_DATE_STR}" +%s) + echo "Will sweep running pods from before ${RUNNING_DATE_STR}" + fi + + if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then + # Shorter time window for succeeded pods + SUCCESS_DATE_STR=$(date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins) + SUCCESS_DATE=$(date -d "${SUCCESS_DATE_STR}" +%s) + echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}" + fi + + if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then + # Longer time window for unsuccessful pods (to debug) + NON_SUCCESS_DATE_STR=$(date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins) + NON_SUCCESS_DATE=$(date -d "${NON_SUCCESS_DATE_STR}" +%s) + echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}" + fi + + echo "Running kubectl command to get job pods..." + get_job_pods | while read -r POD; do + IFS=' ' read -r POD_NAME POD_STATUS POD_DATE_STR POD_START_DATE_STR <<< "$POD" + + POD_DATE=$(date -d "${POD_DATE_STR:-$POD_START_DATE_STR}" '+%s') + echo "Evaluating pod: $POD_NAME with status $POD_STATUS since $POD_DATE_STR" + + if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then + if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && { [[ "$POD_STATUS" = "Succeeded" ]] || [[ "$POD_STATUS" = "Completed" ]]; }; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + + echo "Completed pod sweeper cycle. Sleeping for 60 seconds..." + sleep 60 +done diff --git a/airbyte_rock/patches/db-jooq.patch b/airbyte_rock/patches/db-jooq.patch new file mode 100644 index 0000000..2197c0b --- /dev/null +++ b/airbyte_rock/patches/db-jooq.patch @@ -0,0 +1,31 @@ +diff --git a/airbyte-db/jooq/build.gradle.kts b/airbyte-db/jooq/build.gradle.kts +index 6a4b2d6138..d031b1bb3e 100644 +--- a/airbyte-db/jooq/build.gradle.kts ++++ b/airbyte-db/jooq/build.gradle.kts +@@ -15,7 +15,7 @@ dependencies { + + // jOOQ code generation) + implementation(libs.jooq.codegen) +- implementation(libs.platform.testcontainers.postgresql) ++ // implementation(libs.platform.testcontainers.postgresql) + + // These are required because gradle might be using lower version of Jna from other + // library transitive dependency. Can be removed if we can figure out which library is the cause. +@@ -50,7 +50,7 @@ jooq { + + configurations { + create("configsDatabase") { +- generateSchemaSourceOnCompilation = true ++ generateSchemaSourceOnCompilation = false + jooqConfiguration.apply { + generator.apply { + name = "org.jooq.codegen.DefaultGenerator" +@@ -68,7 +68,7 @@ jooq { + } + + create("jobsDatabase") { +- generateSchemaSourceOnCompilation = true ++ generateSchemaSourceOnCompilation = false + jooqConfiguration.apply { + generator.apply { + name = "org.jooq.codegen.DefaultGenerator" diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml new file mode 100644 index 0000000..48eb317 --- /dev/null +++ b/airbyte_rock/rockcraft.yaml @@ -0,0 +1,165 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +name: airbyte +summary: Airbyte rock +description: Airbyte OCI image for the Airbyte charm +version: "1.0" +base: ubuntu@22.04 +build-base: ubuntu@22.04 +license: Apache-2.0 +platforms: + amd64: + +# Please refer to +# https://discourse.ubuntu.com/t/unifying-user-identity-across-snaps-and-rocks/36469 +# for more information about shared user. +# The UID 584792 corresponds to _daemon_ user. +run_user: _daemon_ + +services: + airbyte-server: + override: replace + summary: "airbyte-server service" + startup: enabled + command: "/bin/bash -c airbyte-server/bin/airbyte-server" + airbyte-workers: + override: replace + summary: "airbyte-workers service" + startup: enabled + command: "/bin/bash -c airbyte-workers/bin/airbyte-workers" + airbyte-api-server: + override: replace + summary: "airbyte-api-server service" + startup: enabled + command: "/bin/bash -c airbyte-api-server/bin/airbyte-api-server" + airbyte-bootloader: + override: replace + summary: "airbyte-bootloader service" + startup: enabled + command: "/bin/bash -c airbyte-bootloader/bin/airbyte-bootloader" + airbyte-connector-builder-server: + override: replace + summary: "airbyte-connector-builder-server service" + startup: enabled + command: "/bin/bash -c airbyte-connector-builder-server/bin/airbyte-connector-builder-server" + airbyte-cron: + override: replace + summary: "airbyte-cron service" + startup: enabled + command: "/bin/bash -c airbyte-cron/bin/airbyte-cron" + airbyte-pod-sweeper: + override: replace + summary: "airbyte-pod-sweeper service" + startup: enabled + command: "/bin/bash -c airbyte-pod-sweeper/bin/airbyte-pod-sweeper" + +environment: + JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 + + +parts: + # patches: + # plugin: dump + # source: ./patches + # organize: + # db-jooq.patch: patches/db-jooq.patch + # stage: + # - patches/db-jooq.patch + # prime: + # - "-*" + + # assemble: + # plugin: dump + # source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line + # source-type: git + # source-tag: v0.60.0 + # stage: + # - '*' + + db-lib: + plugin: nil + source: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-db/db-lib/0.60.0/db-lib-0.60.0.jar + source-checksum: sha512/5417e491007a80276a51a81c49815da51ecf855eb243d5c48c4ddce75cb330c2930d6dd5100a24d6185e1d1067bf8c176d28d7aadc62d21db078b62327aa77e8 + source-type: zip + stage: + - '*' + + jooq: + plugin: nil + source: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-db/jooq/0.60.0/jooq-0.60.0.jar + source-checksum: sha512/da09c7f4e127af269e6733baa68e3b9ffccd85eadead6d32e441e9b5cb014056720e23039c76914e242ef9ee22d607d20983ffb71b12e1f2748e6630e9bbda86 + source-type: zip + stage: + - '*' + + assemble: + # after: [patches] + plugin: dump + source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line + source-type: git + source-tag: v0.60.0 + build-packages: + - jq + - curl + - coreutils + - bash + - gradle + - openjdk-21-jdk-headless + stage-packages: + - openjdk-21-jdk-headless + override-build: | + # git apply ${CRAFT_STAGE}/patches/*.patch + # ./gradlew :airbyte-db:jooq:compileJava + ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace --info + + # ./gradlew :airbyte-server:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-api-server:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-workers:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-bootloader:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-cron:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-connector-builder-server:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq + + + tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar + + cp -r ./airbyte-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-server + cp -r ./airbyte-api-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-api-server + cp -r ./airbyte-workers/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-workers + cp -r ./airbyte-bootloader/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-bootloader + cp -r ./airbyte-cron/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-cron + cp -r ./airbyte-connector-builder-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server + stage: + - airbyte-server + - airbyte-api-server + - airbyte-workers + - airbyte-bootloader + - airbyte-cron + - airbyte-connector-builder-server + + local-files: + plugin: dump + source: ./local-files + organize: + pod-sweeper.sh: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + stage: + - airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + permissions: + - path: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + owner: 584792 + group: 584792 + mode: "755" + + # ./gradlew distTar -x dockerBuildImage + # ./gradlew distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-api-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-workers:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-bootloader:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-cron:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-connector-builder-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin \ No newline at end of file diff --git a/charmcraft.yaml b/charmcraft.yaml index e20d97e..564c6bc 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -448,28 +448,21 @@ resources: airbyte-api-server: type: oci-image description: OCI image for Airbyte API server - upstream-source: airbyte/airbyte-api-server:0.60.0 airbyte-bootloader: type: oci-image description: OCI image for Airbyte Bootloader - upstream-source: airbyte/bootloader:0.60.0 airbyte-connector-builder-server: type: oci-image description: OCI image for Airbyte Connector Builder Server - upstream-source: airbyte/connector-builder-server:0.60.0 airbyte-cron: type: oci-image description: OCI image for Airbyte Cron - upstream-source: airbyte/cron:0.60.0 airbyte-pod-sweeper: type: oci-image description: OCI image for Airbyte Pod Sweeper - upstream-source: bitnami/kubectl:1.29.4 airbyte-server: type: oci-image description: OCI image for Airbyte Server - upstream-source: airbyte/server:0.60.0 airbyte-workers: type: oci-image description: OCI image for Airbyte Worker - upstream-source: airbyte/worker:0.60.0 diff --git a/src/charm.py b/src/charm.py index 80e2412..3f1d565 100755 --- a/src/charm.py +++ b/src/charm.py @@ -5,7 +5,6 @@ """Charm the application.""" import logging -from pathlib import Path import ops from botocore.exceptions import ClientError @@ -53,7 +52,7 @@ def get_pebble_layer(application_name, context): "services": { application_name: { "summary": application_name, - "command": f"/bin/bash -c airbyte-app/bin/{application_name}", + "command": f"/bin/bash -c {application_name}/bin/{application_name}", "startup": "enabled", "override": "replace", # Including config values here so that a change in the @@ -293,18 +292,6 @@ def _update(self, event): event.defer() return - if container_name == "airbyte-pod-sweeper": - script_path = Path(__file__).parent / "scripts/pod-sweeper.sh" - - with open(script_path, "r") as file_source: - logger.info("pushing pod-sweeper script...") - container.push( - f"/airbyte-app/bin/{container_name}", - file_source, - make_dirs=True, - permissions=0o755, - ) - env = create_env(self.model.name, self.app.name, container_name, self.config, self._state) env = {k: v for k, v in env.items() if v is not None} pebble_layer = get_pebble_layer(container_name, env) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index cf09a4d..0a4ca83 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,6 +4,7 @@ """Charm integration test config.""" import logging +from pathlib import Path import pytest_asyncio from helpers import ( @@ -16,17 +17,27 @@ perform_temporal_integrations, run_sample_workflow, ) +from pytest import FixtureRequest from pytest_operator.plugin import OpsTest logger = logging.getLogger(__name__) +@pytest_asyncio.fixture(scope="module", name="charm") +async def charm_fixture(request: FixtureRequest, ops_test: OpsTest) -> str | Path: + """Fetch the path to charm.""" + charms = request.config.getoption("--charm-file") + if not charms: + charm = await ops_test.build_charm(".") + assert charm, "Charm not built" + return charm + return charms[0] + + @pytest_asyncio.fixture(name="deploy", scope="module") -async def deploy(ops_test: OpsTest): +async def deploy(ops_test: OpsTest, charm: str): """Test the app is up and running.""" await ops_test.model.set_config({"update-status-hook-interval": "1m"}) - - charm = await ops_test.build_charm(".") resources = get_airbyte_charm_resources() await ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME_AIRBYTE_SERVER, trust=True) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 9968c3f..058e839 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -348,7 +348,7 @@ def create_plan(container_name, storage_type): "services": { container_name: { "summary": container_name, - "command": f"/bin/bash -c airbyte-app/bin/{container_name}", + "command": f"/bin/bash -c {container_name}/bin/{container_name}", "startup": "enabled", "override": "replace", "environment": {