diff --git a/airbyte_rock/local-files/pod-sweeper.sh b/airbyte_rock/local-files/pod-sweeper.sh new file mode 100644 index 0000000..d5e0d0b --- /dev/null +++ b/airbyte_rock/local-files/pod-sweeper.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +# https://github.com/airbytehq/airbyte-platform/blob/main/charts/airbyte-pod-sweeper/templates/configmap.yaml +# TODO(kelkawi-a): Move this to Airbyte ROCK + +get_job_pods() { + # echo "Running kubectl command to get job pods..." + kubectl -n "${JOB_KUBE_NAMESPACE}" -L airbyte -l airbyte=job-pod \ + get pods \ + -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' +} + +delete_pod() { + printf "From status '%s' since '%s', " "$2" "$3" + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n "${JOB_KUBE_NAMESPACE}" delete pod +} + +while : +do + echo "Starting pod sweeper cycle:" + + if [ -n "${RUNNING_TTL_MINUTES}" ]; then + # Time window for running pods + RUNNING_DATE_STR=$(date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins) + RUNNING_DATE=$(date -d "${RUNNING_DATE_STR}" +%s) + echo "Will sweep running pods from before ${RUNNING_DATE_STR}" + fi + + if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then + # Shorter time window for succeeded pods + SUCCESS_DATE_STR=$(date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins) + SUCCESS_DATE=$(date -d "${SUCCESS_DATE_STR}" +%s) + echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}" + fi + + if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then + # Longer time window for unsuccessful pods (to debug) + NON_SUCCESS_DATE_STR=$(date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins) + NON_SUCCESS_DATE=$(date -d "${NON_SUCCESS_DATE_STR}" +%s) + echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}" + fi + + echo "Running kubectl command to get job pods..." + get_job_pods | while read -r POD; do + IFS=' ' read -r POD_NAME POD_STATUS POD_DATE_STR POD_START_DATE_STR <<< "$POD" + + POD_DATE=$(date -d "${POD_DATE_STR:-$POD_START_DATE_STR}" '+%s') + echo "Evaluating pod: $POD_NAME with status $POD_STATUS since $POD_DATE_STR" + + if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then + if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && { [[ "$POD_STATUS" = "Succeeded" ]] || [[ "$POD_STATUS" = "Completed" ]]; }; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + + echo "Completed pod sweeper cycle. Sleeping for 60 seconds..." + sleep 60 +done diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml new file mode 100644 index 0000000..9ad62a9 --- /dev/null +++ b/airbyte_rock/rockcraft.yaml @@ -0,0 +1,112 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +name: airbyte +summary: Airbyte rock +description: Airbyte OCI image for the Airbyte charm +version: "1.0" +base: ubuntu@22.04 +build-base: ubuntu@22.04 +license: Apache-2.0 +platforms: + amd64: + +# Please refer to +# https://discourse.ubuntu.com/t/unifying-user-identity-across-snaps-and-rocks/36469 +# for more information about shared user. +# The UID 584792 corresponds to _daemon_ user. +run_user: _daemon_ + +services: + airbyte-server: + override: replace + summary: "airbyte-server service" + startup: enabled + command: "/bin/bash -c airbyte-app/bin/airbyte-server" + airbyte-workers: + override: replace + summary: "airbyte-workers service" + startup: enabled + command: "/bin/bash -c airbyte-app/bin/airbyte-workers" + airbyte-api-server: + override: replace + summary: "airbyte-api-server service" + startup: enabled + command: "/bin/bash -c airbyte-app/bin/airbyte-api-server" + airbyte-bootloader: + override: replace + summary: "airbyte-bootloader service" + startup: enabled + command: "/bin/bash -c airbyte-app/bin/airbyte-bootloader" + airbyte-connector-builder-server: + override: replace + summary: "airbyte-connector-builder-server service" + startup: enabled + command: "/bin/bash -c airbyte-app/bin/airbyte-connector-builder-server" + airbyte-cron: + override: replace + summary: "airbyte-cron service" + startup: enabled + command: "/bin/bash -c airbyte-app/bin/airbyte-cron" + airbyte-pod-sweeper: + override: replace + summary: "airbyte-pod-sweeper service" + startup: enabled + command: "/bin/bash -c airbyte-app/bin/airbyte-pod-sweeper" + +environment: + JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 + + +parts: + assemble: + plugin: dump + source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line + source-type: git + source-tag: v0.60.0 + build-packages: + - jq + - curl + - coreutils + - bash + - gradle + - openjdk-21-jdk-headless + stage-packages: + - openjdk-21-jdk-headless + override-build: | + ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform + + tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar + + cp -r ./airbyte-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-server + cp -r ./airbyte-api-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-api-server + cp -r ./airbyte-workers/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-workers + cp -r ./airbyte-bootloader/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-bootloader + cp -r ./airbyte-cron/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-cron + cp -r ./airbyte-connector-builder-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server + stage: + - airbyte-server + - airbyte-api-server + - airbyte-workers + - airbyte-bootloader + - airbyte-cron + - airbyte-connector-builder-server + + local-files: + plugin: dump + source: ./local-files + organize: + pod-sweeper.sh: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + stage: + - airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + permissions: + - path: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + owner: 584792 + group: 584792 + mode: "755" diff --git a/charmcraft.yaml b/charmcraft.yaml index e20d97e..564c6bc 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -448,28 +448,21 @@ resources: airbyte-api-server: type: oci-image description: OCI image for Airbyte API server - upstream-source: airbyte/airbyte-api-server:0.60.0 airbyte-bootloader: type: oci-image description: OCI image for Airbyte Bootloader - upstream-source: airbyte/bootloader:0.60.0 airbyte-connector-builder-server: type: oci-image description: OCI image for Airbyte Connector Builder Server - upstream-source: airbyte/connector-builder-server:0.60.0 airbyte-cron: type: oci-image description: OCI image for Airbyte Cron - upstream-source: airbyte/cron:0.60.0 airbyte-pod-sweeper: type: oci-image description: OCI image for Airbyte Pod Sweeper - upstream-source: bitnami/kubectl:1.29.4 airbyte-server: type: oci-image description: OCI image for Airbyte Server - upstream-source: airbyte/server:0.60.0 airbyte-workers: type: oci-image description: OCI image for Airbyte Worker - upstream-source: airbyte/worker:0.60.0 diff --git a/src/charm.py b/src/charm.py index 80e2412..3f1d565 100755 --- a/src/charm.py +++ b/src/charm.py @@ -5,7 +5,6 @@ """Charm the application.""" import logging -from pathlib import Path import ops from botocore.exceptions import ClientError @@ -53,7 +52,7 @@ def get_pebble_layer(application_name, context): "services": { application_name: { "summary": application_name, - "command": f"/bin/bash -c airbyte-app/bin/{application_name}", + "command": f"/bin/bash -c {application_name}/bin/{application_name}", "startup": "enabled", "override": "replace", # Including config values here so that a change in the @@ -293,18 +292,6 @@ def _update(self, event): event.defer() return - if container_name == "airbyte-pod-sweeper": - script_path = Path(__file__).parent / "scripts/pod-sweeper.sh" - - with open(script_path, "r") as file_source: - logger.info("pushing pod-sweeper script...") - container.push( - f"/airbyte-app/bin/{container_name}", - file_source, - make_dirs=True, - permissions=0o755, - ) - env = create_env(self.model.name, self.app.name, container_name, self.config, self._state) env = {k: v for k, v in env.items() if v is not None} pebble_layer = get_pebble_layer(container_name, env) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index cf09a4d..0a4ca83 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,6 +4,7 @@ """Charm integration test config.""" import logging +from pathlib import Path import pytest_asyncio from helpers import ( @@ -16,17 +17,27 @@ perform_temporal_integrations, run_sample_workflow, ) +from pytest import FixtureRequest from pytest_operator.plugin import OpsTest logger = logging.getLogger(__name__) +@pytest_asyncio.fixture(scope="module", name="charm") +async def charm_fixture(request: FixtureRequest, ops_test: OpsTest) -> str | Path: + """Fetch the path to charm.""" + charms = request.config.getoption("--charm-file") + if not charms: + charm = await ops_test.build_charm(".") + assert charm, "Charm not built" + return charm + return charms[0] + + @pytest_asyncio.fixture(name="deploy", scope="module") -async def deploy(ops_test: OpsTest): +async def deploy(ops_test: OpsTest, charm: str): """Test the app is up and running.""" await ops_test.model.set_config({"update-status-hook-interval": "1m"}) - - charm = await ops_test.build_charm(".") resources = get_airbyte_charm_resources() await ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME_AIRBYTE_SERVER, trust=True) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 9968c3f..058e839 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -348,7 +348,7 @@ def create_plan(container_name, storage_type): "services": { container_name: { "summary": container_name, - "command": f"/bin/bash -c airbyte-app/bin/{container_name}", + "command": f"/bin/bash -c {container_name}/bin/{container_name}", "startup": "enabled", "override": "replace", "environment": {