add airbyte rock

.licenserc.yaml

@@ -34,4 +34,5 @@ header:
     - 'LICENSE'
     - 'trivy.yaml'
     - 'lib/**'
+    - '**/*.patch'
   comment: on-failure

airbyte_rock/local-files/pod-sweeper.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# https://github.com/airbytehq/airbyte-platform/blob/main/charts/airbyte-pod-sweeper/templates/configmap.yaml
+# TODO(kelkawi-a): Move this to Airbyte ROCK
+
+get_job_pods() {
+    # echo "Running kubectl command to get job pods..."
+    kubectl -n "${JOB_KUBE_NAMESPACE}" -L airbyte -l airbyte=job-pod \
+        get pods \
+        -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}'
+}
+
+delete_pod() {
+    printf "From status '%s' since '%s', " "$2" "$3"
+    echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n "${JOB_KUBE_NAMESPACE}" delete pod
+}
+
+while :
+do
+    echo "Starting pod sweeper cycle:"
+
+    if [ -n "${RUNNING_TTL_MINUTES}" ]; then 
+        # Time window for running pods
+        RUNNING_DATE_STR=$(date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins)
+        RUNNING_DATE=$(date -d "${RUNNING_DATE_STR}" +%s)
+        echo "Will sweep running pods from before ${RUNNING_DATE_STR}"
+    fi
+
+    if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then 
+        # Shorter time window for succeeded pods
+        SUCCESS_DATE_STR=$(date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins)
+        SUCCESS_DATE=$(date -d "${SUCCESS_DATE_STR}" +%s)
+        echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}"
+    fi
+
+    if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then 
+        # Longer time window for unsuccessful pods (to debug)
+        NON_SUCCESS_DATE_STR=$(date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins)
+        NON_SUCCESS_DATE=$(date -d "${NON_SUCCESS_DATE_STR}" +%s)
+        echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}"
+    fi
+
+    echo "Running kubectl command to get job pods..."
+    get_job_pods | while read -r POD; do
+        IFS=' ' read -r POD_NAME POD_STATUS POD_DATE_STR POD_START_DATE_STR <<< "$POD"
+
+        POD_DATE=$(date -d "${POD_DATE_STR:-$POD_START_DATE_STR}" '+%s')
+        echo "Evaluating pod: $POD_NAME with status $POD_STATUS since $POD_DATE_STR"
+
+        if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then
+            if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then
+                delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"
+            fi
+        elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && { [[ "$POD_STATUS" = "Succeeded" ]] || [[ "$POD_STATUS" = "Completed" ]]; }; then
+            if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then
+                delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"
+            fi
+        elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then
+            if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then
+                delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"
+            fi
+        fi
+    done
+
+    echo "Completed pod sweeper cycle. Sleeping for 60 seconds..."
+    sleep 60
+done

airbyte_rock/patches/db-jooq.patch

@@ -0,0 +1,31 @@
+diff --git a/airbyte-db/jooq/build.gradle.kts b/airbyte-db/jooq/build.gradle.kts
+index 6a4b2d6138..d031b1bb3e 100644
+--- a/airbyte-db/jooq/build.gradle.kts
++++ b/airbyte-db/jooq/build.gradle.kts
+@@ -15,7 +15,7 @@ dependencies {
+
+   // jOOQ code generation)
+   implementation(libs.jooq.codegen)
+-  implementation(libs.platform.testcontainers.postgresql)
++  // implementation(libs.platform.testcontainers.postgresql)
+
+   // These are required because gradle might be using lower version of Jna from other
+   // library transitive dependency. Can be removed if we can figure out which library is the cause.
+@@ -50,7 +50,7 @@ jooq {
+
+   configurations {
+     create("configsDatabase") {
+-      generateSchemaSourceOnCompilation = true
++      generateSchemaSourceOnCompilation = false
+       jooqConfiguration.apply {
+         generator.apply {
+           name = "org.jooq.codegen.DefaultGenerator"
+@@ -68,7 +68,7 @@ jooq {
+     }
+
+     create("jobsDatabase") {
+-      generateSchemaSourceOnCompilation = true
++      generateSchemaSourceOnCompilation = false
+       jooqConfiguration.apply {
+         generator.apply {
+           name = "org.jooq.codegen.DefaultGenerator"

airbyte_rock/rockcraft.yaml

@@ -0,0 +1,165 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+name: airbyte
+summary: Airbyte rock
+description: Airbyte OCI image for the Airbyte charm
+version: "1.0"
+base: [email protected]
+build-base: [email protected]
+license: Apache-2.0
+platforms:
+  amd64:
+
+# Please refer to
+# https://discourse.ubuntu.com/t/unifying-user-identity-across-snaps-and-rocks/36469
+# for more information about shared user.
+# The UID 584792 corresponds to _daemon_ user.
+run_user: _daemon_
+
+services:
+  airbyte-server:
+    override: replace
+    summary: "airbyte-server service"
+    startup: enabled
+    command: "/bin/bash -c airbyte-server/bin/airbyte-server"
+  airbyte-workers:
+    override: replace
+    summary: "airbyte-workers service"
+    startup: enabled
+    command: "/bin/bash -c airbyte-workers/bin/airbyte-workers"
+  airbyte-api-server:
+    override: replace
+    summary: "airbyte-api-server service"
+    startup: enabled
+    command: "/bin/bash -c airbyte-api-server/bin/airbyte-api-server"
+  airbyte-bootloader:
+    override: replace
+    summary: "airbyte-bootloader service"
+    startup: enabled
+    command: "/bin/bash -c airbyte-bootloader/bin/airbyte-bootloader"
+  airbyte-connector-builder-server:
+    override: replace
+    summary: "airbyte-connector-builder-server service"
+    startup: enabled
+    command: "/bin/bash -c airbyte-connector-builder-server/bin/airbyte-connector-builder-server"
+  airbyte-cron:
+    override: replace
+    summary: "airbyte-cron service"
+    startup: enabled
+    command: "/bin/bash -c airbyte-cron/bin/airbyte-cron"
+  airbyte-pod-sweeper:
+    override: replace
+    summary: "airbyte-pod-sweeper service"
+    startup: enabled
+    command: "/bin/bash -c airbyte-pod-sweeper/bin/airbyte-pod-sweeper"
+
+environment:
+  JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64
+
+
+parts:
+  # patches:
+  #   plugin: dump
+  #   source: ./patches
+  #   organize:
+  #     db-jooq.patch: patches/db-jooq.patch
+  #   stage:
+  #     - patches/db-jooq.patch
+  #   prime:
+  #     - "-*"
+
+  # assemble:
+  #   plugin: dump
+  #   source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line
+  #   source-type: git
+  #   source-tag: v0.60.0
+  #   stage:
+  #     - '*'
+
+  db-lib:
+    plugin: nil
+    source: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-db/db-lib/0.60.0/db-lib-0.60.0.jar
+    source-checksum: sha512/5417e491007a80276a51a81c49815da51ecf855eb243d5c48c4ddce75cb330c2930d6dd5100a24d6185e1d1067bf8c176d28d7aadc62d21db078b62327aa77e8
+    source-type: zip
+    stage:
+      - '*'
+
+  jooq:
+    plugin: nil
+    source: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-db/jooq/0.60.0/jooq-0.60.0.jar
+    source-checksum: sha512/da09c7f4e127af269e6733baa68e3b9ffccd85eadead6d32e441e9b5cb014056720e23039c76914e242ef9ee22d607d20983ffb71b12e1f2748e6630e9bbda86
+    source-type: zip
+    stage:
+      - '*'
+
+  assemble:
+    # after: [patches]
+    plugin: dump
+    source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line
+    source-type: git
+    source-tag: v0.60.0
+    build-packages:
+      - jq
+      - curl
+      - coreutils
+      - bash
+      - gradle
+      - openjdk-21-jdk-headless
+    stage-packages:
+      - openjdk-21-jdk-headless
+    override-build: |
+      # git apply ${CRAFT_STAGE}/patches/*.patch
+      # ./gradlew :airbyte-db:jooq:compileJava
+      ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace --info
+
+      # ./gradlew :airbyte-server:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq
+      # ./gradlew :airbyte-api-server:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq
+      # ./gradlew :airbyte-workers:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq
+      # ./gradlew :airbyte-bootloader:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq
+      # ./gradlew :airbyte-cron:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq
+      # ./gradlew :airbyte-connector-builder-server:assemble --exclude-task dockerBuildImage --exclude-task :airbyte-db:jooq:generateConfigsDatabaseJooq --exclude-task :airbyte-db:jooq:generateJobsDatabaseJooq
+
+
+      tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar
+      tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar
+      tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar
+      tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar
+      tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar
+      tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar
+
+      cp -r ./airbyte-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-server
+      cp -r ./airbyte-api-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-api-server
+      cp -r ./airbyte-workers/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-workers
+      cp -r ./airbyte-bootloader/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-bootloader
+      cp -r ./airbyte-cron/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-cron
+      cp -r ./airbyte-connector-builder-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server
+    stage:
+      - airbyte-server
+      - airbyte-api-server
+      - airbyte-workers
+      - airbyte-bootloader
+      - airbyte-cron
+      - airbyte-connector-builder-server
+
+  local-files:
+    plugin: dump
+    source: ./local-files
+    organize:
+      pod-sweeper.sh: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh
+    stage:
+      - airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh
+    permissions:
+      - path: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh
+        owner: 584792
+        group: 584792
+        mode: "755"
+
+              # ./gradlew distTar -x dockerBuildImage
+      # ./gradlew distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq
+      # ./gradlew :airbyte-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin
+      # ./gradlew :airbyte-api-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin
+      # ./gradlew :airbyte-workers:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin
+      # ./gradlew :airbyte-bootloader:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin
+      # ./gradlew :airbyte-cron:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin
+      # ./gradlew :airbyte-connector-builder-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin

charmcraft.yaml

@@ -448,28 +448,21 @@ resources:
   airbyte-api-server:
     type: oci-image
     description: OCI image for Airbyte API server
-    upstream-source: airbyte/airbyte-api-server:0.60.0
   airbyte-bootloader:
     type: oci-image
     description: OCI image for Airbyte Bootloader
-    upstream-source: airbyte/bootloader:0.60.0
   airbyte-connector-builder-server:
     type: oci-image
     description: OCI image for Airbyte Connector Builder Server
-    upstream-source: airbyte/connector-builder-server:0.60.0
   airbyte-cron:
     type: oci-image
     description: OCI image for Airbyte Cron
-    upstream-source: airbyte/cron:0.60.0
   airbyte-pod-sweeper:
     type: oci-image
     description: OCI image for Airbyte Pod Sweeper
-    upstream-source: bitnami/kubectl:1.29.4
   airbyte-server:
     type: oci-image
     description: OCI image for Airbyte Server
-    upstream-source: airbyte/server:0.60.0
   airbyte-workers:
     type: oci-image
     description: OCI image for Airbyte Worker
-    upstream-source: airbyte/worker:0.60.0

src/charm.py

@@ -5,7 +5,6 @@
 """Charm the application."""
 
 import logging
-from pathlib import Path
 
 import ops
 from botocore.exceptions import ClientError
@@ -53,7 +52,7 @@ def get_pebble_layer(application_name, context):
         "services": {
             application_name: {
                 "summary": application_name,
-                "command": f"/bin/bash -c airbyte-app/bin/{application_name}",
+                "command": f"/bin/bash -c {application_name}/bin/{application_name}",
                 "startup": "enabled",
                 "override": "replace",
                 # Including config values here so that a change in the
@@ -293,18 +292,6 @@ def _update(self, event):
                 event.defer()
                 return
 
-            if container_name == "airbyte-pod-sweeper":
-                script_path = Path(__file__).parent / "scripts/pod-sweeper.sh"
-
-                with open(script_path, "r") as file_source:
-                    logger.info("pushing pod-sweeper script...")
-                    container.push(
-                        f"/airbyte-app/bin/{container_name}",
-                        file_source,
-                        make_dirs=True,
-                        permissions=0o755,
-                    )
-
             env = create_env(self.model.name, self.app.name, container_name, self.config, self._state)
             env = {k: v for k, v in env.items() if v is not None}
             pebble_layer = get_pebble_layer(container_name, env)

tests/integration/conftest.py

@@ -4,6 +4,7 @@
 """Charm integration test config."""
 
 import logging
+from pathlib import Path
 
 import pytest_asyncio
 from helpers import (
@@ -16,17 +17,27 @@
     perform_temporal_integrations,
     run_sample_workflow,
 )
+from pytest import FixtureRequest
 from pytest_operator.plugin import OpsTest
 
 logger = logging.getLogger(__name__)
 
 
+@pytest_asyncio.fixture(scope="module", name="charm")
+async def charm_fixture(request: FixtureRequest, ops_test: OpsTest) -> str | Path:
+    """Fetch the path to charm."""
+    charms = request.config.getoption("--charm-file")
+    if not charms:
+        charm = await ops_test.build_charm(".")
+        assert charm, "Charm not built"
+        return charm
+    return charms[0]
+
+
 @pytest_asyncio.fixture(name="deploy", scope="module")
-async def deploy(ops_test: OpsTest):
+async def deploy(ops_test: OpsTest, charm: str):
     """Test the app is up and running."""
     await ops_test.model.set_config({"update-status-hook-interval": "1m"})
-
-    charm = await ops_test.build_charm(".")
     resources = get_airbyte_charm_resources()
 
     await ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME_AIRBYTE_SERVER, trust=True)

tests/unit/test_charm.py

@@ -348,7 +348,7 @@ def create_plan(container_name, storage_type):
         "services": {
             container_name: {
                 "summary": container_name,
-                "command": f"/bin/bash -c airbyte-app/bin/{container_name}",
+                "command": f"/bin/bash -c {container_name}/bin/{container_name}",
                 "startup": "enabled",
                 "override": "replace",
                 "environment": {