Add sonataflow postgres db migrator tool for data index and jobs serv…

…ice (apache#517)

.github/workflows/db-migrator-tests.yml

@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Sonataflow DB Migrator Unit Tests
+
+on:
+  pull_request:
+    types: [opened, reopened, edited, synchronize]
+    branches:
+      - main
+    paths:
+      - 'images/tools/sonataflow-db-migrator/**'
+      - '!images/tools/sonataflow-db-migrator/README.md'
+      - '.github/workflows/db-migrator-tests.yml'
+
+jobs:
+  unit-tests:
+    strategy:
+      matrix:
+        OS: [ubuntu-latest]
+        JAVA_VERSION: [17]
+        MAVEN_VERSION: [3.9.8]
+    timeout-minutes: 10
+    name: Unit Tests
+    runs-on: ${{ matrix.OS }}
+    steps:
+      - name: Java and Maven Setup
+        uses: apache/incubator-kie-kogito-pipelines/.ci/actions/maven@main
+        with:
+          java-version: ${{ matrix.JAVA_VERSION }}
+          maven-version: ${{ matrix.MAVEN_VERSION }}
+          cache-key-prefix: ${{ matrix.OS }}-${{ matrix.JAVA_VERSION }}-maven${{ matrix.MAVEN_VERSION }}
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: |
+          cd images/tools/sonataflow-db-migrator
+          mvn test

.gitignore

@@ -29,4 +29,5 @@ Dockerfile
 /target/
 
 database/index.db
-e2e-test-report.xml
+e2e-test-report*.xml
+*.tar

Makefile

@@ -338,7 +338,9 @@ generate-all: generate generate-deploy bundle addheaders vet fmt
 
 .PHONY: test-e2e # You will need to have a Minikube/Kind cluster up in running to run this target, and run container-builder before the test
 test-e2e:
-	go test ./test/e2e/* -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report.xml -timeout 120m
+	go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/clusterplatform_test.go  -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report-clusterplatform_test.xml -timeout 60m;
+	go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/platform_test.go  -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report-platform_test.xml -timeout 60m;
+	go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/workflow_test.go  -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report-workflow_test.xml -timeout 60m;
 
 .PHONY: before-pr
 before-pr: test generate-all

hack/bump-version.sh

@@ -23,13 +23,20 @@ source "${script_dir_path}"/env.sh
 imageTag='docker.io/apache/incubator-kie-sonataflow-operator'
 # shellcheck disable=SC2034
 old_version=$(getImageTagVersion)
+old_operator_version=$(getOperatorVersion)
 new_version=$1
+quarkus_version=$2
 
 if [ -z "${new_version}" ]; then
   echo "Please inform the new version"
   exit 1
 fi
 
+if [ -z "${quarkus_version}" ]; then
+  quarkus_version="3.13.0"
+  echo "Warning: The Quarkus version is not supplied, defaulting to version: $quarkus_version"
+fi
+
 oldMajorMinorVersion=${old_version%.*}
 newMajorMinorVersion=${new_version%.*}
 
@@ -48,6 +55,13 @@ sed -i -r "s|operatorVersion =.*|operatorVersion = \"${new_version}\"|g" version
 
 sed -i "s|containerImage:.*|containerImage: ${imageTag}:${newMajorMinorVersion}|g" $(getCsvFile)
 
+# Begin: Sonataflow DB Migrator tool
+sed -i "s|OPERATOR_VERSION=${old_operator_version}|OPERATOR_VERSION=${new_version}|g" images/tools/sonataflow-db-migrator/build-container-image.sh
+sed -i "s|OPERATOR_VERSION=${old_operator_version}|OPERATOR_VERSION=${new_version}|g" images/tools/sonataflow-db-migrator/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh
+sed -i "s|<version>${old_operator_version}</version>|<version>${new_version}</version>|g" images/tools/sonataflow-db-migrator/pom.xml
+sed -i "s|<quarkus.platform.version>.*</quarkus.platform.version>|<quarkus.platform.version>${quarkus_version}</quarkus.platform.version>|g" images/tools/sonataflow-db-migrator/pom.xml
+# End: Sonataflow DB Migrator tool
+
 make generate-all
 make vet
 

images/tools/sonataflow-db-migrator/.dockerignore

@@ -0,0 +1,5 @@
+*
+!target/*-runner
+!target/*-runner.jar
+!target/lib/*
+!target/quarkus-app/*

images/tools/sonataflow-db-migrator/.gitignore

@@ -0,0 +1,44 @@
+#Maven
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+release.properties
+.flattened-pom.xml
+
+# Eclipse
+.project
+.classpath
+.settings/
+bin/
+
+# IntelliJ
+.idea
+*.ipr
+*.iml
+*.iws
+
+# NetBeans
+nb-configuration.xml
+
+# Visual Studio Code
+.vscode
+.factorypath
+
+# OSX
+.DS_Store
+
+# Vim
+*.swp
+*.swo
+
+# patch
+*.orig
+*.rej
+
+# Local environment
+.env
+
+# Plugin directory
+/.quarkus/cli/plugins/
+*.iml

images/tools/sonataflow-db-migrator/README.md

@@ -0,0 +1,56 @@
+# sonataflow-db-migrator
+
+This is a quarkus postgres database migrator application for Sonataflow Data Index and Jobs Service applications.
+
+## Running the application in dev mode
+Though you can run the application locally in dev mode but it is advisable to run this application as a container image as described in the next section. 
+The primary reason not to run as standalone application in dev mode, is that by default there are no DDL migration files included in the source. 
+However the DDL files are dynamically included from its respective sources when a container image is created.
+
+You can run your application in dev mode that enables live coding using:
+
+```shell script
+./mvnw compile quarkus:dev
+```
+
+## Build and Run container image locally
+You can build the cekit container image by using the provided image builder shell script
+```shell
+./build-container-image.sh
+```
+Ensure the script completes without errors.
+If you may have a cekit specific Python virtual environment, be sure to activate it, so the script can find the cekit command.
+```shell
+virtualenv ~/cekit
+source ~/cekit/bin/activate;
+```
+
+Assuming you have a Postgres database running locally, e.g., a `di` database for data index and a `js` database for jobs service, you can run the image with the following command. Substitute appropriate values:
+```shell
+podman run \ 
+--env MIGRATE_DB_DATAINDEX=true \
+--env QUARKUS_DATASOURCE_DATAINDEX_JDBC_URL=<data-index-db-url e.g. jdbc:postgresql://host.docker.internal:5432/di> \
+--env QUARKUS_DATASOURCE_DATAINDEX_USERNAME=<data-index-db-user> \ 
+--env QUARKUS_DATASOURCE_DATAINDEX_PASSWORD=<data-index-db-password> \
+--env QUARKUS_FLYWAY_DATAINDEX_SCHEMAS=dataindex \
+--env MIGRATE_DB_JOBSSERVICE=true \
+--env QUARKUS_DATASOURCE_JOBSSERVICE_JDBC_URL=<jobs-service-db-url e.g. jdbc:postgresql://host.docker.internal:5432/js> \
+--env QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME=<jobs-service-db-user> \
+--env QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD=<jobs-service-db-password> \
+--env QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS=jobsservice \
+docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql:999-SNAPSHOT
+```
+
+### Environment variables
+| NAME  | DESCRIPTION  | DEFAULT |
+|---|---|---|
+| MIGRATE_DB_DATAINDEX  |  Set to true if you want to migrate data index database, set to false otherwise | false |
+| QUARKUS_DATASOURCE_DATAINDEX_JDBC_URL  | Data index database url  e.g. jdbc:postgresql://host.docker.internal:5432/di|  jdbc:postgresql://localhost:5432/postgres |
+| QUARKUS_DATASOURCE_DATAINDEX_USERNAME  | Data index database username|  postgres |
+| QUARKUS_DATASOURCE_DATAINDEX_PASSWORD  | Data index database password|  postgres |
+| QUARKUS_FLYWAY_DATAINDEX_SCHEMAS  | Data index database schema|  dataindex |
+| MIGRATE_DB_JOBSSERVICE  |  Set to true if you want to migrate jobs service database, set to false otherwise | false |
+| QUARKUS_DATASOURCE_JOBSSERVICE_JDBC_URL  | Jobs service database url  e.g. jdbc:postgresql://host.docker.internal:5432/js|  jdbc:postgresql://localhost:5432/postgres |
+| QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME  | Jobs service database username|  postgres |
+| QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD  | Jobs service database password|  postgres |
+| QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS  | Jobs service database schema|  jobsservice |

images/tools/sonataflow-db-migrator/build-container-image.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -x
+set -e
+
+# cleanup temporary files
+cleanup () {
+    echo "Cleaning up"
+    rm -rf target
+    rm -rf src/main/resources/postgresql
+    rm -rf tmp
+    rm -f src/main/cekit/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar
+}
+
+# Script variables with default values. These values will be updated by hack/bump-version.sh, don't change it.
+OPERATOR_VERSION=999.0.0 # comes from version.go
+DDL_VERSION=10.0.999-SNAPSHOT
+DDL_FILE=kogito-ddl-10.0.999-20240806.011718-23-db-scripts.zip
+DDL_BASE_URL=https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl
+CEKIT_BUILDER=podman
+
+# Use specific variable values, if passed
+for arg in "$@"; do
+   case "$arg" in
+      OPERATOR_VERSION=*) OPERATOR_VERSION="${arg#*=}" ;;
+      DDL_VERSION=*) DDL_VERSION="${arg#*=}" ;;
+      DDL_FILE=*) DDL_FILE="${arg#*=}" ;;
+      DDL_BASE_URL=*) DDL_BASE_URL="${arg#*=}" ;;
+      CEKIT_BUILDER=*) CEKIT_BUILDER="${arg#*=}" ;;
+   esac
+done
+
+DDL_URL=$DDL_BASE_URL/$DDL_VERSION/$DDL_FILE
+
+printf "Variables being used for the image: \n OPERATOR_VERSION: %s, \n DDL_VERSION: %s, \n DDL_FILE: %s, \n DDL_BASE_URL: %s, \n CEKIT_BUILDER=%s\n\n" "$OPERATOR_VERSION" "$DDL_VERSION" "$DDL_FILE" "$DDL_BASE_URL" "$CEKIT_BUILDER"
+
+# Start with cleanup
+cleanup "$OPERATOR_VERSION"
+
+# Get Data Index/ Jobs Service DDL Files
+mkdir -p tmp
+# Change the variables below, as needed
+wget "$DDL_URL"
+mv "$DDL_FILE" tmp
+cd tmp || exit
+unzip "$DDL_FILE"
+mv ./postgresql ../src/main/resources
+cd .. || exit
+
+# Create an Uber jar
+mvn package -Dquarkus.package.jar.type=uber-jar
+cp target/sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar src/main/cekit/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar
+
+# Build the container image
+cd src/main/cekit || exit
+cekit -v build "$CEKIT_BUILDER"
+
+# Cleanup
+cd ../../.. || exit
+cleanup "$OPERATOR_VERSION"