#140 Changed the testing workflow

.github/workflows/build.yaml

@@ -18,29 +18,9 @@ jobs:
     environment: aws
 
     steps:
-      - name: Set pytest markers
-        id: pytest-markers
-        if: ${{ ! inputs.slow-tests }}
-        run: echo slow-tests='-m "not slow"' >> "$GITHUB_OUTPUT"
-
       - name: SCM Checkout
         uses: actions/checkout@v4
 
-      # - name: Free disk space
-      #   uses: jlumbroso/free-disk-space@main
-      #   with:
-      #     tool-cache: true
-      #     large-packages: false
-
-      # - name: Free disk space by removing large directories
-      #   run: |
-      #     sudo rm -rf /usr/local/graalvm/
-      #     sudo rm -rf /usr/local/.ghcup/
-      #     sudo rm -rf /usr/local/share/powershell
-      #     sudo rm -rf /usr/local/share/chromium
-      #     sudo rm -rf /usr/local/lib/node_modules
-      #     sudo rm -rf /opt/ghc
-
       - name: Setup Python & Poetry Environment
         uses: ./.github/actions/prepare_poetry_env
         with:
@@ -53,17 +33,25 @@ jobs:
         run: pip install dist/*.whl
 
       - name:  Install Lua dependencies
+        if: ${{ ! inputs.slow-tests }}
         run: |
           sudo apt-get install luarocks
           sudo luarocks install --only-deps *.rockspec
 
       - name: Run Lua unit tests and static code analyzer
+        if: ${{ ! inputs.slow-tests }}
         run: poetry run poe lua-tests
 
-      # - name: Setup integration test environment
-      #   run: ./scripts/setup_integration_test.sh
+      - name: Run unit tests
+        if: ${{ ! inputs.slow-tests }}
+        run: poetry run pytest tests/unit_tests
 
-      - name: Poetry run pytest integration tests
+      - name: Run integration tests
+        if: ${{ ! inputs.slow-tests }}
+        run: poetry run pytest --backend=all tests/integration_tests
+
+      - name: Run ci tests
+        if: ${{ inputs.slow-tests }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -72,7 +60,4 @@ jobs:
           SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
           SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
           SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
-          PYTEST_ADDOPTS: '${{ steps.pytest-markers.outputs.slow-tests }}'
-        run: |
-          echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS"
-          poetry run pytest --backend=onprem --itde-nameserver="8.8.8.8" tests
+        run: poetry run pytest --backend=all --itde-nameserver="8.8.8.8" tests/ci_tests

.github/workflows/release_droid_upload_github_release_assets.yml

@@ -31,7 +31,7 @@ jobs:
         run: pip install dist/*.whl
 
       - name: Release language container
-        run: ./scripts/release_language_container.sh "$RELEASE_DIR"
+        run: poetry run python3 -u "./exasol_sagemaker_extension/deployment/language_container.py" "$RELEASE_DIR"
 
       - name: Upload assets to the GitHub release draft
         uses: shogo82148/actions-upload-release-asset@v1

exasol_sagemaker_extension/deployment/language_container.py

@@ -1,4 +1,5 @@
 import sys
+import os
 from contextlib import contextmanager
 
 from exasol.python_extension_common.deployment.language_container_builder import (
@@ -16,5 +17,12 @@ def language_container_factory():
 
 
 def export_slc():
+    export_dir = sys.argv[1]
+    if not os.path.isdir(export_dir):
+        os.makedirs(export_dir)
     with language_container_factory() as container_builder:
-        container_builder.export(sys.argv[1])
+        container_builder.export(export_dir)
+
+
+if __name__ == '__main__':
+    export_slc()

pyproject.toml

@@ -19,9 +19,6 @@ keywords = ['exasol', 'sagemaker']
 
 include = ["lua", "resources", "deployment"]
 
-[tool.poetry.scripts]
-export_slc = "exasol_sagemaker_extension.deployment.language_container:export_slc"
-
 [tool.poetry.dependencies]
 python = ">=3.10,<4.0"
 pandas = ">=1.4.2,<2.0.0"

tests/conftest.py → tests/ci_tests/conftest.py

@@ -1,7 +1,5 @@
 pytest_plugins = [
-    "tests.fixtures.aws_connection_fixture",
     "tests.fixtures.language_container_fixture",
-    "tests.fixtures.localstack_fixture",
     "tests.fixtures.setup_database_fixture",
     "tests.fixtures.prepare_environment_fixture",
     "tests.fixtures.script_deployment_fixture",

tests/ci_tests/test_deploying_autopilot.py

@@ -1,8 +1,6 @@
 import time
 from datetime import datetime
 
-import pytest
-
 from tests.fixtures.prepare_environment_fixture import CITestEnvironment
 from tests.ci_tests.utils import parameters
 from tests.ci_tests.utils.autopilot_deployment import AutopilotTestDeployment
@@ -53,7 +51,6 @@ def _deploy_endpoint(job_name, endpoint_name, model_setup_params, ci_test_env: C
     assert endpoint_name in list(map(lambda x: x[0], all_scripts))
 
 
-@pytest.mark.slow
 def test_deploy_autopilot_endpoint(prepare_ci_test_environment):
     curr_datetime = datetime.now().strftime("%y%m%d%H%M%S")
     model_name = ''.join((cls_model_setup_params.model_type, curr_datetime))

tests/ci_tests/test_polling_autopilot.py

@@ -1,13 +1,10 @@
 from datetime import datetime
 
-import pytest
-
 from tests.ci_tests.utils.autopilot_polling import AutopilotTestPolling
 from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining
 from tests.ci_tests.utils.parameters import cls_model_setup_params
 
 
-@pytest.mark.slow
 def test_poll_autopilot_job(prepare_ci_test_environment):
     curr_datetime = datetime.now().strftime("%y%m%d%H%M%S")
     model_name = ''.join((cls_model_setup_params.model_type, curr_datetime))

tests/ci_tests/test_predicting_autopilot.py

@@ -1,8 +1,6 @@
 import time
 from datetime import datetime
 
-import pytest
-
 from tests.fixtures.prepare_environment_fixture import CITestEnvironment
 from tests.ci_tests.utils import parameters
 from tests.ci_tests.utils.autopilot_deployment import AutopilotTestDeployment
@@ -54,7 +52,6 @@ def _make_prediction(job_name, endpoint_name, model_setup_params, ci_test_env: C
     assert predictions
 
 
-@pytest.mark.slow
 def test_predict_autopilot_regression_job(prepare_ci_test_environment):
     curr_datetime = datetime.now().strftime("%y%m%d%H%M%S")
     model_name = ''.join((reg_model_setup_params.model_type, curr_datetime))
@@ -73,7 +70,6 @@ def test_predict_autopilot_regression_job(prepare_ci_test_environment):
         db_conn=prepare_ci_test_environment)
 
 
-@pytest.mark.slow
 def test_predict_autopilot_classification_job(prepare_ci_test_environment):
     curr_datetime = datetime.now().strftime("%y%m%d%H%M%S")
     model_name = ''.join((cls_model_setup_params.model_type, curr_datetime))

tests/ci_tests/test_training_autopilot.py

@@ -1,15 +1,12 @@
 from datetime import datetime
 
-import pytest
-
 from tests.fixtures.prepare_environment_fixture import CITestEnvironment
 from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining
 from tests.ci_tests.utils.parameters import reg_model_setup_params, \
     cls_model_setup_params
 from tests.ci_tests.utils.queries import DatabaseQueries
 
 
-@pytest.mark.slow
 def test_train_autopilot_regression_job(prepare_ci_test_environment):
     curr_datetime = datetime.now().strftime("%y%m%d%H%M%S")
     model_name = ''.join((reg_model_setup_params.model_type, curr_datetime))
@@ -23,7 +20,6 @@ def test_train_autopilot_regression_job(prepare_ci_test_environment):
         job_name, reg_model_setup_params, prepare_ci_test_environment)
 
 
-@pytest.mark.slow
 def test_train_autopilot_classification_job(prepare_ci_test_environment):
     curr_datetime = datetime.now().strftime("%y%m%d%H%M%S")
     model_name = ''.join((cls_model_setup_params.model_type, curr_datetime))

tests/integration_tests/conftest.py

@@ -0,0 +1,4 @@
+pytest_plugins = [
+    "tests.fixtures.localstack_fixture",
+    "tests.fixtures.script_deployment_fixture",
+]

tests/integration_tests/utils/parameters.py

@@ -25,21 +25,6 @@ def get_aws_params():
     )
 
 
-def get_db_params():
-    DBParams = namedtuple("DBParams", [
-        "host", "port", "bfs_port", "user", "bfs_user", "password", "bfs_password"])
-
-    return DBParams(
-        host="127.0.0.1",
-        port="9563",
-        bfs_port="6666",
-        user="sys",
-        bfs_user="w",
-        password="exasol",
-        bfs_password="write"
-    )
-
-
 def get_regression_setup_params():
     RegressionSetupParams = namedtuple("RegressionSetupParams", [
         "schema_name", "table_name", "target_col", "data", "aws_output_path",
@@ -75,6 +60,5 @@ def get_classification_setup_params():
 
 
 aws_params = get_aws_params()
-db_params = get_db_params()
 reg_setup_params = get_regression_setup_params()
 cls_setup_params = get_classification_setup_params()

tests/unit_tests/conftest.py

@@ -0,0 +1,3 @@
+pytest_plugins = [
+    "tests.fixtures.aws_connection_fixture",
+]