#120 Running tests in SaaS [run aws tests] (#131)

* #120 Running tests in SaaS [run aws tests] * #120 Running tests in SaaS [run aws tests] * #120 Running tests in SaaS [run aws tests] * #120 Running tests in SaaS [run aws tests] * #120 Running tests in SaaS [run aws tests] * #120 Running tests in SaaS [run aws tests] * #120 Refactored integration tests * #120 Refactored integration tests * #120 Fixed the deployment tests * #120 Fixed the deployment tests * #120 Added debugging * #120 Added debugging * #120 Added debugging * #120 I don't know what is wrong * #120 Tried to use the newest version of the ITDE * #120 Fixed the db setup * #120 Fixed the arg list of the script deployer * #120 Fixed the arg list of the script deployer * #120 All tests are enabled * #120 Moved pyexasol connect to the python-extension-common * #120 Added the backend fixture * #120 Rolled back changes in integration tests * #120 Changed arg_list creation * #120 Changed arg_list creation * #120 Added delay after uploading SLC to SaaS * #120 Made the SaaS waiting time longer. * #120 Made the SaaS waiting time longer. * #120 Removed double underscore * #120 Running tests in SaaS only * #120 Enabled all tests again * #120 Disabled all tests except script deployment * #120 Enabled all tests * #120 Removed debugging prints * #120 Print output of the dodgy cli test before it fails
exasol · Jul 3, 2024 · e425408 · e425408
1 parent 45fd300
commit e425408
Show file tree

Hide file tree

Showing 37 changed files with 453 additions and 447 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -35,18 +35,12 @@ jobs:
         run: ./scripts/setup_integration_test.sh
 
       - name: Poetry run pytest integration tests
-        run: poetry run pytest tests
-
-      - name: Fail, if AWS-tests are not activated
-        if: "!contains(github.event.head_commit.message, '[run aws tests]')"
-        run: |
-          echo "Failed, because AWS-tests are not activated"
-          exit 1
-      - name: Run AWS-tests if it is activated
-        if: "contains(github.event.head_commit.message, '[run aws tests]')"
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_ROLE: ${{ secrets.AWS_ROLE }}
           AWS_DEFAULT_REGION: "eu-central-1"
-        run: poetry run pytest tests/ci_tests -s
+          SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }}
+          SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
+          SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
+        run: poetry run pytest tests
diff --git a/doc/changes/changes_0.10.0.md b/doc/changes/changes_0.10.0.md
@@ -1,4 +1,4 @@
-# SageMaker Extension 0.10.0, released 2024-06-20
+# SageMaker Extension 0.10.0, T.B.D.
 
 Code name: Python 3.10
 
@@ -8,7 +8,7 @@ Improving documentation, refactoring and moving to Python 3.10
 
 ### Features
 
-n/a
+- #120: Running CI tests in the SaaS as well as the Docker DB.
 
 ### Bug Fixes
 

diff --git a/exasol_sagemaker_extension/deployment/deploy_cli.py b/exasol_sagemaker_extension/deployment/deploy_cli.py
@@ -1,34 +1,77 @@
+from __future__ import annotations
+
 import click
 import logging
+from exasol.python_extension_common.deployment.language_container_deployer_cli import (
+    SecretParams, SECRET_DISPLAY, secret_callback)
+
 from exasol_sagemaker_extension.deployment.deploy_create_statements \
     import DeployCreateStatements
 
 
 @click.command()
-@click.option('--host', type=str, required=True, help="db host address")
-@click.option('--port', type=str, required=True, help="db host port")
-@click.option('--user', type=str, required=True, help="db user name")
-@click.option('--pass', 'pwd', required=True, help="db user password")
+@click.option('--dsn', type=str)
+@click.option('--db-user', type=str)
+@click.option(f'--{SecretParams.DB_PASSWORD.value}', type=str,
+              prompt='DB password', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option('--saas-url', type=str,
+              default='https://cloud.exasol.com')
+@click.option(f'--{SecretParams.SAAS_ACCOUNT_ID.value}', type=str,
+              prompt='SaaS account id', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option(f'--{SecretParams.SAAS_DATABASE_ID.value}', type=str,
+              prompt='SaaS database id', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option('--saas-database-name', type=str)
+@click.option(f'--{SecretParams.SAAS_TOKEN.value}', type=str,
+              prompt='SaaS token', prompt_required=False,
+              hide_input=True, default=SECRET_DISPLAY, callback=secret_callback)
+@click.option('--ssl-cert-path', type=str, default="")
+@click.option('--ssl-client-cert-path', type=str, default="")
+@click.option('--ssl-client-private-key', type=str, default="")
+@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True)
 @click.option('--schema', type=str, required=True, help="schema name")
-@click.option('--print', 'verbose', type=bool, required=False,
+@click.option('--print', 'verbose', type=bool, default=False,
               is_flag=True, help="print out statements")
-@click.option('--develop', type=bool, required=False,
+@click.option('--develop', type=bool, default=False,
               is_flag=True, help="generate and execute the scripts")
-def main(host: str, port: str, user: str, pwd: str, schema: str,
-         verbose: bool = False, develop: bool = False):
+def main(
+        dsn: str | None,
+        db_user: str | None,
+        db_pass: str | None,
+        saas_url: str | None,
+        saas_account_id: str | None,
+        saas_database_id: str | None,
+        saas_database_name: str | None,
+        saas_token: str | None,
+        ssl_cert_path: str,
+        ssl_client_cert_path: str,
+        ssl_client_private_key: str,
+        use_ssl_cert_validation: bool,
+        schema: str,
+        verbose: bool,
+        develop: bool):
 
     logging.basicConfig(format='%(asctime)s - %(module)s  - %(message)s',
                         level=logging.DEBUG)
 
     DeployCreateStatements.create_and_run(
-        db_host=host,
-        db_port=port,
-        db_user=user,
-        db_pass=pwd,
         schema=schema,
+        dsn=dsn,
+        db_user=db_user,
+        db_pass=db_pass,
+        saas_url=saas_url,
+        saas_account_id=saas_account_id,
+        saas_database_id=saas_database_id,
+        saas_database_name=saas_database_name,
+        saas_token=saas_token,
+        use_ssl_cert_validation=use_ssl_cert_validation,
+        ssl_trusted_ca=ssl_cert_path,
+        ssl_client_certificate=ssl_client_cert_path,
+        ssl_private_key=ssl_client_private_key,
         to_print=verbose,
-        develop=develop
-    )
+        develop=develop)
 
 
 if __name__ == "__main__":

diff --git a/exasol_sagemaker_extension/deployment/deploy_create_statements.py b/exasol_sagemaker_extension/deployment/deploy_create_statements.py
@@ -1,7 +1,9 @@
+from __future__ import annotations
 import logging
-import ssl
 
 import pyexasol
+from exasol.python_extension_common.connections.pyexasol_connection import open_pyexasol_connection
+
 from exasol_sagemaker_extension.deployment import constants
 from exasol_sagemaker_extension.deployment. \
     generate_create_statement_autopilot_endpoint_deletion \
@@ -120,39 +122,11 @@ def create_statements():
 
     @classmethod
     def create_and_run(cls,
-               db_host: str,
-               db_port: str,
-               db_user: str,
-               db_pass: str,
-               schema: str,
-               to_print: bool,
-               develop: bool):
-        """
-        Creates a database connection object based on the provided credentials
-        Creates an instance of the DeployCreateStatements passing the connection
-        object to it and calls its run method.
-
-        Parameters:
-            db_host     - database host address
-            db_port     - database port
-            db_user     - database username
-            db_pass     - the user password
-            schema      - schema where the scripts should be created
-            to_print    - if True the script creation SQL commands will be
-                          printed rather than executed
-            develop     - if True the scripts will be generated from scratch
-        """
-
-        exasol_conn = pyexasol.connect(
-            dsn=f"{db_host}:{db_port}",
-            user=db_user,
-            password=db_pass,
-            compression=True,
-            encryption=True,
-            websocket_sslopt={
-                "cert_reqs": ssl.CERT_NONE,
-            }
-        )
+                       schema: str,
+                       to_print: bool = False,
+                       develop: bool = False,
+                       **kwargs):
 
+        exasol_conn = open_pyexasol_connection(**kwargs)
         deployer = cls(exasol_conn, schema, to_print, develop)
         deployer.run()
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -29,7 +29,7 @@ sagemaker = "^2.59.1"
 pyexasol = "^0.25.0"
 importlib-resources = "^5.2.0"
 click = "^8.0.3"
-exasol-python-extension-common = ">=0.2.0"
+exasol-python-extension-common = ">=0.3.0"
 exasol-script-languages-container-tool = "^0.19.0"
 
 [tool.poetry.dev-dependencies]

diff --git a/scripts/setup_integration_test.sh b/scripts/setup_integration_test.sh
@@ -20,7 +20,7 @@ function install_localstack {
 function checkout_exasol_test_container {
   cnt_func=$((cnt_func+1))
   echo -e "${YEL} Step-$cnt_func: ${GRA} Checkout Exasol test container ${NC}"
-   git clone --branch 1.6.0 https://github.com/exasol/integration-test-docker-environment.git
+   git clone --branch 3.0.0 https://github.com/exasol/integration-test-docker-environment.git
 }
 
 function spawn_exasol_environment {

diff --git a/tests/ci_tests/fixtures/build_language_container_fixture.py b/tests/ci_tests/fixtures/build_language_container_fixture.py