diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0a8ab9ff..646c4108 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -35,18 +35,12 @@ jobs: run: ./scripts/setup_integration_test.sh - name: Poetry run pytest integration tests - run: poetry run pytest tests - - - name: Fail, if AWS-tests are not activated - if: "!contains(github.event.head_commit.message, '[run aws tests]')" - run: | - echo "Failed, because AWS-tests are not activated" - exit 1 - - name: Run AWS-tests if it is activated - if: "contains(github.event.head_commit.message, '[run aws tests]')" env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ROLE: ${{ secrets.AWS_ROLE }} AWS_DEFAULT_REGION: "eu-central-1" - run: poetry run pytest tests/ci_tests -s \ No newline at end of file + SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} + SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} + SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} + run: poetry run pytest tests diff --git a/doc/changes/changes_0.10.0.md b/doc/changes/changes_0.10.0.md index c607aaa0..ee2fc7af 100644 --- a/doc/changes/changes_0.10.0.md +++ b/doc/changes/changes_0.10.0.md @@ -1,4 +1,4 @@ -# SageMaker Extension 0.10.0, released 2024-06-20 +# SageMaker Extension 0.10.0, T.B.D. Code name: Python 3.10 @@ -8,7 +8,7 @@ Improving documentation, refactoring and moving to Python 3.10 ### Features -n/a +- #120: Running CI tests in the SaaS as well as the Docker DB. ### Bug Fixes diff --git a/exasol_sagemaker_extension/deployment/deploy_cli.py b/exasol_sagemaker_extension/deployment/deploy_cli.py index f4e14e74..0301fdf0 100644 --- a/exasol_sagemaker_extension/deployment/deploy_cli.py +++ b/exasol_sagemaker_extension/deployment/deploy_cli.py @@ -1,34 +1,77 @@ +from __future__ import annotations + import click import logging +from exasol.python_extension_common.deployment.language_container_deployer_cli import ( + SecretParams, SECRET_DISPLAY, secret_callback) + from exasol_sagemaker_extension.deployment.deploy_create_statements \ import DeployCreateStatements @click.command() -@click.option('--host', type=str, required=True, help="db host address") -@click.option('--port', type=str, required=True, help="db host port") -@click.option('--user', type=str, required=True, help="db user name") -@click.option('--pass', 'pwd', required=True, help="db user password") +@click.option('--dsn', type=str) +@click.option('--db-user', type=str) +@click.option(f'--{SecretParams.DB_PASSWORD.value}', type=str, + prompt='DB password', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) +@click.option('--saas-url', type=str, + default='https://cloud.exasol.com') +@click.option(f'--{SecretParams.SAAS_ACCOUNT_ID.value}', type=str, + prompt='SaaS account id', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) +@click.option(f'--{SecretParams.SAAS_DATABASE_ID.value}', type=str, + prompt='SaaS database id', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) +@click.option('--saas-database-name', type=str) +@click.option(f'--{SecretParams.SAAS_TOKEN.value}', type=str, + prompt='SaaS token', prompt_required=False, + hide_input=True, default=SECRET_DISPLAY, callback=secret_callback) +@click.option('--ssl-cert-path', type=str, default="") +@click.option('--ssl-client-cert-path', type=str, default="") +@click.option('--ssl-client-private-key', type=str, default="") +@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) @click.option('--schema', type=str, required=True, help="schema name") -@click.option('--print', 'verbose', type=bool, required=False, +@click.option('--print', 'verbose', type=bool, default=False, is_flag=True, help="print out statements") -@click.option('--develop', type=bool, required=False, +@click.option('--develop', type=bool, default=False, is_flag=True, help="generate and execute the scripts") -def main(host: str, port: str, user: str, pwd: str, schema: str, - verbose: bool = False, develop: bool = False): +def main( + dsn: str | None, + db_user: str | None, + db_pass: str | None, + saas_url: str | None, + saas_account_id: str | None, + saas_database_id: str | None, + saas_database_name: str | None, + saas_token: str | None, + ssl_cert_path: str, + ssl_client_cert_path: str, + ssl_client_private_key: str, + use_ssl_cert_validation: bool, + schema: str, + verbose: bool, + develop: bool): logging.basicConfig(format='%(asctime)s - %(module)s - %(message)s', level=logging.DEBUG) DeployCreateStatements.create_and_run( - db_host=host, - db_port=port, - db_user=user, - db_pass=pwd, schema=schema, + dsn=dsn, + db_user=db_user, + db_pass=db_pass, + saas_url=saas_url, + saas_account_id=saas_account_id, + saas_database_id=saas_database_id, + saas_database_name=saas_database_name, + saas_token=saas_token, + use_ssl_cert_validation=use_ssl_cert_validation, + ssl_trusted_ca=ssl_cert_path, + ssl_client_certificate=ssl_client_cert_path, + ssl_private_key=ssl_client_private_key, to_print=verbose, - develop=develop - ) + develop=develop) if __name__ == "__main__": diff --git a/exasol_sagemaker_extension/deployment/deploy_create_statements.py b/exasol_sagemaker_extension/deployment/deploy_create_statements.py index a6e38256..6d4487b7 100644 --- a/exasol_sagemaker_extension/deployment/deploy_create_statements.py +++ b/exasol_sagemaker_extension/deployment/deploy_create_statements.py @@ -1,7 +1,9 @@ +from __future__ import annotations import logging -import ssl import pyexasol +from exasol.python_extension_common.connections.pyexasol_connection import open_pyexasol_connection + from exasol_sagemaker_extension.deployment import constants from exasol_sagemaker_extension.deployment. \ generate_create_statement_autopilot_endpoint_deletion \ @@ -120,39 +122,11 @@ def create_statements(): @classmethod def create_and_run(cls, - db_host: str, - db_port: str, - db_user: str, - db_pass: str, - schema: str, - to_print: bool, - develop: bool): - """ - Creates a database connection object based on the provided credentials - Creates an instance of the DeployCreateStatements passing the connection - object to it and calls its run method. - - Parameters: - db_host - database host address - db_port - database port - db_user - database username - db_pass - the user password - schema - schema where the scripts should be created - to_print - if True the script creation SQL commands will be - printed rather than executed - develop - if True the scripts will be generated from scratch - """ - - exasol_conn = pyexasol.connect( - dsn=f"{db_host}:{db_port}", - user=db_user, - password=db_pass, - compression=True, - encryption=True, - websocket_sslopt={ - "cert_reqs": ssl.CERT_NONE, - } - ) + schema: str, + to_print: bool = False, + develop: bool = False, + **kwargs): + exasol_conn = open_pyexasol_connection(**kwargs) deployer = cls(exasol_conn, schema, to_print, develop) deployer.run() diff --git a/poetry.lock b/poetry.lock index 6faacbf8..0ad1b5db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -611,20 +611,20 @@ simplejson = ">=3.16.0" [[package]] name = "exasol-python-extension-common" -version = "0.2.0" +version = "0.3.0" description = "A collection of common utilities for Exasol extensions." optional = false -python-versions = "<4.0,>=3.8.0" +python-versions = "<4.0.0,>=3.10.0" files = [ - {file = "exasol_python_extension_common-0.2.0-py3-none-any.whl", hash = "sha256:c3bb039556491bd4494582c42a8cb9499cfc567c2cc19f960e444fbae56d6163"}, - {file = "exasol_python_extension_common-0.2.0.tar.gz", hash = "sha256:9e120b984866c2252617f84e448f8cf009193e4d7d8b9e01855b60a46ad5e892"}, + {file = "exasol_python_extension_common-0.3.0-py3-none-any.whl", hash = "sha256:1f9fb756ce20a079e17408d290443e345b5787a5c304a347b39e755e59b3b865"}, + {file = "exasol_python_extension_common-0.3.0.tar.gz", hash = "sha256:c36e6551bc2ff9db2236cbe1425f896ad203f010453e446d274df3bfc702cbf2"}, ] [package.dependencies] click = ">=8.1.7,<9.0.0" exasol-bucketfs = ">=0.10.0" exasol-saas-api = ">=0.7.0,<1.0.0" -pyexasol = ">=0.25.0,<0.26.0" +pyexasol = ">=0.25.0,<1.0.0" requests = "<2.32.0" tenacity = ">=8.3.0,<9.0.0" @@ -1388,13 +1388,13 @@ poetry-plugin = ["poetry (>=1.0,<2.0)"] [[package]] name = "portalocker" -version = "2.8.2" +version = "2.10.0" description = "Wraps the portalocker recipe for easy usage" optional = false python-versions = ">=3.8" files = [ - {file = "portalocker-2.8.2-py3-none-any.whl", hash = "sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e"}, - {file = "portalocker-2.8.2.tar.gz", hash = "sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33"}, + {file = "portalocker-2.10.0-py3-none-any.whl", hash = "sha256:48944147b2cd42520549bc1bb8fe44e220296e56f7c3d551bc6ecce69d9b0de1"}, + {file = "portalocker-2.10.0.tar.gz", hash = "sha256:49de8bc0a2f68ca98bf9e219c81a3e6b27097c7bf505a87c5a112ce1aaeb9b81"}, ] [package.dependencies] @@ -2002,13 +2002,13 @@ files = [ [[package]] name = "setuptools" -version = "70.1.0" +version = "70.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.1.0-py3-none-any.whl", hash = "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267"}, - {file = "setuptools-70.1.0.tar.gz", hash = "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5"}, + {file = "setuptools-70.1.1-py3-none-any.whl", hash = "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95"}, + {file = "setuptools-70.1.1.tar.gz", hash = "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650"}, ] [package.extras] @@ -2329,13 +2329,13 @@ files = [ [[package]] name = "tenacity" -version = "8.4.1" +version = "8.4.2" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" files = [ - {file = "tenacity-8.4.1-py3-none-any.whl", hash = "sha256:28522e692eda3e1b8f5e99c51464efcc0b9fc86933da92415168bc1c4e2308fa"}, - {file = "tenacity-8.4.1.tar.gz", hash = "sha256:54b1412b878ddf7e1f1577cd49527bad8cdef32421bd599beac0c6c3f10582fd"}, + {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"}, + {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"}, ] [package.extras] @@ -2567,4 +2567,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "9be2bb349969b364ca5b6b2dae3cf2c60316b5fec12347f962f5a1c3f255bc32" +content-hash = "3133668f1094c0a8c7f2efedf1925083348206a6b59f5f7bb778d3b437bf2890" diff --git a/pyproject.toml b/pyproject.toml index 5c5e4459..1cf5c09e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ sagemaker = "^2.59.1" pyexasol = "^0.25.0" importlib-resources = "^5.2.0" click = "^8.0.3" -exasol-python-extension-common = ">=0.2.0" +exasol-python-extension-common = ">=0.3.0" exasol-script-languages-container-tool = "^0.19.0" [tool.poetry.dev-dependencies] diff --git a/scripts/setup_integration_test.sh b/scripts/setup_integration_test.sh index 1ea2a2c6..34079f5c 100755 --- a/scripts/setup_integration_test.sh +++ b/scripts/setup_integration_test.sh @@ -20,7 +20,7 @@ function install_localstack { function checkout_exasol_test_container { cnt_func=$((cnt_func+1)) echo -e "${YEL} Step-$cnt_func: ${GRA} Checkout Exasol test container ${NC}" - git clone --branch 1.6.0 https://github.com/exasol/integration-test-docker-environment.git + git clone --branch 3.0.0 https://github.com/exasol/integration-test-docker-environment.git } function spawn_exasol_environment { diff --git a/tests/ci_tests/fixtures/build_language_container_fixture.py b/tests/ci_tests/fixtures/build_language_container_fixture.py deleted file mode 100644 index 2d3241f0..00000000 --- a/tests/ci_tests/fixtures/build_language_container_fixture.py +++ /dev/null @@ -1,91 +0,0 @@ -import time - -import pytest -import subprocess -from pathlib import Path -import exasol_bucketfs_utils_python.upload as bfsupload -from exasol_bucketfs_utils_python.bucket_config import BucketConfig -from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig -from exasol_bucketfs_utils_python.bucketfs_connection_config import \ - BucketFSConnectionConfig - -from tests.ci_tests.utils.parameters import db_params - - -def find_script(script_name: str): - current_path = Path("../..").absolute() - script_path = None - while current_path != current_path.root: - script_path = Path(current_path, script_name) - if script_path.exists(): - break - current_path = current_path.parent - if script_path.exists(): - return script_path - else: - raise RuntimeError("Could not find build_language_container.sh") - - -@pytest.fixture(scope="session") -def language_container(): - script_dir = find_script("build_language_container.sh") - completed_process = subprocess.run( - [script_dir], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = completed_process.stdout.decode("UTF-8") - print(output) - completed_process.check_returncode() - - lines = output.splitlines() - - alter_session_selector = "ALTER SYSTEM SET SCRIPT_LANGUAGES='" - alter_session = \ - [line for line in lines if line.startswith(alter_session_selector)][0] - alter_session = alter_session[len(alter_session_selector):-2] - - container_path_selector = "Cached container under " - container_path = \ - [line for line in lines if line.startswith(container_path_selector)][0] - container_path = container_path[len(container_path_selector):] - - return { - "container_path": container_path, - "alter_session": alter_session - } - - -@pytest.fixture(scope="session") -def upload_language_container(language_container, db_conn): - connection_config = BucketFSConnectionConfig( - host=db_params.host, - port=6666, - user="w", - pwd="write", - is_https=False - ) - - bucket_config = BucketConfig( - bucket_name="default", - bucketfs_config=BucketFSConfig( - bucketfs_name="bfsdefault", - connection_config=connection_config) - ) - - path_in_bucket = "container" - container_name = "exasol_sagemaker_extension_container.tar.gz" - container_path = Path(language_container["container_path"]) - - with open(container_path, "rb") as container_file: - bfsupload.upload_fileobj_to_bucketfs( - bucket_config=bucket_config, - bucket_file_path=f"{path_in_bucket}/{container_name}", - fileobj=container_file) - - alter_session = language_container["alter_session"] - time.sleep(3 * 60) # Wait for SLC extraction in BucketFS - return alter_session - - -@pytest.fixture(scope="session") -def register_language_container(upload_language_container, db_conn): - alter_session = upload_language_container - db_conn.execute(f"ALTER SYSTEM SET SCRIPT_LANGUAGES='{alter_session}'") diff --git a/tests/ci_tests/fixtures/database_connection_fixture.py b/tests/ci_tests/fixtures/database_connection_fixture.py deleted file mode 100644 index 9ce2160b..00000000 --- a/tests/ci_tests/fixtures/database_connection_fixture.py +++ /dev/null @@ -1,22 +0,0 @@ -import ssl - -import pyexasol -import pytest -from tests.ci_tests.utils.parameters import db_params - - -@pytest.fixture(scope="session") -def db_conn(): - conn = pyexasol.connect( - dsn="{host}:{port}".format( - host=db_params.host, - port=db_params.port), - user=db_params.user, - password=db_params.password, - encryption=True, - websocket_sslopt={ - "cert_reqs": ssl.CERT_NONE, - } - ) - - return conn diff --git a/tests/ci_tests/fixtures/setup_ci_test_environment.py b/tests/ci_tests/fixtures/setup_ci_test_environment.py deleted file mode 100644 index 14d60b4a..00000000 --- a/tests/ci_tests/fixtures/setup_ci_test_environment.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest - -from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment - - -@pytest.fixture(scope="session") -def setup_ci_test_environment(register_language_container, - prepare_ci_test_environment) -> CITestEnvironment: - return prepare_ci_test_environment diff --git a/tests/ci_tests/test_deploying_autopilot.py b/tests/ci_tests/test_deploying_autopilot.py index 6067da87..6fbc9d4d 100644 --- a/tests/ci_tests/test_deploying_autopilot.py +++ b/tests/ci_tests/test_deploying_autopilot.py @@ -1,12 +1,13 @@ import time from datetime import datetime -from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment +import pytest + +from tests.fixtures.prepare_environment_fixture import CITestEnvironment from tests.ci_tests.utils import parameters from tests.ci_tests.utils.autopilot_deployment import AutopilotTestDeployment from tests.ci_tests.utils.autopilot_polling import AutopilotTestPolling from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining -from tests.ci_tests.utils.checkers import skip_if_aws_credentials_not_set from tests.ci_tests.utils.cleanup import cleanup from tests.ci_tests.utils.parameters import cls_model_setup_params from tests.ci_tests.utils.queries import DatabaseQueries @@ -52,8 +53,8 @@ def _deploy_endpoint(job_name, endpoint_name, model_setup_params, ci_test_env: C assert endpoint_name in list(map(lambda x: x[0], all_scripts)) -@skip_if_aws_credentials_not_set -def test_deploy_autopilot_endpoint(setup_ci_test_environment): +@pytest.mark.slow +def test_deploy_autopilot_endpoint(prepare_ci_test_environment): curr_datetime = datetime.now().strftime("%y%m%d%H%M%S") model_name = ''.join((cls_model_setup_params.model_type, curr_datetime)) job_name = ''.join((model_name, 'job')) @@ -61,11 +62,11 @@ def test_deploy_autopilot_endpoint(setup_ci_test_environment): # train AutopilotTestTraining.train_autopilot_classification_job( - job_name, setup_ci_test_environment) + job_name, prepare_ci_test_environment) # deploy _deploy_endpoint( job_name=job_name, endpoint_name=endpoint_name, model_setup_params=cls_model_setup_params, - db_conn=setup_ci_test_environment) + db_conn=prepare_ci_test_environment) diff --git a/tests/ci_tests/test_polling_autopilot.py b/tests/ci_tests/test_polling_autopilot.py index 834e97fc..96a824ef 100644 --- a/tests/ci_tests/test_polling_autopilot.py +++ b/tests/ci_tests/test_polling_autopilot.py @@ -1,26 +1,27 @@ from datetime import datetime +import pytest + from tests.ci_tests.utils.autopilot_polling import AutopilotTestPolling from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining -from tests.ci_tests.utils.checkers import skip_if_aws_credentials_not_set from tests.ci_tests.utils.parameters import cls_model_setup_params -@skip_if_aws_credentials_not_set -def test_poll_autopilot_job(setup_ci_test_environment): +@pytest.mark.slow +def test_poll_autopilot_job(prepare_ci_test_environment): curr_datetime = datetime.now().strftime("%y%m%d%H%M%S") model_name = ''.join((cls_model_setup_params.model_type, curr_datetime)) job_name = ''.join((model_name, 'job')) # train AutopilotTestTraining.train_autopilot_classification_job( - job_name, setup_ci_test_environment) + job_name, prepare_ci_test_environment) # poll status = AutopilotTestPolling.poll_autopilot_job( job_name, cls_model_setup_params.schema_name, - setup_ci_test_environment) + prepare_ci_test_environment) assert len(status) == 1 assert len(status[0]) == 2 diff --git a/tests/ci_tests/test_predicting_autopilot.py b/tests/ci_tests/test_predicting_autopilot.py index 97c462f7..b17c83ca 100644 --- a/tests/ci_tests/test_predicting_autopilot.py +++ b/tests/ci_tests/test_predicting_autopilot.py @@ -1,13 +1,14 @@ import time from datetime import datetime -from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment +import pytest + +from tests.fixtures.prepare_environment_fixture import CITestEnvironment from tests.ci_tests.utils import parameters from tests.ci_tests.utils.autopilot_deployment import AutopilotTestDeployment from tests.ci_tests.utils.autopilot_polling import AutopilotTestPolling from tests.ci_tests.utils.autopilot_prediction import AutopilotTestPrediction from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining -from tests.ci_tests.utils.checkers import skip_if_aws_credentials_not_set from tests.ci_tests.utils.cleanup import cleanup from tests.ci_tests.utils.parameters import cls_model_setup_params, \ reg_model_setup_params @@ -53,8 +54,8 @@ def _make_prediction(job_name, endpoint_name, model_setup_params, ci_test_env: C assert predictions -@skip_if_aws_credentials_not_set -def test_predict_autopilot_regression_job(setup_ci_test_environment): +@pytest.mark.slow +def test_predict_autopilot_regression_job(prepare_ci_test_environment): curr_datetime = datetime.now().strftime("%y%m%d%H%M%S") model_name = ''.join((reg_model_setup_params.model_type, curr_datetime)) job_name = ''.join((model_name, 'job')) @@ -62,18 +63,18 @@ def test_predict_autopilot_regression_job(setup_ci_test_environment): # train AutopilotTestTraining.train_autopilot_regression_job( - job_name, setup_ci_test_environment) + job_name, prepare_ci_test_environment) # deploy endpoint and make prediction on it _make_prediction( job_name=job_name, endpoint_name=endpoint_name, model_setup_params=reg_model_setup_params, - db_conn=setup_ci_test_environment) + db_conn=prepare_ci_test_environment) -@skip_if_aws_credentials_not_set -def test_predict_autopilot_classification_job(setup_ci_test_environment): +@pytest.mark.slow +def test_predict_autopilot_classification_job(prepare_ci_test_environment): curr_datetime = datetime.now().strftime("%y%m%d%H%M%S") model_name = ''.join((cls_model_setup_params.model_type, curr_datetime)) job_name = ''.join((model_name, 'job')) @@ -81,11 +82,11 @@ def test_predict_autopilot_classification_job(setup_ci_test_environment): # train AutopilotTestTraining.train_autopilot_classification_job( - job_name, setup_ci_test_environment) + job_name, prepare_ci_test_environment) # deploy endpoint and make prediction on it _make_prediction( job_name=job_name, endpoint_name=endpoint_name, model_setup_params=cls_model_setup_params, - db_conn=setup_ci_test_environment) + db_conn=prepare_ci_test_environment) diff --git a/tests/ci_tests/test_training_autopilot.py b/tests/ci_tests/test_training_autopilot.py index 68367726..291fe3d4 100644 --- a/tests/ci_tests/test_training_autopilot.py +++ b/tests/ci_tests/test_training_autopilot.py @@ -1,29 +1,30 @@ from datetime import datetime -from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment +import pytest + +from tests.fixtures.prepare_environment_fixture import CITestEnvironment from tests.ci_tests.utils.autopilot_training import AutopilotTestTraining -from tests.ci_tests.utils.checkers import skip_if_aws_credentials_not_set from tests.ci_tests.utils.parameters import reg_model_setup_params, \ cls_model_setup_params from tests.ci_tests.utils.queries import DatabaseQueries -@skip_if_aws_credentials_not_set -def test_train_autopilot_regression_job(setup_ci_test_environment): +@pytest.mark.slow +def test_train_autopilot_regression_job(prepare_ci_test_environment): curr_datetime = datetime.now().strftime("%y%m%d%H%M%S") model_name = ''.join((reg_model_setup_params.model_type, curr_datetime)) job_name = ''.join((model_name, 'job')) # train AutopilotTestTraining.train_autopilot_regression_job( - job_name, setup_ci_test_environment) + job_name, prepare_ci_test_environment) _assert_training_job( - job_name, reg_model_setup_params, setup_ci_test_environment) + job_name, reg_model_setup_params, prepare_ci_test_environment) -@skip_if_aws_credentials_not_set -def test_train_autopilot_classification_job(setup_ci_test_environment): +@pytest.mark.slow +def test_train_autopilot_classification_job(prepare_ci_test_environment): curr_datetime = datetime.now().strftime("%y%m%d%H%M%S") model_name = ''.join((cls_model_setup_params.model_type, curr_datetime)) job_name = ''.join((model_name, 'job')) @@ -31,10 +32,10 @@ def test_train_autopilot_classification_job(setup_ci_test_environment): # train AutopilotTestTraining.train_autopilot_classification_job( - job_name, setup_ci_test_environment, objective) + job_name, prepare_ci_test_environment, objective) _assert_training_job( - job_name, cls_model_setup_params, setup_ci_test_environment) + job_name, cls_model_setup_params, prepare_ci_test_environment) def _assert_training_job(job_name, model_setup_params, ci_test_env: CITestEnvironment): diff --git a/tests/ci_tests/utils/autopilot_deployment.py b/tests/ci_tests/utils/autopilot_deployment.py index ea046024..b4a49f26 100644 --- a/tests/ci_tests/utils/autopilot_deployment.py +++ b/tests/ci_tests/utils/autopilot_deployment.py @@ -1,6 +1,6 @@ import os -from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment +from tests.fixtures.prepare_environment_fixture import CITestEnvironment INSTANCE_TYPE = "ml.c5.large" INSTANCE_COUNT = 1 diff --git a/tests/ci_tests/utils/autopilot_polling.py b/tests/ci_tests/utils/autopilot_polling.py index 5c0be684..e17223e4 100644 --- a/tests/ci_tests/utils/autopilot_polling.py +++ b/tests/ci_tests/utils/autopilot_polling.py @@ -1,6 +1,6 @@ import os -from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment +from tests.fixtures.prepare_environment_fixture import CITestEnvironment class AutopilotTestPolling: diff --git a/tests/ci_tests/utils/autopilot_prediction.py b/tests/ci_tests/utils/autopilot_prediction.py index 157a8f75..5a7b6d40 100644 --- a/tests/ci_tests/utils/autopilot_prediction.py +++ b/tests/ci_tests/utils/autopilot_prediction.py @@ -1,4 +1,4 @@ -from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment +from tests.fixtures.prepare_environment_fixture import CITestEnvironment class AutopilotTestPrediction: diff --git a/tests/ci_tests/utils/autopilot_training.py b/tests/ci_tests/utils/autopilot_training.py index 9b29dfd4..33032e60 100644 --- a/tests/ci_tests/utils/autopilot_training.py +++ b/tests/ci_tests/utils/autopilot_training.py @@ -1,6 +1,6 @@ import json -from tests.ci_tests.fixtures.prepare_environment_fixture import CITestEnvironment +from tests.fixtures.prepare_environment_fixture import CITestEnvironment from tests.ci_tests.utils.parameters import reg_model_setup_params, \ cls_model_setup_params diff --git a/tests/ci_tests/utils/build_language_container.py b/tests/ci_tests/utils/build_language_container.py new file mode 100644 index 00000000..486e482d --- /dev/null +++ b/tests/ci_tests/utils/build_language_container.py @@ -0,0 +1,89 @@ +import time + +import pyexasol +import subprocess +from pathlib import Path +import exasol.bucketfs as bfs +from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer + +from tests.integration_tests.utils.parameters import db_params + +LANGUAGE_ALIAS = "PYTHON3_SME" +CONTAINER_FILE_NAME = "exasol_sagemaker_extension_container.tar.gz" + + +def find_script(script_name: str): + current_path = Path("../..").absolute() + script_path = None + while current_path != current_path.root: + script_path = Path(current_path, script_name) + if script_path.exists(): + break + current_path = current_path.parent + if script_path.exists(): + return script_path + else: + raise RuntimeError("Could not find build_language_container.sh") + + +def build_language_container() -> Path: + """ + Returns the path of the built container + """ + script_dir = find_script("build_language_container.sh") + completed_process = subprocess.run( + [script_dir], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = completed_process.stdout.decode("UTF-8") + print(output) + completed_process.check_returncode() + + lines = output.splitlines() + container_path_selector = "Cached container under " + container_path = \ + next(line for line in lines if line.startswith(container_path_selector)) + container_path = container_path[len(container_path_selector):] + return Path(container_path) + + +def upload_language_container(db_conn: pyexasol.ExaConnection, + bucketfs_location: bfs.path.PathLike) -> None: + container_path = build_language_container() + + deployer = LanguageContainerDeployer(pyexasol_connection=db_conn, + language_alias=LANGUAGE_ALIAS, + bucketfs_path=bucketfs_location) + + deployer.run(container_file=container_path, + bucket_file_path=CONTAINER_FILE_NAME, + wait_for_completion=True) + + +def upload_language_container_onprem(db_conn: pyexasol.ExaConnection) -> None: + + bucketfs_location = bfs.path.build_path(backend=bfs.path.StorageBackend.onprem, + url=f'http://{db_params.host}:{db_params.bfs_port}', + username=db_params.bfs_user, + password=db_params.bfs_password, + verify=False, + bucket_name='default', + service_name='bfsdefault', + path='container') + + upload_language_container(db_conn, bucketfs_location) + + +def upload_language_container_saas(db_conn: pyexasol.ExaConnection, + saas_url: str, + saas_account_id: str, + saas_database_id: str, + saas_token: str) -> None: + + bucketfs_location = bfs.path.build_path(backend=bfs.path.StorageBackend.saas, + url=saas_url, + account_id=saas_account_id, + database_id=saas_database_id, + pat=saas_token) + + upload_language_container(db_conn, bucketfs_location) + + time.sleep(180) diff --git a/tests/ci_tests/utils/checkers.py b/tests/ci_tests/utils/checkers.py deleted file mode 100644 index 0877c036..00000000 --- a/tests/ci_tests/utils/checkers.py +++ /dev/null @@ -1,15 +0,0 @@ -import os - -import pytest - - -def is_aws_credentials_not_set(): - return not ( - "AWS_ACCESS_KEY_ID" in os.environ and - "AWS_SECRET_ACCESS_KEY" in os.environ and - "AWS_DEFAULT_REGION" in os.environ - ) - - -skip_if_aws_credentials_not_set = pytest.mark.skipif(is_aws_credentials_not_set() == True, - reason="AWS credentials are not set") diff --git a/tests/ci_tests/utils/parameters.py b/tests/ci_tests/utils/parameters.py index c50ec736..8fd2208e 100644 --- a/tests/ci_tests/utils/parameters.py +++ b/tests/ci_tests/utils/parameters.py @@ -1,3 +1,5 @@ +from __future__ import annotations +from typing import Any from collections import namedtuple POLLING_INTERVAL = 5 * 60 # seconds @@ -28,16 +30,9 @@ ) -def get_db_params(): - DBParams = namedtuple("DBParams", [ - "host", "port", "user", "password"]) - - return DBParams( - host="127.0.0.1", - port="9563", - user="sys", - password="exasol" - ) - - -db_params = get_db_params() +def get_arg_list(**kwargs) -> list[str]: + args_list: list[str] = [] + for k, v in kwargs.items(): + args_list.append(f'--{k.replace("_", "-")}') + args_list.append(str(v)) + return args_list diff --git a/tests/conftest.py b/tests/conftest.py index 2b053aa6..05a7a44a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,7 @@ pytest_plugins = [ - "tests.unit_tests.fixtures.aws_connection_fixture", - "tests.integration_tests.fixtures.database_connection_fixture", - "tests.integration_tests.fixtures.build_language_container_fixture", - "tests.integration_tests.fixtures.setup_database_fixture", - "tests.integration_tests.fixtures.deploy_scripts_fixture", - - "tests.ci_tests.fixtures.database_connection_fixture", - "tests.ci_tests.fixtures.build_language_container_fixture", - "tests.ci_tests.fixtures.prepare_environment_fixture", - "tests.ci_tests.fixtures.setup_ci_test_environment" + "tests.fixtures.aws_connection_fixture", + "tests.fixtures.database_connection_fixture", + "tests.fixtures.setup_database_fixture", + "tests.fixtures.prepare_environment_fixture", + "tests.fixtures.script_deployment_fixture", ] diff --git a/tests/deployment/test_deploy_cli.py b/tests/deployment/test_deploy_cli.py index fb093727..4b297970 100644 --- a/tests/deployment/test_deploy_cli.py +++ b/tests/deployment/test_deploy_cli.py @@ -1,6 +1,9 @@ +import pytest from click.testing import CliRunner +import exasol.bucketfs as bfs + from exasol_sagemaker_extension.deployment import deploy_cli -from tests.integration_tests.utils.parameters import db_params +from tests.ci_tests.utils.parameters import get_arg_list DB_SCHEMA = "TEST_CLI_SCHEMA" AUTOPILOT_TRAINING_LUA_SCRIPT_NAME = \ @@ -34,16 +37,22 @@ def get_all_scripts(db_conn): return list(map(lambda x: x[0], all_scripts)) -def test_deploy_cli_main(db_conn, register_language_container): - args_list = [ - "--host", db_params.host, - "--port", db_params.port, - "--user", db_params.user, - "--pass", db_params.password, - "--schema", DB_SCHEMA - ] +@pytest.mark.slow +def test_deploy_cli_main(backend, db_conn, deploy_params): + + args_list = get_arg_list(**deploy_params, schema=DB_SCHEMA) + if backend == bfs.path.StorageBackend.saas: + args_list.append("--use-ssl-cert-validation") + else: + args_list.append("--no-use-ssl-cert-validation") + runner = CliRunner() result = runner.invoke(deploy_cli.main, args_list) + + if result.exit_code != 0: + print(result.output) + + assert not result.exception assert result.exit_code == 0 all_schemas = get_all_schemas(db_conn) diff --git a/tests/deployment/test_deploy_create_statements.py b/tests/deployment/test_deploy_create_statements.py index 8a90c549..8c2dc164 100644 --- a/tests/deployment/test_deploy_create_statements.py +++ b/tests/deployment/test_deploy_create_statements.py @@ -1,6 +1,8 @@ +import pytest +import exasol.bucketfs as bfs + from exasol_sagemaker_extension.deployment.deploy_create_statements import \ DeployCreateStatements -from tests.integration_tests.utils.parameters import db_params DB_SCHEMA = "TEST_DEPLOY_SCHEMA" AUTOPILOT_TRAINING_LUA_SCRIPT_NAME = \ @@ -26,16 +28,13 @@ def get_all_scripts(db_conn): return list(map(lambda x: x[0], all_scripts)) -def test_deploy_create_statements(db_conn, register_language_container): - DeployCreateStatements.create_and_run( - db_host=db_params.host, - db_port=db_params.port, - db_user=db_params.user, - db_pass=db_params.password, - schema=DB_SCHEMA, - to_print=False, - develop=False - ) +@pytest.mark.slow +def test_deploy_create_statements(backend, db_conn, deploy_params): + + # We validate the server certificate in SaaS, but not in the Docker DB + cert_validation = backend == bfs.path.StorageBackend.saas + DeployCreateStatements.create_and_run(**deploy_params, schema=DB_SCHEMA, + use_ssl_cert_validation=cert_validation) all_schemas = get_all_schemas(db_conn) all_scripts = get_all_scripts(db_conn) diff --git a/tests/ci_tests/fixtures/__init__.py b/tests/fixtures/__init__.py similarity index 100% rename from tests/ci_tests/fixtures/__init__.py rename to tests/fixtures/__init__.py diff --git a/tests/unit_tests/fixtures/aws_connection_fixture.py b/tests/fixtures/aws_connection_fixture.py similarity index 99% rename from tests/unit_tests/fixtures/aws_connection_fixture.py rename to tests/fixtures/aws_connection_fixture.py index 094d6de7..c4e9dc25 100644 --- a/tests/unit_tests/fixtures/aws_connection_fixture.py +++ b/tests/fixtures/aws_connection_fixture.py @@ -15,4 +15,3 @@ def get_mock_aws_params(): "AWS_OUTPUT_PATH": "train" } return mock_params - diff --git a/tests/fixtures/database_connection_fixture.py b/tests/fixtures/database_connection_fixture.py new file mode 100644 index 00000000..7062f7ea --- /dev/null +++ b/tests/fixtures/database_connection_fixture.py @@ -0,0 +1,121 @@ +from __future__ import annotations +import os +from datetime import timedelta +from contextlib import ExitStack + +import ssl +import pytest +import pyexasol +import exasol.bucketfs as bfs +from exasol.saas.client.api_access import ( + OpenApiAccess, + create_saas_client, + timestamp_name, + get_connection_params +) + +from tests.ci_tests.utils.build_language_container import ( + upload_language_container_saas, upload_language_container_onprem) +from tests.integration_tests.utils.parameters import db_params + + +def _env(var: str) -> str: + result = os.environ.get(var) + if result: + return result + raise RuntimeError(f"Environment variable {var} is empty.") + + +def _open_pyexasol_connection(**kwargs) -> pyexasol.ExaConnection: + return pyexasol.connect(**kwargs, + encryption=True, + websocket_sslopt={"cert_reqs": ssl.CERT_NONE}, + compression=True) + + +@pytest.fixture(scope='session', params=[bfs.path.StorageBackend.onprem, bfs.path.StorageBackend.saas]) +def backend(request) -> bfs.path.StorageBackend: + # Here we are going to add + # pytest.skip() + # if there is an instruction to skip a particular backed in the command line. + return request.param + + +@pytest.fixture(scope="session") +def db_conn_onprem(backend) -> pyexasol.ExaConnection | None: + if backend == bfs.path.StorageBackend.onprem: + conn = _open_pyexasol_connection(dsn=f"{db_params.host}:{db_params.port}", + user=db_params.user, + password=db_params.password) + upload_language_container_onprem(db_conn=conn) + return conn + return None + + +@pytest.fixture(scope="session") +def saas_url() -> str: + return _env("SAAS_HOST") + + +@pytest.fixture(scope="session") +def saas_account_id() -> str: + return _env("SAAS_ACCOUNT_ID") + + +@pytest.fixture(scope="session") +def saas_token() -> str: + return _env("SAAS_PAT") + + +@pytest.fixture(scope="session") +def saas_database_id(backend, saas_url, saas_account_id, saas_token) -> str: + + if backend == bfs.path.StorageBackend.saas: + with ExitStack() as stack: + # Create and configure the SaaS client. + client = create_saas_client(host=saas_url, pat=saas_token) + api_access = OpenApiAccess(client=client, account_id=saas_account_id) + stack.enter_context(api_access.allowed_ip()) + + # Create a temporary database and waite till it becomes operational + db = stack.enter_context(api_access.database( + name=timestamp_name('SME_CI'), + idle_time=timedelta(hours=12))) + api_access.wait_until_running(db.id, timeout=timedelta(minutes=45)) + yield db.id + else: + yield '' + + +@pytest.fixture(scope="session") +def db_conn_saas(backend, saas_url, saas_account_id, saas_database_id, saas_token) -> pyexasol.ExaConnection | None: + + if backend == bfs.path.StorageBackend.saas: + # Create a connection to the database. + conn_params = get_connection_params(host=saas_url, + account_id=saas_account_id, + database_id=saas_database_id, + pat=saas_token) + conn = _open_pyexasol_connection(**conn_params) + + # Build, upload and activate the language container + upload_language_container_saas(db_conn=conn, + saas_url=saas_url, + saas_account_id=saas_account_id, + saas_database_id=saas_database_id, + saas_token=saas_token) + yield conn + else: + yield None + + +@pytest.fixture(scope="session") +def db_conn(backend, + db_conn_onprem, + db_conn_saas) -> pyexasol.ExaConnection: + if backend == bfs.path.StorageBackend.saas: + assert db_conn_saas is not None + yield db_conn_saas + else: + assert db_conn_onprem is not None + yield db_conn_onprem diff --git a/tests/ci_tests/fixtures/prepare_environment_fixture.py b/tests/fixtures/prepare_environment_fixture.py similarity index 82% rename from tests/ci_tests/fixtures/prepare_environment_fixture.py rename to tests/fixtures/prepare_environment_fixture.py index 27e0136a..4ef98e62 100644 --- a/tests/ci_tests/fixtures/prepare_environment_fixture.py +++ b/tests/fixtures/prepare_environment_fixture.py @@ -1,3 +1,5 @@ +from __future__ import annotations +from typing import Any import dataclasses import os from inspect import cleandoc @@ -5,32 +7,27 @@ import boto3 import pyexasol +import exasol.bucketfs as bfs import pytest -from click.testing import CliRunner -from exasol_sagemaker_extension.deployment import deploy_cli -from tests.ci_tests.utils.parameters import db_params, \ - reg_model_setup_params, cls_model_setup_params +from exasol_sagemaker_extension.deployment.deploy_create_statements import DeployCreateStatements +from tests.ci_tests.utils.parameters import ( + reg_model_setup_params, cls_model_setup_params) -def __open_schema(db_conn, model_setup): +def _open_schema(db_conn: pyexasol.ExaConnection, model_setup): query = "CREATE SCHEMA IF NOT EXISTS {schema_name}" db_conn.execute(query.format(schema_name=model_setup.schema_name)) -def __deploy_scripts(model_setup): - args_list = [ - "--host", db_params.host, - "--port", db_params.port, - "--user", db_params.user, - "--pass", db_params.password, - "--schema", model_setup.schema_name - ] - runner = CliRunner() - runner.invoke(deploy_cli.main, args_list) +def _deploy_scripts(backend: bfs.path.StorageBackend, deploy_params: dict[str, Any], schema: str): + cert_validation = backend == bfs.path.StorageBackend.saas + DeployCreateStatements.create_and_run(**deploy_params, schema=schema, + use_ssl_cert_validation=cert_validation) -def __create_tables(db_conn, model_setup): + +def _create_tables(db_conn, model_setup): query = "CREATE OR REPLACE TABLE {schema_name}.{table_name} " \ "(col1 FLOAT, col2 FLOAT, output_col INTEGER)". \ format(schema_name=model_setup.schema_name, @@ -38,7 +35,7 @@ def __create_tables(db_conn, model_setup): db_conn.execute(query) -def __insert_into_tables(db_conn, model_setup): +def _insert_into_tables(db_conn, model_setup): values = ",".join(model_setup.data) query = "INSERT INTO {schema_name}.{table_name} VALUES {values}". \ format(schema_name=model_setup.schema_name, @@ -47,12 +44,13 @@ def __insert_into_tables(db_conn, model_setup): db_conn.execute(query) -def _setup_database(db_conn): +def _setup_database(backend: bfs.path.StorageBackend, db_conn: pyexasol.ExaConnection, + deploy_params: dict[str, Any]): for model_setup in [reg_model_setup_params, cls_model_setup_params]: - __open_schema(db_conn, model_setup) - __deploy_scripts(model_setup) - __create_tables(db_conn, model_setup) - __insert_into_tables(db_conn, model_setup) + _open_schema(db_conn, model_setup) + _deploy_scripts(backend, deploy_params, model_setup.schema_name) + _create_tables(db_conn, model_setup) + _insert_into_tables(db_conn, model_setup) @pytest.fixture(scope="session") @@ -197,11 +195,13 @@ def aws_bucket_uri(self) -> str: @pytest.fixture(scope="session") -def prepare_ci_test_environment(db_conn, +def prepare_ci_test_environment(backend, + db_conn, + deploy_params, aws_s3_bucket, connection_object_for_aws_credentials, aws_sagemaker_role) -> CITestEnvironment: - _setup_database(db_conn) + _setup_database(backend, db_conn, deploy_params) yield CITestEnvironment(db_conn=db_conn, aws_s3_bucket=aws_s3_bucket, connection_object_for_aws_credentials=connection_object_for_aws_credentials, diff --git a/tests/fixtures/script_deployment_fixture.py b/tests/fixtures/script_deployment_fixture.py new file mode 100644 index 00000000..141bb3aa --- /dev/null +++ b/tests/fixtures/script_deployment_fixture.py @@ -0,0 +1,36 @@ +from __future__ import annotations +from typing import Any + +import pytest +import exasol.bucketfs as bfs + +from tests.integration_tests.utils.parameters import db_params + + +@pytest.fixture(scope="session") +def deploy_params_onprem() -> dict[str, Any]: + return { + 'dsn': f"{db_params.host}:{db_params.port}", + 'db_user': db_params.user, + 'db_pass': db_params.password + } + + +@pytest.fixture(scope="session") +def deploy_params_saas(saas_url, saas_account_id, saas_database_id, saas_token) -> dict[str, Any]: + yield { + 'saas_url': saas_url, + 'saas_account_id': saas_account_id, + 'saas_database_id': saas_database_id, + 'saas_token': saas_token + } + + +@pytest.fixture(scope="session") +def deploy_params(backend, + deploy_params_onprem, + deploy_params_saas) -> dict[str, Any]: + if backend == bfs.path.StorageBackend.saas: + yield deploy_params_saas + else: + yield deploy_params_onprem diff --git a/tests/integration_tests/fixtures/setup_database_fixture.py b/tests/fixtures/setup_database_fixture.py similarity index 100% rename from tests/integration_tests/fixtures/setup_database_fixture.py rename to tests/fixtures/setup_database_fixture.py diff --git a/tests/integration_tests/fixtures/__init__.py b/tests/integration_tests/fixtures/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/integration_tests/fixtures/build_language_container_fixture.py b/tests/integration_tests/fixtures/build_language_container_fixture.py deleted file mode 100644 index bc63a9ab..00000000 --- a/tests/integration_tests/fixtures/build_language_container_fixture.py +++ /dev/null @@ -1,88 +0,0 @@ -import pytest -import subprocess -from pathlib import Path -import exasol_bucketfs_utils_python.upload as bfsupload -from exasol_bucketfs_utils_python.bucket_config import BucketConfig -from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig -from exasol_bucketfs_utils_python.bucketfs_connection_config import \ - BucketFSConnectionConfig - -from tests.integration_tests.utils.parameters import db_params - - -def find_script(script_name: str): - current_path = Path("../..").absolute() - script_path = None - while current_path != current_path.root: - script_path = Path(current_path, script_name) - if script_path.exists(): - break - current_path = current_path.parent - if script_path.exists(): - return script_path - else: - raise RuntimeError("Could not find build_language_container.sh") - - -@pytest.fixture(scope="session") -def language_container(): - script_dir = find_script("build_language_container.sh") - completed_process = subprocess.run( - [script_dir], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = completed_process.stdout.decode("UTF-8") - completed_process.check_returncode() - - print(output) - lines = output.splitlines() - - alter_session_selector = "ALTER SYSTEM SET SCRIPT_LANGUAGES='" - alter_session = \ - [line for line in lines if line.startswith(alter_session_selector)][0] - alter_session = alter_session[len(alter_session_selector):-2] - - container_path_selector = "Cached container under " - container_path = \ - [line for line in lines if line.startswith(container_path_selector)][0] - container_path = container_path[len(container_path_selector):] - - return { - "container_path": container_path, - "alter_session": alter_session - } - - -@pytest.fixture(scope="session") -def upload_language_container(language_container, db_conn): - connection_config = BucketFSConnectionConfig( - host=db_params.host, - port=6666, - user="w", - pwd="write", - is_https=False - ) - - bucket_config = BucketConfig( - bucket_name="default", - bucketfs_config=BucketFSConfig( - bucketfs_name="bfsdefault", - connection_config=connection_config) - ) - - path_in_bucket = "container" - container_name = "exasol_sagemaker_extension_container.tar.gz" - container_path = Path(language_container["container_path"]) - - with open(container_path, "rb") as container_file: - bfsupload.upload_fileobj_to_bucketfs( - bucket_config=bucket_config, - bucket_file_path=f"{path_in_bucket}/{container_name}", - fileobj=container_file) - - alter_session = Path(language_container["alter_session"]) - return alter_session - - -@pytest.fixture(scope="session") -def register_language_container(upload_language_container, db_conn): - alter_session = upload_language_container - db_conn.execute(f"ALTER SYSTEM SET SCRIPT_LANGUAGES='{alter_session}'") \ No newline at end of file diff --git a/tests/integration_tests/fixtures/database_connection_fixture.py b/tests/integration_tests/fixtures/database_connection_fixture.py deleted file mode 100644 index 0b21620a..00000000 --- a/tests/integration_tests/fixtures/database_connection_fixture.py +++ /dev/null @@ -1,22 +0,0 @@ -import ssl - -import pyexasol -import pytest -from tests.integration_tests.utils.parameters import db_params - - -@pytest.fixture(scope="session") -def db_conn(): - conn = pyexasol.connect( - dsn="{host}:{port}".format( - host=db_params.host, - port=db_params.port), - user=db_params.user, - password=db_params.password, - encryption=True, - websocket_sslopt={ - "cert_reqs": ssl.CERT_NONE, - } - ) - - return conn diff --git a/tests/integration_tests/fixtures/deploy_scripts_fixture.py b/tests/integration_tests/fixtures/deploy_scripts_fixture.py deleted file mode 100644 index 845085a2..00000000 --- a/tests/integration_tests/fixtures/deploy_scripts_fixture.py +++ /dev/null @@ -1,15 +0,0 @@ -import pytest -from exasol_sagemaker_extension.deployment import deploy_cli -from tests.integration_tests.utils.parameters import db_params, reg_setup_params - - -@pytest.fixture(scope="session") -def deploy_scripts(): - args_list = [ - "--host", db_params.host, - "--port", db_params.port, - "--user", db_params.user, - "--pass", db_params.password, - "--schema", reg_setup_params.schema_name - ] - deploy_cli.main(args_list) diff --git a/tests/integration_tests/test_exporting_to_localstack_s3.py b/tests/integration_tests/test_exporting_to_localstack_s3.py index aa40207c..abe6e56f 100644 --- a/tests/integration_tests/test_exporting_to_localstack_s3.py +++ b/tests/integration_tests/test_exporting_to_localstack_s3.py @@ -1,9 +1,12 @@ import json -import pytest import os.path + +import pytest import localstack_client.session +import exasol.bucketfs as bfs + from tests.integration_tests.utils.generate_create_statement_s3_exporting \ - import S3ExportingLuaScriptCreateStatementGenerator + import S3ExportingLuaScriptCreateStatementGenerator DB_CONNECTION_ADDR = "127.0.0.1:9563" DB_CONNECTION_USER = "sys" @@ -134,7 +137,11 @@ def get_comparison_query(import_table_name): import_table_name=import_table_name) -def test_export_table(get_database_conn, s3_client): +def test_export_table(backend, get_database_conn, s3_client): + if backend != bfs.path.StorageBackend.onprem: + pytest.skip(("The test can only run locally, because " + "localstack is running locally and SaaS can't access it.'")) + db_conn = get_database_conn create_s3_bucket(s3_client) diff --git a/tests/integration_tests/utils/parameters.py b/tests/integration_tests/utils/parameters.py index 26e98b3e..21e216de 100644 --- a/tests/integration_tests/utils/parameters.py +++ b/tests/integration_tests/utils/parameters.py @@ -27,13 +27,16 @@ def get_aws_params(): def get_db_params(): DBParams = namedtuple("DBParams", [ - "host", "port", "user", "password"]) + "host", "port", "bfs_port", "user", "bfs_user", "password", "bfs_password"]) return DBParams( host="127.0.0.1", port="9563", + bfs_port="6666", user="sys", - password="exasol" + bfs_user="w", + password="exasol", + bfs_password="write" ) diff --git a/tests/unit_tests/fixtures/__init__.py b/tests/unit_tests/fixtures/__init__.py deleted file mode 100644 index e69de29b..00000000