From 97b1bb2468d68c7cefa9d329c33487287835ec88 Mon Sep 17 00:00:00 2001 From: Abram Date: Mon, 23 Dec 2024 09:07:05 +0100 Subject: [PATCH] feat (sdk:tests): create TestAgentaInitCommand and TestAgentaVariantServeCommand test suite and test cases for each --- agenta-cli/tests/cli/test_init.py | 120 +++++++++++++++++++ agenta-cli/tests/cli/test_variant_serve.py | 127 +++++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 agenta-cli/tests/cli/test_init.py create mode 100644 agenta-cli/tests/cli/test_variant_serve.py diff --git a/agenta-cli/tests/cli/test_init.py b/agenta-cli/tests/cli/test_init.py new file mode 100644 index 000000000..7662f7617 --- /dev/null +++ b/agenta-cli/tests/cli/test_init.py @@ -0,0 +1,120 @@ +import os +import toml +import uuid +from pathlib import Path + +import pytest + +from .fixtures import * + + +class TestAgentaInitCommand: + @pytest.fixture(autouse=True) + def _setup(self): + self.asset_example_folder = "greetings" + self.assets_folder = str( + get_assets_folder(example_folder=self.asset_example_folder) + ) + + @pytest.mark.cli_testing + def test_cloud_blank_app_success(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "n" + provide_api_key = os.environ.get("AGENTA_API_KEY") + + # ACT: Add configuration + inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + provide_api_key, + ] + result = run_agenta_init(inputs, self.asset_example_folder) + cli_output = next(result) + + # ASSERT: Verify response + assert cli_output["exit_status"] == 0 + assert "App initialized successfully" in cli_output["output"] + + config_path = Path(f"{self.assets_folder}/config.toml") + assert config_path.exists() + + config = toml.load(config_path) + assert config["app_id"] is not None + assert config["app_name"] == app_name + assert config["backend_host"] == os.environ.get("AGENTA_HOST") + + agentaignore_path = Path(f"{self.assets_folder}/.agentaignore") + assert agentaignore_path.exists() + + # CLEANUP: Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files(self.asset_example_folder) + assert next(cleanup) == "ok" + + @pytest.mark.cli_testing + def test_cloud_blank_app_already_exists(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "N" + provide_api_key = os.environ.get("AGENTA_API_KEY") + + # ACT: Add configuration + inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + provide_api_key, + ] + result_1 = run_agenta_init( + inputs, self.asset_example_folder + ) # create app the first time + _ = next(result_1) + result_2 = run_agenta_init( + inputs, self.asset_example_folder + ) # tries to create app with the same name + cli_output = next(result_2) + + # ASSERT: Verify response + assert cli_output["exit_status"] == 1 + assert "App with the same name already exists" in cli_output["output"] + + # CLEANUP: Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files(self.asset_example_folder) + assert next(cleanup) == "ok" + + @pytest.mark.cli_testing + def test_cloud_blank_app_with_invalid_credential(self): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + provide_api_key = "dummy_key\n" + environ_keys = os.environ.copy() + os.environ["AGENTA_API_KEY"] = "dummy_key" + + # ACT: Add configuration + inputs = [ + f"{app_name}\n", + where_to_run_agenta, + provide_api_key, + ] + result = run_agenta_init(inputs, self.asset_example_folder) + cli_output = next(result) + + # ASSERT: Verify response + assert ( + cli_output["exit_status"] == 1 + ) # Ensure non-zero exit status indicating failure + assert "Unauthorized" in cli_output["output"] + + config_path = Path(f"{self.assets_folder}/config.toml") + assert not config_path.exists() + + agentaignore_path = Path(f"{self.assets_folder}/.agentaignore") + assert not agentaignore_path.exists() + + # CLEANUP: Reset environment variables + os.environ.clear() + os.environ.update(environ_keys) diff --git a/agenta-cli/tests/cli/test_variant_serve.py b/agenta-cli/tests/cli/test_variant_serve.py new file mode 100644 index 000000000..e88b25195 --- /dev/null +++ b/agenta-cli/tests/cli/test_variant_serve.py @@ -0,0 +1,127 @@ +import os +import toml +import uuid +from pathlib import Path + +import pytest + +from .fixtures import * + + +class TestAgentaVariantServeCommand: + @pytest.fixture(autouse=True) + def _setup(self): + self.asset_example_folder = "salutations" + self.assets_folder = str( + get_assets_folder(example_folder=self.asset_example_folder) + ) + + @pytest.mark.cli_testing + def test_variant_serve_success(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "n" + provide_api_key = os.environ.get("AGENTA_API_KEY") + + # ACT: Add configuration + init_inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + provide_api_key, + ] + result = run_agenta_init(init_inputs, self.asset_example_folder) + cli_output = next(result) + + if cli_output["exit_status"] == 1: + pytest.fail("Creating an app from the CLI failed.") + + serve_inputs = [] + result = run_variant_serve(serve_inputs, self.asset_example_folder) + cli_serve_output = next(result) + + # ASSERT: Verify response + assert cli_serve_output["exit_status"] == 0 + assert "Adding app.default to server..." in cli_serve_output["output"] + assert "Waiting for the variant to be ready" in cli_serve_output["output"] + assert "Variant added successfully!" in cli_serve_output["output"] + assert "Congratulations!" in cli_serve_output["output"] + assert ( + "Your app has been deployed locally as an API." + in cli_serve_output["output"] + ) + assert "Read the API documentation." in cli_serve_output["output"] + assert ( + "Start experimenting with your app in the playground." + in cli_serve_output["output"] + ) + + config_path = Path(f"{self.assets_folder}/config.toml") + assert config_path.exists() + + config = toml.load(config_path) + assert config["app_id"] is not None + assert config["app_name"] == app_name + assert config["backend_host"] == os.environ.get("AGENTA_HOST") + + agentaignore_path = Path(f"{self.assets_folder}/.agentaignore") + assert agentaignore_path.exists() + + # CLEANUP: Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files(self.asset_example_folder) + assert next(cleanup) == "ok" + + @pytest.mark.cli_testing + def test_variant_reserve_success(self, cleanup_application_and_files): + # ARRANGE: Prepare test data + app_name = f"greetings_{uuid.uuid4().hex[:6]}" + where_to_run_agenta = "\n" + use_this_key = "n" + provide_api_key = os.environ.get("AGENTA_API_KEY") + + # ACT: Add configuration + init_inputs = [ + f"{app_name}\n", + where_to_run_agenta, + use_this_key, + provide_api_key, + ] + result = run_agenta_init(init_inputs, self.asset_example_folder) + cli_output = next(result) + + if cli_output["exit_status"] == 1: + pytest.fail("Creating an app from the CLI failed.") + + serve_inputs = [] + serve_result = run_variant_serve(serve_inputs, self.asset_example_folder) + cli_serve_output = next(serve_result) + + if cli_serve_output["exit_status"] == 1: + pytest.fail("Serving a variant from the CLI failed.") + + reserve_inputs = ["y\n"] + reserve_result = run_variant_serve(reserve_inputs, self.asset_example_folder) + cli_reserve_output = next(reserve_result) + + # ASSERT: Verify response + assert cli_reserve_output["exit_status"] == 0 + assert ( + f"Variant app.default for App {app_name} updated successfully" + in cli_reserve_output["output"] + ) + + config_path = Path(f"{self.assets_folder}/config.toml") + assert config_path.exists() + + config = toml.load(config_path) + assert config["app_id"] is not None + assert config["app_name"] == app_name + assert config["backend_host"] == os.environ.get("AGENTA_HOST") + + agentaignore_path = Path(f"{self.assets_folder}/.agentaignore") + assert agentaignore_path.exists() + + # CLEANUP: Remove application from backend, db and local filesystem + cleanup = cleanup_application_and_files(self.asset_example_folder) + assert next(cleanup) == "ok"