From ab5e312bfc36b40d138d2398504a7807fd356a76 Mon Sep 17 00:00:00 2001 From: Matthew D'Alonzo Date: Sat, 23 Mar 2024 13:39:20 -0400 Subject: [PATCH] Add Github Actions to run BDD Scripts on push (#15) * Add Github Workflow * Add uploading of behave-test-reports as artifact in workflow * Modify Behave report output format to HTML --------- Co-authored-by: Matthew D'Alonzo Co-authored-by: Patrick Yang <47698870+hostilechild007@users.noreply.github.com> Co-authored-by: Neelam Kushwah --- .github/workflows/tck-test.yml | 139 ++++++++++++++++++ test_manager/features/environment.py | 16 +- .../steps/tck_step_implementations.py | 22 +-- test_manager/features/utils/loggerutils.py | 44 +----- test_manager/testmanager.py | 2 +- 5 files changed, 156 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/tck-test.yml diff --git a/.github/workflows/tck-test.yml b/.github/workflows/tck-test.yml new file mode 100644 index 00000000..5e418413 --- /dev/null +++ b/.github/workflows/tck-test.yml @@ -0,0 +1,139 @@ +name: TCK Tests + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + run_tests: + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: maven + - name: Build up_client_socket_java with Maven + working-directory: up_client_socket/java + run: | + mvn clean install --file pom.xml + - name: Build java_test_agent with Maven + working-directory: test_agent/java + run: | + mvn clean install --file pom.xml + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + pip install -r requirements.txt + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Get Behave Scripts + uses: actions/github-script@v6 + id: check-env + with: + result-encoding: string + script: | + const feature_file_list = []; + const fs = require('fs'); + const path = require('path'); + + function traverseDir(dir) { + fs.readdirSync(dir).forEach(file => { + let fullPath = path.join(dir, file); + if (fs.lstatSync(fullPath).isDirectory()) { + traverseDir(fullPath); + } else { + core.info("Adding file: " + fullPath); + feature_file_list.push({ filename: file, path: fullPath.replace("test_manager/", "") }); + } + }); + } + + traverseDir("./test_manager/features/tests"); + fs.writeFileSync('./test_manager/feature_file_list.json', JSON.stringify(feature_file_list)); + - name: TCK Behave Tests + run: | + pwd + ls -l + cd test_manager + # Read JSON file content + content=$(<./feature_file_list.json) + + # Loop through each JSON object + echo "$content" | jq -c '.[]' | while IFS='' read -r obj; do + # Extract filename and full path from the current JSON object + filename=$(echo "$obj" | jq -r '.filename') + full_path=$(echo "$obj" | jq -r '.path') + + # Run behave command + echo "Running Test: $filename" + behave --format json --outfile "./reports/${filename}.json" --format html --outfile "./reports/${filename}.html" "$full_path" + echo "Finished Test: $filename" + sleep 70 + done + - name: Get Behave Scripts + uses: actions/github-script@v6 + with: + result-encoding: string + script: | + const feature_file_list = [] + const fs = require('fs') + const path = require('path'); + + function traverseDir(dir) { + fs.readdirSync(dir).forEach(file => { + let fullPath = path.join(dir, file); + if (fs.lstatSync(fullPath).isDirectory()) { + traverseDir(fullPath); + } else { + feature_file_list.push({ filename: file, path: fullPath }); + } + }); + } + + traverseDir("./test_manager/reports"); + const json_list = [] + try { + for (let i = 0; i < feature_file_list.length; i++){ + file_extension = path.parse(feature_file_list[i]["filename"]).ext + file_name = path.parse(feature_file_list[i]["filename"]).name + if (file_extension == ".json" && file_name != "summary") { + json_list.push(JSON.parse(fs.readFileSync(feature_file_list[i]["path"]))); + } + } + for (let i = 0; i < json_list.length; i++) { + if (json_list[i][0].status != "passed") { + core.setFailed("One or more features failed") + core.error("\u001b[38;2;255;0;0mFeature:" + json_list[i][0].name + " [failed]") + } else{ + core.info("\u001b[38;2;0;255;0mFeature:" + json_list[i][0].name + " [passed]") + } + } + } catch(err) { + core.error("\u001b[38;2;255;0;0mError while reading or parsing the JSON") + core.setFailed(err) + } + - name: Upload Test Reports + uses: actions/upload-artifact@v4 + with: + name: behave-test-reports + path: ./test_manager/reports/*.html \ No newline at end of file diff --git a/test_manager/features/environment.py b/test_manager/features/environment.py index f262d66c..92da95ae 100644 --- a/test_manager/features/environment.py +++ b/test_manager/features/environment.py @@ -46,13 +46,6 @@ def create_command(filepath_from_root_repo: str) -> List[str]: command: List[str] = [] - if sys.platform == "win32": - pass - elif sys.platform == "linux" or sys.platform == "linux2": - command.append('gnome-terminal') - command.append('--') - else: - raise Exception("only handle Windows and Linux commands for now") if filepath_from_root_repo.endswith('.jar'): command.append("java") @@ -130,11 +123,8 @@ def after_all(context: Context): context.tm.close() try: - context.dispatcher_process.kill() - context.dispatcher_process.communicate() - context.java_ta_process.kill() - context.java_ta_process.communicate() - context.python_ta_process.kill() - context.python_ta_process.communicate() + context.dispatcher_process.terminate() + context.java_ta_process.terminate() + context.python_ta_process.terminate() except Exception as e: context.logger.error(e) diff --git a/test_manager/features/steps/tck_step_implementations.py b/test_manager/features/steps/tck_step_implementations.py index 106224eb..dd69001c 100644 --- a/test_manager/features/steps/tck_step_implementations.py +++ b/test_manager/features/steps/tck_step_implementations.py @@ -34,7 +34,7 @@ @given(u'"{sdk_name}" creates data for "{command}"') @when(u'"{sdk_name}" creates data for "{command}"') -def step_impl(context, sdk_name: str, command: str): +def create_sdk_data(context, sdk_name: str, command: str): context.logger.info("Inside create register listener data") context.json_dict = {} context.status_json = None @@ -55,7 +55,7 @@ def step_impl(context, sdk_name: str, command: str): @then(u'the serialized uri received is "{expected_uri}"') -def step_impl(context, expected_uri): +def serialized_received(context, expected_uri): try: rec_field_value = context.on_receive_serialized_uri assert_that(expected_uri, equal_to(rec_field_value)) @@ -67,7 +67,7 @@ def step_impl(context, expected_uri): @when(u'sends a "{command}" request with the value "{serialized_uri}"') -def step_impl(context, command, serialized_uri): +def send_serialized_command(context, command, serialized_uri): context.logger.info(f"Json request for {command} -> {serialized_uri}") context.tm.receive_from_bdd(context.ue, command, serialized_uri) @@ -89,26 +89,26 @@ def verify_received_properties(context): @given(u'sets "{key}" to "{value}"') @when(u'sets "{key}" to "{value}"') -def step_impl(context: Context, key: str, value: str): +def set_key_to_val(context: Context, key: str, value: str): if key not in context.json_dict: context.json_dict[key] = value @given(u'sets "{key}" to ""') -def step_impl(context, key): +def set_blank_key(context, key): pass @given(u'sets "{key}" to b"{value}"') @when(u'sets "{key}" to b"{value}"') -def step_impl(context, key, value): +def set_key_to_bytes(context, key, value): if key not in context.json_dict: context.json_dict[key] = "BYTES:" + value @given(u'sends "{command}" request') @when(u'sends "{command}" request') -def step_impl(context, command: str): +def send_command_request(context, command: str): context.json_dict = unflatten_dict(context.json_dict) context.logger.info(f"Json request for {command} -> {str(context.json_dict)}") context.tm.receive_from_bdd(context.ue, context.action, context.json_dict) @@ -116,12 +116,12 @@ def step_impl(context, command: str): @when(u'user waits "{sec}" second') @then(u'user waits "{sec}" second') -def step_impl(context, sec): +def user_wait(context, sec): time.sleep(int(sec)) @then(u'the status received with "{field}" is "{field_value}"') -def step_impl(context, field, field_value): +def receive_status(context, field, field_value): try: rec_field_value = context.status_json[field] assert_that(field_value, equal_to(rec_field_value)) @@ -133,7 +133,7 @@ def step_impl(context, field, field_value): @then(u'"{sdk_name}" receives "{key}" as b"{value}"') -def step_impl(context, sdk_name, key, value): +def receive_value_as_bytes(context, sdk_name, key, value): try: value = value.strip() val = access_nested_dict(context.on_receive_msg[sdk_name], key) @@ -149,7 +149,7 @@ def step_impl(context, sdk_name, key, value): @then(u'"{sdk_name}" receives rpc response having "{key}" as b"{value}"') -def step_impl(context, sdk_name, key, value): +def receive_rpc_response_as_bytes(context, sdk_name, key, value): try: val = access_nested_dict(context.on_receive_rpc_response[sdk_name], key) rec_field_value = base64.b64decode(val.encode('utf-8')) diff --git a/test_manager/features/utils/loggerutils.py b/test_manager/features/utils/loggerutils.py index c61ef2f2..247e4642 100644 --- a/test_manager/features/utils/loggerutils.py +++ b/test_manager/features/utils/loggerutils.py @@ -41,38 +41,7 @@ 'version': 1, 'disable_existing_loggers': False, - 'handlers': { - 'formatted_console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - 'formatter': 'detailed', - 'stream': 'ext://sys.stdout', - }, - 'formatted_file': { - 'class': 'logging.handlers.RotatingFileHandler', - 'level': 'INFO', - 'formatter': 'detailed', - 'filename': "logs" + os.path.sep + "uitests_" + strftime("%Y-%m-%d_%H_%M_%S") + ".log", - 'mode': 'a', - 'maxBytes': 10485760, - 'backupCount': 5, - }, - 'unformatted_console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - 'formatter': 'blank', - 'stream': 'ext://sys.stdout', - }, - 'unformatted_file': { - 'class': 'logging.handlers.RotatingFileHandler', - 'level': 'INFO', - 'formatter': 'blank', - 'filename': "logs" + os.path.sep + "uitests_" + strftime("%Y-%m-%d_%H_%M_%S") + ".log", - 'mode': 'a', - 'maxBytes': 10485760, - 'backupCount': 5, - }, - }, + 'handlers': {}, 'formatters': { 'detailed': { 'format': '[%(levelname)s] : %(asctime)s : %(filename)s : %(funcName)s:%(lineno)d : %(message)s', @@ -81,16 +50,7 @@ 'format': '', }, }, - 'loggers': { - 'formatted_log': { - 'level': 'DEBUG', - 'handlers': ['formatted_file', 'formatted_console'] - }, - 'unformatted_log': { - 'level': 'DEBUG', - 'handlers': ['unformatted_file', 'unformatted_console'] - }, - } + 'loggers': {} } diff --git a/test_manager/testmanager.py b/test_manager/testmanager.py index 9de4ef05..bc330387 100644 --- a/test_manager/testmanager.py +++ b/test_manager/testmanager.py @@ -127,7 +127,7 @@ def has_sdk_connection(self, sdk_name: str) -> bool: def listen_for_client_connections(self): while not self.exit_manager: # Wait until some registered file objects or sockets become ready, or the timeout expires. - events = self.selector.select() + events = self.selector.select(timeout=0) for key, mask in events: callback = key.data callback(key.fileobj)