Add Github Actions to run BDD Scripts on push (eclipse-uprotocol#15)

* Add Github Workflow
* Add uploading of behave-test-reports as artifact in workflow
* Modify Behave report output format to HTML

---------

Co-authored-by: Matthew D'Alonzo <[email protected]>
Co-authored-by: Patrick Yang <[email protected]>
Co-authored-by: Neelam Kushwah <[email protected]>

.github/workflows/tck-test.yml

@@ -0,0 +1,139 @@
+name: TCK Tests
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  run_tests:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up JDK 11
+      uses: actions/setup-java@v3
+      with:
+        java-version: '11'
+        distribution: 'temurin'
+        cache: maven
+    - name: Build up_client_socket_java with Maven
+      working-directory: up_client_socket/java
+      run: |
+            mvn clean install --file pom.xml
+    - name: Build java_test_agent with Maven
+      working-directory: test_agent/java
+      run: |
+            mvn clean install --file pom.xml
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        pip install -r requirements.txt
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Get Behave Scripts
+      uses: actions/github-script@v6
+      id: check-env
+      with:
+        result-encoding: string
+        script: |
+          const feature_file_list = [];
+          const fs = require('fs');
+          const path = require('path');
+          
+          function traverseDir(dir) {
+            fs.readdirSync(dir).forEach(file => {
+              let fullPath = path.join(dir, file);
+              if (fs.lstatSync(fullPath).isDirectory()) {
+                traverseDir(fullPath);
+              } else {
+                core.info("Adding file: " + fullPath);
+                feature_file_list.push({ filename: file, path: fullPath.replace("test_manager/", "") });
+              }
+            });
+          }
+          
+          traverseDir("./test_manager/features/tests");
+          fs.writeFileSync('./test_manager/feature_file_list.json', JSON.stringify(feature_file_list));
+    - name: TCK Behave Tests
+      run: |
+        pwd
+        ls -l
+        cd test_manager
+        # Read JSON file content
+        content=$(<./feature_file_list.json)
+
+        # Loop through each JSON object
+        echo "$content" | jq -c '.[]' | while IFS='' read -r obj; do
+            # Extract filename and full path from the current JSON object
+            filename=$(echo "$obj" | jq -r '.filename')
+            full_path=$(echo "$obj" | jq -r '.path')
+            
+            # Run behave command
+            echo "Running Test: $filename"
+            behave --format json --outfile "./reports/${filename}.json" --format html --outfile "./reports/${filename}.html" "$full_path"
+            echo "Finished Test: $filename"
+            sleep 70
+        done
+    - name: Get Behave Scripts
+      uses: actions/github-script@v6
+      with:
+        result-encoding: string
+        script: |
+          const feature_file_list = []
+          const fs = require('fs')
+          const path = require('path');
+
+          function traverseDir(dir) {
+            fs.readdirSync(dir).forEach(file => {
+              let fullPath = path.join(dir, file);
+              if (fs.lstatSync(fullPath).isDirectory()) {
+                traverseDir(fullPath);
+              } else {
+                feature_file_list.push({ filename: file, path: fullPath });
+              }
+            });
+          }
+
+          traverseDir("./test_manager/reports");
+          const json_list = []
+          try {
+            for (let i = 0; i < feature_file_list.length; i++){
+              file_extension = path.parse(feature_file_list[i]["filename"]).ext
+              file_name = path.parse(feature_file_list[i]["filename"]).name
+              if (file_extension == ".json" && file_name != "summary") {
+                json_list.push(JSON.parse(fs.readFileSync(feature_file_list[i]["path"])));
+              }
+            }
+            for (let i = 0; i < json_list.length; i++) {
+              if (json_list[i][0].status != "passed") {
+                core.setFailed("One or more features failed")
+                core.error("\u001b[38;2;255;0;0mFeature:" + json_list[i][0].name + " [failed]")
+              } else{
+                core.info("\u001b[38;2;0;255;0mFeature:" + json_list[i][0].name + " [passed]")
+              }
+            }
+          } catch(err) {
+            core.error("\u001b[38;2;255;0;0mError while reading or parsing the JSON")
+            core.setFailed(err)
+          }
+    - name: Upload Test Reports
+      uses: actions/upload-artifact@v4
+      with:
+        name: behave-test-reports
+        path: ./test_manager/reports/*.html

test_manager/features/environment.py

@@ -46,13 +46,6 @@
 
 def create_command(filepath_from_root_repo: str) -> List[str]:
     command: List[str] = []
-    if sys.platform == "win32":
-        pass
-    elif sys.platform == "linux" or sys.platform == "linux2":
-        command.append('gnome-terminal')
-        command.append('--')
-    else:
-        raise Exception("only handle Windows and Linux commands for now")
 
     if filepath_from_root_repo.endswith('.jar'):
         command.append("java")
@@ -130,11 +123,8 @@ def after_all(context: Context):
     context.tm.close()
 
     try:
-        context.dispatcher_process.kill()
-        context.dispatcher_process.communicate()
-        context.java_ta_process.kill()
-        context.java_ta_process.communicate()
-        context.python_ta_process.kill()
-        context.python_ta_process.communicate()
+        context.dispatcher_process.terminate()
+        context.java_ta_process.terminate()
+        context.python_ta_process.terminate()
     except Exception as e:
         context.logger.error(e)

test_manager/features/steps/tck_step_implementations.py

@@ -34,7 +34,7 @@
 
 @given(u'"{sdk_name}" creates data for "{command}"')
 @when(u'"{sdk_name}" creates data for "{command}"')
-def step_impl(context, sdk_name: str, command: str):
+def create_sdk_data(context, sdk_name: str, command: str):
     context.logger.info("Inside create register listener data")
     context.json_dict = {}
     context.status_json = None
@@ -55,7 +55,7 @@ def step_impl(context, sdk_name: str, command: str):
 
 
 @then(u'the serialized uri received is "{expected_uri}"')
-def step_impl(context, expected_uri):
+def serialized_received(context, expected_uri):
     try:
         rec_field_value = context.on_receive_serialized_uri
         assert_that(expected_uri, equal_to(rec_field_value))
@@ -67,7 +67,7 @@ def step_impl(context, expected_uri):
 
 
 @when(u'sends a "{command}" request with the value "{serialized_uri}"')
-def step_impl(context, command, serialized_uri):
+def send_serialized_command(context, command, serialized_uri):
     context.logger.info(f"Json request for {command} -> {serialized_uri}")
     context.tm.receive_from_bdd(context.ue, command, serialized_uri)
 
@@ -89,39 +89,39 @@ def verify_received_properties(context):
 
 @given(u'sets "{key}" to "{value}"')
 @when(u'sets "{key}" to "{value}"')
-def step_impl(context: Context, key: str, value: str):
+def set_key_to_val(context: Context, key: str, value: str):
     if key not in context.json_dict:
         context.json_dict[key] = value
 
 
 @given(u'sets "{key}" to ""')
-def step_impl(context, key):
+def set_blank_key(context, key):
     pass
 
 
 @given(u'sets "{key}" to b"{value}"')
 @when(u'sets "{key}" to b"{value}"')
-def step_impl(context, key, value):
+def set_key_to_bytes(context, key, value):
     if key not in context.json_dict:
         context.json_dict[key] = "BYTES:" + value
 
 
 @given(u'sends "{command}" request')
 @when(u'sends "{command}" request')
-def step_impl(context, command: str):
+def send_command_request(context, command: str):
     context.json_dict = unflatten_dict(context.json_dict)
     context.logger.info(f"Json request for {command} -> {str(context.json_dict)}")
     context.tm.receive_from_bdd(context.ue, context.action, context.json_dict)
 
 
 @when(u'user waits "{sec}" second')
 @then(u'user waits "{sec}" second')
-def step_impl(context, sec):
+def user_wait(context, sec):
     time.sleep(int(sec))
 
 
 @then(u'the status received with "{field}" is "{field_value}"')
-def step_impl(context, field, field_value):
+def receive_status(context, field, field_value):
     try:
         rec_field_value = context.status_json[field]
         assert_that(field_value, equal_to(rec_field_value))
@@ -133,7 +133,7 @@ def step_impl(context, field, field_value):
 
 
 @then(u'"{sdk_name}" receives "{key}" as b"{value}"')
-def step_impl(context, sdk_name, key, value):
+def receive_value_as_bytes(context, sdk_name, key, value):
     try:
         value = value.strip()
         val = access_nested_dict(context.on_receive_msg[sdk_name], key)
@@ -149,7 +149,7 @@ def step_impl(context, sdk_name, key, value):
 
 
 @then(u'"{sdk_name}" receives rpc response having "{key}" as b"{value}"')
-def step_impl(context, sdk_name, key, value):
+def receive_rpc_response_as_bytes(context, sdk_name, key, value):
     try:
         val = access_nested_dict(context.on_receive_rpc_response[sdk_name], key)
         rec_field_value = base64.b64decode(val.encode('utf-8'))

test_manager/features/utils/loggerutils.py

@@ -41,38 +41,7 @@
 
     'version': 1,
     'disable_existing_loggers': False,
-    'handlers': {
-        'formatted_console': {
-            'class': 'logging.StreamHandler',
-            'level': 'INFO',
-            'formatter': 'detailed',
-            'stream': 'ext://sys.stdout',
-        },
-        'formatted_file': {
-            'class': 'logging.handlers.RotatingFileHandler',
-            'level': 'INFO',
-            'formatter': 'detailed',
-            'filename': "logs" + os.path.sep + "uitests_" + strftime("%Y-%m-%d_%H_%M_%S") + ".log",
-            'mode': 'a',
-            'maxBytes': 10485760,
-            'backupCount': 5,
-        },
-        'unformatted_console': {
-            'class': 'logging.StreamHandler',
-            'level': 'INFO',
-            'formatter': 'blank',
-            'stream': 'ext://sys.stdout',
-        },
-        'unformatted_file': {
-            'class': 'logging.handlers.RotatingFileHandler',
-            'level': 'INFO',
-            'formatter': 'blank',
-            'filename': "logs" + os.path.sep + "uitests_" + strftime("%Y-%m-%d_%H_%M_%S") + ".log",
-            'mode': 'a',
-            'maxBytes': 10485760,
-            'backupCount': 5,
-        },
-    },
+    'handlers': {},
     'formatters': {
         'detailed': {
             'format': '[%(levelname)s] : %(asctime)s : %(filename)s : %(funcName)s:%(lineno)d : %(message)s',
@@ -81,16 +50,7 @@
             'format': '',
         },
     },
-    'loggers': {
-        'formatted_log': {
-                'level': 'DEBUG',
-                'handlers': ['formatted_file', 'formatted_console']
-            },
-        'unformatted_log': {
-            'level': 'DEBUG',
-            'handlers': ['unformatted_file', 'unformatted_console']
-        },
-    }
+    'loggers': {}
 }
 
 

test_manager/testmanager.py

@@ -127,7 +127,7 @@ def has_sdk_connection(self, sdk_name: str) -> bool:
     def listen_for_client_connections(self):
         while not self.exit_manager:
             # Wait until some registered file objects or sockets become ready, or the timeout expires.
-            events = self.selector.select()
+            events = self.selector.select(timeout=0)
             for key, mask in events:
                 callback = key.data
                 callback(key.fileobj)