added remote tests

soumeh01 · Apr 29, 2024 · f0c7f52 · f0c7f52
1 parent 5ac0d03
commit f0c7f52
Show file tree

Hide file tree

Showing 8 changed files with 73 additions and 63 deletions.
diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml
@@ -18,14 +18,14 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-latest, macos-latest]
+        os: [windows-latest, ubuntu-latest, macos-13]
         arch: [amd64, arm64]
         include:
           - os: windows-latest
             target: win
           - os: ubuntu-latest
             target: lin
-          - os: macos-latest
+          - os: macos-13
             target: mac
     steps:
       - uses: actions/checkout@v4
@@ -55,7 +55,7 @@ jobs:
 
       - name: Run Test
         run: |
-          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+          python -m robot --consolewidth 100 --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
 
       - name: Archieve test results
         if: always()
@@ -83,7 +83,7 @@ jobs:
           pip install --upgrade pip
           pip install -r test/requirements.txt
 
-    - name: Download reports darwin arm64
+    - name: Download reports
       uses: actions/download-artifact@v4
       with:
         path: artifacts
@@ -95,7 +95,7 @@ jobs:
         python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
           ./reports-win-amd64/output.xml ./reports-win-arm64/output.xml \
           ./reports-lin-amd64/output.xml ./reports-lin-arm64/output.xml \
-          ./reports-mac-amd64/output.xml ./reports-mac-arm64/output.xml
+          ./reports-mac-amd64/output.xml ./reports-mac-arm64/output.xml    
 
     - name: Generate Summary report
       if: always()
@@ -104,7 +104,9 @@ jobs:
 
     - name: Print E2E Report
       if: always()
-      run: cat summary_report.md >> $GITHUB_STEP_SUMMARY
+      run: |
+        cat summary_report.md >> $GITHUB_STEP_SUMMARY
+        echo "📜 Review the test result summary at: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
 
     - name: Archieve consolidated test results
       if: always()

diff --git a/test/RobotTests.md b/test/RobotTests.md
@@ -8,15 +8,34 @@ Before running Robot Framework tests, ensure you have the following prerequisite
 
 - Python (minimum recommended version **3.11**)
 - pip (python package manager)
+- [vcpkg](https://github.com/Open-CMSIS-Pack/cmsis-toolbox/blob/main/docs/installation.md#vcpkg---setup-using-cli)
 
-## Install Robot Framework
+## Install Test Environment
+
+### Install Robot Framework
 
 Install Robot Framework and its dependencies using pip:
 
 ```bash
-cd <root_dir>
+cd <root_dir>/test
 pip install --upgrade pip
-pip install -r test/requirements.txt
+pip install -r requirements.txt
+```
+
+### Install Toolchains using vcpkg
+
+These commands will install all the required toolchains listed [here](./vcpkg-configuration.json):
+
+```bash
+cd <root_dir>/test
+vcpkg x-update-registry --all
+vcpkg activate
+```
+
+```txt
+👉 Important:
+    The AC6 toolchain installation includes the "fromelf" utility, which can be found in the AC6 toolchain installation
+    directory. Ensure that "fromelf" utlity is in PATH.
 ```
 
 ## Running Tests
@@ -27,6 +46,12 @@ This command will run all tests located in the `test` directory and place the te
 
 ```bash
 robot -d <output_directory> <path_to_tests>
+```
+
+for e.g.
+
+```bash
+vcpkg activate
 robot -d results ./test
 ```
 
@@ -41,6 +66,7 @@ robot --test <test_name> <path_to_tests>
 for e.g.
 
 ```bash
+vcpkg activate
 robot --test "Validate build-c Example"  test/test.robot
 ```
 
@@ -65,6 +91,6 @@ The test cases in [test.robot](./test.robot) are implemented in a data-driven st
 
 ```txt
 ☑️ Note:
-    All options in the tests should be separated by **TABs**.
+    All options in the tests should be separated by "TABs".
     For more information on robot test follow https://docs.robotframework.org/docs/testcase_styles/datadriven
 ```
diff --git a/test/__init__.robot b/test/__init__.robot
@@ -1,22 +1,3 @@
 *** Settings ***
-Documentation       Initailization for the robot tests in this directory
+Documentation       Initialization for the robot tests in this directory
 Test Timeout        10 minutes
-
-
-# Library            OperatingSystem
-# Resource           ./global.robot
-# Resource           ${RESOURCES}/utils.resource
-# Suite Setup        Global Setup
-# Suite Teardown     Global Teardown
-
-
-# *** Keywords ***
-# Global Setup
-#     ${parent_dir}=         Join Path           ${CURDIR}        ..
-#     ${src_dir}=            Join Path           ${parent_dir}    data
-#     ${dest_dir}=           Get Test Data directory
-#     Set Global Variable    ${TEST_DATA_DIR}    ${dest_dir}
-#     Copy Directory         ${src_dir}          ${dest_dir}
-
-# Global Teardown
-#     Remove Directory with Content    ${TEST_DATA_DIR}
diff --git a/test/data/trustzone/nonsecurecode/nonsecure.cproject.yml b/test/data/trustzone/nonsecurecode/nonsecure.cproject.yml
@@ -17,5 +17,5 @@ project:
     - group: Non-secure Code
       files: 
         - file: main_ns.c 
-        - file: $Source(secure)$/interface.h
-        - file: $Output(secure)$_CMSE_Lib.o
+        - file: $ProjectDir(secure)$/interface.h
+        - file: $cmse-lib(secure)$
diff --git a/test/lib/execution_summary.py b/test/lib/execution_summary.py
@@ -37,7 +37,6 @@ def visit_test(self, test: TestCase):
     def end_result(self, result: Result):
         with open(self.markdown_file, "w") as f:
             f.write("# Robot Framework Report\n\n")
-            self.__write_test_env(f)
             f.write("## Summary\n\n")
             f.write("|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total|\n")
             f.write("|:----:|:----:|:-----:|:---:|\n")
@@ -46,28 +45,6 @@ def end_result(self, result: Result):
             self.__write_test_section(f, self.failed_tests, "Failed Tests", "|Tag|Test|Message|:clock1030: Duration|Suite|\n")
             self.__write_test_section(f, self.skipped_tests, "Skipped Tests", "|Tag|Test|Suite|\n")
 
-    def __write_test_env(self, file):
-        tool_dict = {"cbuild": shutil.which("cbuild"),
-                    "cpackget": shutil.which("cpackget"),
-                    "csolution": shutil.which("csolution")}
-
-        file.write("## Test Environment\n\n")
-        for tool, path in tool_dict.items():
-            version = "unknown"
-            version_cmd = f"{tool} -V"
-            try:
-                output = subprocess.run(version_cmd, shell=True, check=True, text=True, capture_output=True)
-                version_match = re.search(r"(\d+\.\d+\.\d+.*) \(C\)", output.stdout)
-                if version_match:
-                    version = version_match.group(1)
-            except subprocess.CalledProcessError:
-                pass
-
-            tool_info = f"- **{tool}**: `version {version}, {path}`"
-            file.write(f"{tool_info}\n")
-        file.write(f"- **CMSIS_PACK_ROOT**: `{os.getenv('CMSIS_PACK_ROOT')}`\n\n")
-
-
     def __write_test_section(self, file, test_dict, section_header, table_header):
         if len(test_dict) != 0:
             file.write(f"\n## {section_header}\n\n")

diff --git a/test/lib/utils.py b/test/lib/utils.py
@@ -0,0 +1,13 @@
+import glob
+from pathlib import Path
+
+def glob_files_in_directory(directory:str, pattern: str, recursive: bool):
+    yaml_files = glob.glob(directory + '/**/' + pattern, recursive=recursive)
+    return yaml_files
+
+def get_parent_directory_name(file_path:str):
+    parent_dir = Path(file_path).parent
+    return parent_dir.name
+
+def get_parent_directory_path(file_path:str):
+    return Path(file_path).parent.absolute()
diff --git a/test/test.robot → test/local_example_tests.robot b/test/test.robot → test/local_example_tests.robot
@@ -1,5 +1,5 @@
 *** Settings ***
-Documentation           Tests to verify the csolution examples can be compiled and executed
+Documentation           Tests to compile & execute local csolution examples
 Suite Setup             Global Setup
 Suite Teardown          Global Teardown
 Resource                resources${/}global.resource

diff --git a/test/github_tests.robot → test/remote_example_tests.robot b/test/github_tests.robot → test/remote_example_tests.robot
@@ -1,4 +1,5 @@
 *** Settings ***
+Documentation           Tests to compile & execute remote csolution examples
 Suite Setup             Global Setup
 Suite Teardown          Global Teardown
 Library                 lib${/}utils.py 
@@ -11,21 +12,31 @@ Test Template           Test github examples
 
 
 *** Variables ***
-${csolution-examples}       https://github.com/Open-CMSIS-Pack/csolution-examples.git
+${csolution-examples}       https://github.com/Open-CMSIS-Pack/csolution-examples
+${Hello_B-U585I-IOT02A}     https://github.com/Arm-Examples/Hello_B-U585I-IOT02A
+${Hello_FRDM-K32L3A6}       https://github.com/Arm-Examples/Hello_FRDM-K32L3A6
 
 
 *** Test Cases ***
-Test github example
-    ${csolution-examples}    ${Pass}
+# Test Csolution example
+#     ${csolution-examples}    ${Pass}
+
+# Test Hello_1 example
+#     ${Hello_B-U585I-IOT02A}    ${Pass}
+
+# Test Hello_2 example
+#     ${Hello_FRDM-K32L3A6}    ${Pass}
 
 *** Keywords ***
 Test github examples
-    [Arguments]                   ${github_url}        ${expect}    ${dest_dir_name}=${Remote_Example_Dir}
-    Checkout GitHub Repository    ${github_url}        ${TEST_DATA_DIR}${/}${dest_dir_name}
-    ${files}    Glob Files In Directory    ${test_data_dir}${/}${dest_dir_name}    *.csolution.yml    ${True}
+    [Arguments]    ${github_url}        ${expect}
+    ${dest_dir}=    Evaluate    "${github_url}".split('/')[-1]
+    ${dest_dir}=    Set Variable    ${TEST_DATA_DIR}${/}${Remote_Example_Dir}${/}${dest_dir}
+    Checkout GitHub Repository    ${github_url}    ${dest_dir}       
+    ${files}    Glob Files In Directory    ${dest_dir}    *.csolution.*    ${True}
     FOR    ${file}    IN    @{files}
         ${example_name}=    Get Parent Directory Name    ${file}
-        Run Keyword If    '${example_name}' == 'Hello'    
+        Run Keyword If    '${example_name}' != 'CubeMX'    
         ...    Run Csolution Project    ${file}    ${expect}
     END