Test restructure

.github/workflows/robot-tests.yml

@@ -41,21 +41,6 @@ jobs:
             pip install --upgrade pip
             pip install -r test/requirements.txt
 
-      # - name: Cache vcpkg
-      #   uses: actions/cache@v4
-      #   with:
-      #     key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}
-      #     restore-keys: |
-      #         vcpkg-${{ runner.os }}-${{ runner.arch }}-
-      #     path: /home/runner/.vcpkg
-
-      # - name: Prepare vcpkg env
-      #   working-directory: ./test
-      #   run: |  
-      #     . <(curl https://aka.ms/vcpkg-init.sh -L)
-      #     vcpkg x-update-registry --all
-      #     vcpkg activate
-
       - name: Setup vcpkg environment
         uses: soumeh01/actions/vcpkg@testwindows
         with:
@@ -64,15 +49,13 @@ jobs:
           cache: "-"
 
       - name: Activate Arm tool license
-        working-directory: ./test
         run: |
           armlm activate --server https://mdk-preview.keil.arm.com --product KEMDK-COM0
+        working-directory: ./test
 
-      # python -m robot --outputdir ../reports-${{ matrix.os }} -x junit-report.xml src
       - name: Run Test
-        working-directory: ./test
         run: |
-          python -m robot --outputdir ../reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./src
+          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
 
       - name: Archieve test results
         if: always()
@@ -89,6 +72,17 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+
+    - name: Install pip dependencies
+      run: |
+          pip install --upgrade pip
+          pip install -r test/requirements.txt
+
     - name: Download reports windows amd64
       uses: actions/download-artifact@v4
       with:
@@ -126,13 +120,22 @@ jobs:
         path: reports-mac-arm64
 
     - name: Consolidate robot test results
+      continue-on-error: true
       run: |
-        python -m robot.rebot --name Collective_Robot_Results --outputdir Collective_Robot_Results \
+        python -m robot.rebot --name Collective_Robot_Results --outputdir Collective_Robot_Results --output output.xml \
           ./reports-win-amd64/output.xml ./reports-win-amd64/output.xml \
-          ./reports-ubuntu-arm64/output.xml ./reports-ubuntu-arm64/output.xml \
-          ./reports-macos-amd64/output.xml ./reports-macos-arm64/output.xml
+          ./reports-lin-arm64/output.xml ./reports-lin-arm64/output.xml \
+          ./reports-mac-amd64/output.xml ./reports-mac-arm64/output.xml
+
+    - name: HTML Preview
+      if: always()
+      id: html_preview
+      uses: pavi2410/html-preview-action@v2
+      with:
+        html_file: 'Collective_Robot_Results/log.html'
 
-    - name: Send macos report to commit
+    - name: Send report to commit
+      if: always()
       uses: joonvena/[email protected]
       with:
         report_path: Collective_Robot_Results

test/RobotTests.md

@@ -0,0 +1,70 @@
+# Installation and Running Robot Framework Tests
+
+This guide will walk you through the installation process and running of Robot Framework tests.
+
+## Prerequisites
+
+Before running Robot Framework tests, ensure you have the following prerequisites installed on your system:
+
+- Python (minimum recommended version **3.11**)
+- pip (python package manager)
+
+## Install Robot Framework
+
+Install Robot Framework and its dependencies using pip:
+
+```bash
+cd <root_dir>
+pip install --upgrade pip
+pip install -r test/requirements.txt
+```
+
+## Running Tests
+
+### Run all tests
+
+This command will run all tests located in the `test` directory and place the test reports and logs under specified directory.
+
+```bash
+robot -d <output_directory> <path_to_tests>
+robot -d results ./test
+```
+
+### Running Specific Tests
+
+To run specific tests, use the `--test` options:
+
+```bash
+robot --test <test_name> <path_to_tests>
+```
+
+for e.g.
+
+```bash
+robot --test "Validate build-c Example"  test/test.robot
+```
+
+## Adding Tests
+
+The test cases in [test.robot](./test.robot) are implemented in a data-driven style, where each test case utilizes a single higher-level keyword to encapsulate the test workflow. To incorporate a new example for validation, follow the steps outlined below.
+
+- Add Example under [data](./data/) directory.
+- Add test details under **Test Cases** section following below conventions
+
+  ```robot
+  # <Name of the Test>
+  #    <Path to the input <project>.csolution.yml file>    <Expected build status>    <Example root directory name>
+  ```
+
+  for e.g.
+
+    ```robot
+    Validate USB Example
+        ${TEST_DATA_DIR}${/}${USB}${/}solution.csolution.yml    ${0}    ${USB}
+    ```
+
+```txt
+☑️ Note:
+    All options in the tests should be separated by **TABs**.
+    For more information on robot test follow https://docs.robotframework.org/docs/testcase_styles/datadriven
+```

test/__init__.robot

@@ -0,0 +1,22 @@
+*** Settings ***
+Documentation       Initailization for the robot tests in this directory
+Test Timeout        10 minutes
+
+
+# Library            OperatingSystem
+# Resource           ./global.robot
+# Resource           ${RESOURCES}/utils.resource
+# Suite Setup        Global Setup
+# Suite Teardown     Global Teardown
+
+
+# *** Keywords ***
+# Global Setup
+#     ${parent_dir}=         Join Path           ${CURDIR}        ..
+#     ${src_dir}=            Join Path           ${parent_dir}    data
+#     ${dest_dir}=           Get Test Data directory
+#     Set Global Variable    ${TEST_DATA_DIR}    ${dest_dir}
+#     Copy Directory         ${src_dir}          ${dest_dir}
+
+# Global Teardown
+#     Remove Directory with Content    ${TEST_DATA_DIR}

test/resources/global.resource

@@ -0,0 +1,4 @@
+*** Variables ***
+${RESOURCES}        ${CURDIR}      # Path to the resource directory
+${Build}            build          # Directory name where example build files are copied
+${Data}             data           # Directory name of test data source

test/resources/utils.resource

@@ -1,21 +1,33 @@
 *** Settings ***
-Documentation    A resource file with reusable keywords
+Documentation    A collection of commonly used keywords across multiple test suites
 Library          Collections
 Library          Process
 Library          OperatingSystem
+Resource         global.resource
 
 
 *** Keywords ***
-Get Test Data directory
-    [Documentation]      Get test input data directory path
+Global Setup
+    ${parent_dir}=                  Join Path           ${CURDIR}        ..
+    ${src_dir}=                     Join Path           ${parent_dir}    ${Data}
+    ${dest_dir}=                    Get Test Data Directory
+    Set Global Variable             ${TEST_DATA_DIR}    ${dest_dir}
+    Copy Directory                  ${src_dir}          ${dest_dir}
+
+Global Teardown
+    Remove Directory with Content    ${TEST_DATA_DIR}
+
+Get Test Data Directory
+    [Documentation]      Retrieve the directory path for test input data
     ${parent_dir}=       Join Path           ${CURDIR}        ..
-    ${test_data_dir}=    Join Path           ${parent_dir}    run
+    ${test_data_dir}=    Join Path           ${parent_dir}    ${Build}
     RETURN               ${test_data_dir}
 
 Run Program
     [Documentation]             Run specified executable with arguments
     [Arguments]                 ${exe_path}      ${input_File}    @{args}
-    ${result}    Run Process    ${exe_path}      ${input_File}    @{args}    shell=True    stdout=${CURDIR}/stdout.txt
+    ${result}    Run Process    ${exe_path}      ${input_File}    @{args}
+    ...                         shell=True       stdout=${CURDIR}/stdout.txt
     Log Many                    StdOut: ${result.stdout}    Return Code: ${result.rc}
     RETURN                      ${result.rc}
 
@@ -48,19 +60,6 @@ Remove Directory with Content
 #     Log                             ${result.stderr}
 #     Should Be Equal As Integers     ${result.rc}          ${0}
 
-Global Setup
-    [Documentation]                 Global test set up
-    ${parent_dir}=                  Join Path           ${CURDIR}        ..
-    ${src_dir}=                     Join Path           ${parent_dir}    data
-    ${dest_dir}=                    Get Test Data directory
-    Set Global Variable             ${TEST_DATA_DIR}    ${dest_dir}
-    Copy Directory                  ${src_dir}          ${dest_dir}
-
-Global Teardown
-    [Documentation]                  Global tear down
-    Remove Directory with Content    ${TEST_DATA_DIR}
-
-*** Keywords ***
 Run Project With cbuild2cmake
     [Arguments]     ${input_file}    ${args}=@{EMPTY}
     ${ex_args}=     Append Additional Arguments     ${args}    --cbuild2cmake

test/src/test.robot → test/test.robot

@@ -1,15 +1,16 @@
 *** Settings ***
-Documentation           Tests to execute solution examples
-Resource                ./global.robot
-Resource                ${RESOURCES}/utils.resource
-Library                 Collections
-Library                 String
-Library                 ../lib/elf_compare.py 
+Documentation           Tests to verify the csolution examples can be compiled and executed
 Suite Setup             Global Setup
 Suite Teardown          Global Teardown
+Resource                resources${/}global.resource
+Resource                resources${/}utils.resource
+Library                 String
+Library                 Collections
+Library                 lib${/}elf_compare.py 
 Test Template           Run CSolution Project
 
 *** Variables ***
+# The directory name of the example to be built
 ${build-asm}                build-asm
 ${build-c}                  build-c
 ${build-cpp}                build-cpp
@@ -21,6 +22,9 @@ ${whitespace}               whitespace
 ${trustzone}                trustzone
 
 *** Test Cases ***
+# <Name of the Test>
+#    <Path to the input *.csolution.yml file>    <Expected build status>    <Example root directory name>
+
 Validate build-asm Example
     ${TEST_DATA_DIR}${/}${build-asm}${/}solution.csolution.yml    ${0}    ${build-asm}
 
@@ -33,20 +37,20 @@ Validate build-cpp Example
 Validate include-define Example
     ${TEST_DATA_DIR}${/}${include-define}${/}solution.csolution.yml    ${0}    ${include-define}
 
-Validate language-scope Example
-    ${TEST_DATA_DIR}${/}${language-scope}${/}solution.csolution.yml    ${0}    ${language-scope}
+# Validate language-scope Example
+#     ${TEST_DATA_DIR}${/}${language-scope}${/}solution.csolution.yml    ${0}    ${language-scope}
 
-Validate linker-pre-processing Example
-    ${TEST_DATA_DIR}${/}${linker-pre-processing}${/}solution.csolution.yml    ${0}    ${linker-pre-processing}
+# Validate linker-pre-processing Example
+#     ${TEST_DATA_DIR}${/}${linker-pre-processing}${/}solution.csolution.yml    ${0}    ${linker-pre-processing}
 
-Validate pre-include Example
-    ${TEST_DATA_DIR}${/}${pre-include}${/}solution.csolution.yml    ${0}    ${pre-include}
+# Validate pre-include Example
+#     ${TEST_DATA_DIR}${/}${pre-include}${/}solution.csolution.yml    ${0}    ${pre-include}
 
-Validate whitespace Example
-    ${TEST_DATA_DIR}${/}${whitespace}${/}solution.csolution.yml    ${0}    ${whitespace}
+# Validate whitespace Example
+#     ${TEST_DATA_DIR}${/}${whitespace}${/}solution.csolution.yml    ${0}    ${whitespace}
 
-Validate trustzone Example
-     ${TEST_DATA_DIR}${/}${trustzone}${/}solution.csolution.yml    ${0}    ${trustzone}
+# Validate trustzone Example
+#      ${TEST_DATA_DIR}${/}${trustzone}${/}solution.csolution.yml    ${0}    ${trustzone}
 
 *** Keywords ***
 Run Csolution Project
@@ -58,7 +62,7 @@ Run Csolution Project
     ...    Run Keyword And Return
     ...    Compare Elf Information   ${input_file}
     ...    ${TEST_DATA_DIR}${/}${example_name}${/}out_dir/out
-    ...     ${TEST_DATA_DIR}${/}${example_name}${/}out
+    ...    ${TEST_DATA_DIR}${/}${example_name}${/}out
     Should Be True    ${result}