Robot tests EXTN

.github/workflows/e2e_test.yml

@@ -0,0 +1,116 @@
+name: E2E Tests
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/robot-tests.yml'
+      - 'test/**'
+      - '!test/README.md'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-latest, macos-13]
+        arch: [amd64, arm64]
+        include:
+          - os: windows-latest
+            target: win
+          - os: ubuntu-latest
+            target: lin
+          - os: macos-13
+            target: mac
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install pip dependencies
+        run: |
+            pip install --upgrade pip
+            pip install -r test/requirements.txt
+
+      - name: Setup vcpkg environment
+        uses: soumeh01/actions/vcpkg@testwindows
+        with:
+          config: "./test/vcpkg-configuration.json"
+          vcpkg-root: "${{ github.workspace }}/.vcpkg"
+          cache: "-"
+
+      - name: Activate Arm tool license
+        run: |
+          armlm activate --server https://mdk-preview.keil.arm.com --product KEMDK-COM0
+        working-directory: ./test
+
+      - name: Run Test
+        run: |
+          python -m robot --consolewidth 100 --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+
+      - name: Archieve test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: reports-${{ matrix.target }}-${{ matrix.arch }}
+          path: reports-${{ matrix.target }}-${{ matrix.arch }}
+
+  report:
+    runs-on: ubuntu-latest
+    if: always()
+    needs: test
+    permissions: write-all
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+
+    - name: Install pip dependencies
+      run: |
+          pip install --upgrade pip
+          pip install -r test/requirements.txt
+
+    - name: Download reports
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
+        pattern: reports-*
+
+    - name: Consolidate robot test results
+      working-directory: artifacts
+      run: |
+        python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
+          ./reports-win-amd64/output.xml ./reports-win-arm64/output.xml \
+          ./reports-lin-amd64/output.xml ./reports-lin-arm64/output.xml \
+          ./reports-mac-amd64/output.xml ./reports-mac-arm64/output.xml    
+
+    - name: Generate Summary report
+      if: always()
+      run: |
+        python ./test/lib/execution_summary.py artifacts/collective_robot_results/output.xml
+
+    - name: Print E2E Report
+      if: always()
+      run: |
+        cat summary_report.md >> $GITHUB_STEP_SUMMARY
+        echo "📜 Review the test result summary at: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+    - name: Archieve consolidated test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: consolidated-reports
+        path: artifacts/collective_robot_results

.gitignore

@@ -1 +1,7 @@
-build
+# Directories
+/build*/
+**/build
+**/__pycache__
+
+# Files
+**/stdout.txt

test/RobotTests.md

@@ -0,0 +1,96 @@
+# Installation and Running Robot Framework Tests
+
+This guide will walk you through the installation process and running of Robot Framework tests.
+
+## Prerequisites
+
+Before running Robot Framework tests, ensure you have the following prerequisites installed on your system:
+
+- Python (minimum recommended version **3.11**)
+- pip (python package manager)
+- [vcpkg](https://github.com/Open-CMSIS-Pack/cmsis-toolbox/blob/main/docs/installation.md#vcpkg---setup-using-cli)
+
+## Install Test Environment
+
+### Install Robot Framework
+
+Install Robot Framework and its dependencies using pip:
+
+```bash
+cd <root_dir>/test
+pip install --upgrade pip
+pip install -r requirements.txt
+```
+
+### Install Toolchains using vcpkg
+
+These commands will install all the required toolchains listed [here](./vcpkg-configuration.json):
+
+```bash
+cd <root_dir>/test
+vcpkg x-update-registry --all
+vcpkg activate
+```
+
+```txt
+👉 Important:
+    The AC6 toolchain installation includes the "fromelf" utility, which can be found in the AC6 toolchain installation
+    directory. Ensure that "fromelf" utlity is in PATH.
+```
+
+## Running Tests
+
+### Run all tests
+
+This command will run all tests located in the `test` directory and place the test reports and logs under specified directory.
+
+```bash
+robot -d <output_directory> <path_to_tests>
+```
+
+for e.g.
+
+```bash
+vcpkg activate
+robot -d results ./test
+```
+
+### Running Specific Tests
+
+To run specific tests, use the `--test` options:
+
+```bash
+robot --test <test_name> <path_to_tests>
+```
+
+for e.g.
+
+```bash
+vcpkg activate
+robot --test "Validate build-c Example"  test/test.robot
+```
+
+## Adding Tests
+
+The test cases in [test.robot](./test.robot) are implemented in a data-driven style, where each test case utilizes a single higher-level keyword to encapsulate the test workflow. To incorporate a new example for validation, follow the steps outlined below.
+
+- Add Example under [data](./data/) directory.
+- Add test details under **Test Cases** section following below conventions
+
+  ```robot
+  # <Name of the Test>
+  #    <Path to the input <project>.csolution.yml file>    <Expected build status>    <Example root directory name>
+  ```
+
+  for e.g.
+
+    ```robot
+    Validate USB Example
+        ${TEST_DATA_DIR}${/}${USB}${/}solution.csolution.yml    ${0}    ${USB}
+    ```
+
+```txt
+☑️ Note:
+    All options in the tests should be separated by "TABs".
+    For more information on robot test follow https://docs.robotframework.org/docs/testcase_styles/datadriven
+```

test/__init__.robot

@@ -0,0 +1,3 @@
+*** Settings ***
+Documentation       Initialization for the robot tests in this directory
+Test Timeout        10 minutes

test/data/build-asm/project/AC6/AsmArm.s

@@ -0,0 +1,16 @@
+
+     AREA   DATA
+
+     IF     HEXADECIMAL_TEST != 11259375 ; 0xABCDEF
+     INFO   1, "HEXADECIMAL_TEST failed!"
+     ENDIF
+
+     IF     DECIMAL_TEST != 1234567890
+     INFO   1, "DECIMAL_TEST failed!"
+     ENDIF
+
+     IF     STRING_TEST != "String0"
+     INFO   1, "STRING_TEST failed!"
+     ENDIF
+
+     END

test/data/build-asm/project/AC6/Auto.s

@@ -0,0 +1,9 @@
+
+     AREA   DATA
+
+     IF     :LNOT::DEF:AUTO_DEF
+     INFO   1, "AUTO_DEF is not defined!"
+     ENDIF
+
+     END
+

test/data/build-asm/project/AC6/GnuSyntax.s

@@ -0,0 +1,10 @@
+
+   .syntax unified
+   .arch armv7-m
+
+   .ifndef GAS_DEF
+   .error "GAS_DEF is not defined!"
+   .endif
+
+   .end
+

test/data/build-asm/project/AC6/PreProcessed.S

@@ -0,0 +1,9 @@
+
+   .syntax unified
+   .arch armv7-m
+
+#ifndef PRE_PROCESSED_DEF
+   .error "PRE_PROCESSED_DEF is not defined!"
+#endif
+
+   .end

test/data/build-asm/project/GCC/GAS.s

@@ -0,0 +1,9 @@
+
+   .syntax unified
+   .arch armv7-m
+
+   .ifndef GAS_DEF
+   .error "GAS_DEF is not defined!"
+   .endif
+
+   .end

test/data/build-asm/project/GCC/NonPreProcessed.s

@@ -0,0 +1,13 @@
+
+   .syntax unified
+   .arch armv7-m
+
+#ifndef PRE_PROCESSED_DEF
+   .equ SET_ERR_DEF,1
+#endif
+
+   .ifndef SET_ERR_DEF
+   .error "SET_ERR_DEF is not defined! It seems this file was preprocessed but it shouldn't!"
+   .endif
+
+   .end

test/data/build-asm/project/GCC/PreProcessed.S

@@ -0,0 +1,9 @@
+
+   .syntax unified
+   .arch armv7-m
+
+#ifndef PRE_PROCESSED_DEF
+   .error "PRE_PROCESSED_DEF is not defined!"
+#endif
+
+   .end

test/data/build-asm/project/IAR/Asm.s

@@ -0,0 +1,15 @@
+                MODULE   ?Asm
+
+#ifndef IAR_ASM_DEF
+#error "IAR_ASM_DEF is not defined!"
+#endif
+
+                EXTERN   Reset_Handler_C
+
+                PUBWEAK  Reset_Handler
+                SECTION  .text:CODE:REORDER:NOROOT(2)
+Reset_Handler
+                LDR      R0, =Reset_Handler_C
+                BX       R0
+
+                END

test/data/build-asm/project/main.c

@@ -0,0 +1,6 @@
+#include "RTE_Components.h"
+#include CMSIS_device_header
+
+int main(void) {
+  return 0;
+}

test/data/build-asm/project/project.cproject.yml

@@ -0,0 +1,53 @@
+project:
+
+  components:
+    - component: ARM::CMSIS:CORE
+    - component: ARM::Device:Startup&C Startup
+
+  groups:
+    - group: Source
+      files:
+        - file: ./main.c
+
+    - group: GCC-CLANG
+      for-compiler: [GCC, CLANG]
+      files:
+        - file: ./GCC/GAS.s
+          for-compiler: GCC
+          define:
+            - GAS_DEF
+        - file: ./GCC/PreProcessed.S
+          define:
+            - PRE_PROCESSED_DEF
+        - file: ./GCC/NonPreProcessed.s
+          misc:
+            - ASM:
+              - -DPRE_PROCESSED_DEF
+
+    - group: AC6
+      for-compiler: AC6
+      files:
+        - file: ./AC6/AsmArm.s
+          define:
+            - HEXADECIMAL_TEST: 0xABCDEF
+            - DECIMAL_TEST: 1234567890
+            - STRING_TEST: "\"String0\""
+        - file: ./AC6/GnuSyntax.s
+          define:
+            - GAS_DEF
+          misc:
+            - ASM:
+              - -masm=gnu
+        - file: ./AC6/PreProcessed.S
+          define:
+            - PRE_PROCESSED_DEF
+        - file: ./AC6/Auto.s
+          define:
+            - AUTO_DEF
+
+    - group: IAR
+      for-compiler: IAR
+      files:
+        - file: ./IAR/Asm.s
+          define:
+            - IAR_ASM_DEF