[EIM-104] [Test] Automated test refactor (#87)

Add default IDF version read on workflow
Refactor of the test infrastructure
Update test descriptions, add check for IDF ver.
Simplified test command, update report upload

.github/workflows/test.yml

@@ -57,6 +57,14 @@ jobs:
                   STRIPPED_TAG=${LATEST_TAG#v}
                   echo "CLI_TAG=$STRIPPED_TAG" >> $GITHUB_ENV
 
+            - name: Fetch latest IDF release
+              if: runner.os != 'Windows'
+              run: |
+                  curl -v -s https://dl.espressif.com/dl/esp-idf/idf_versions.json > IDF_latest_release.json
+                  IDF_TAG=$(jq -r .VERSIONS[1].name IDF_latest_release.json)
+                  echo "IDF_TAG=$IDF_TAG" >> $GITHUB_ENV
+                  echo "Latest IDF for default install: $IDF_TAG"
+
             - name: Extract artifact (non-Windows)
               if: runner.os != 'Windows'
               run: |
@@ -70,35 +78,66 @@ jobs:
                   chmod +x ./test-bin/eim
 
             - name: Run prerequisites test script (non-Windows), skip for CNRunner
+              continue-on-error: true
               if: runner.os != 'Windows' && matrix.run_on != 'CNRunner'
               run: |
                   export LOG_TO_FILE="true"
-                  chmod +x ./tests/run_pre_test.sh
-                  . ./tests/run_pre_test.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}"
+                  export EIM_FILE_PATH="../test-bin/eim"
+                  export EIM_VERSION="eim ${{ env.CLI_TAG }}"
+                  cd tests
+                  npm ci
+                  npm run pre-test
 
             - name: Install dependencies (Ubuntu)
-              if: runner.os == 'Linux'
+              if: |
+                  runner.os == 'Linux'
               run: |
                   sudo apt-get install -y git cmake ninja-build wget flex bison gperf ccache libffi-dev libssl-dev dfu-util libusb-1.0-0-dev python3 python3-venv python3-pip
 
             - name: Install dependencies (MacOS)
-              if: runner.os == 'macOS'
+              if: |
+                  runner.os == 'macOS'
               run: |
                   brew install cmake ninja dfu-util
 
-            - name: Run IDF installation and post install test script (non-Windows)
-              if: runner.os != 'Windows' && matrix.run_on != 'CNRunner'
+            - name: Run IDF basic install test script (non-Windows)
+              continue-on-error: true
+              if: |
+                  runner.os != 'Windows' && 
+                  matrix.run_on != 'CNRunner'
               run: |
                   export LOG_TO_FILE="true"
-                  chmod +x ./tests/run_test.sh
-                  . ./tests/run_test.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}"
+                  export EIM_FILE_PATH="../test-bin/eim"
+                  export EIM_VERSION="eim ${{ env.CLI_TAG }}"
+                  export IDF_VERSION="${{ env.IDF_TAG }}"
+                  cd tests
+                  npm run test --file=basic-test
+
+            - name: Run IDF extended install test script (non-Windows)
+              continue-on-error: true
+              if: |
+                  runner.os != 'Windows' && 
+                  matrix.run_on != 'CNRunner'
 
-            - name: Run IDF installation from alternative mirrors in mainland China
+              run: |
+                  export LOG_TO_FILE="true"
+                  export EIM_FILE_PATH="../test-bin/eim"
+                  export EIM_VERSION="eim ${{ env.CLI_TAG }}"
+                  export IDF_VERSION="${{ env.IDF_TAG }}"                 
+                  cd tests
+                  npm run test --file=extended-test
+
+            - name: Run IDF installation from alternative mirrors
+              continue-on-error: true
               if: matrix.run_on == 'CNRunner'
               run: |
                   export LOG_TO_FILE="true"
-                  chmod +x ./tests/run_cnrunner.sh
-                  . ./tests/run_cnrunner.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}"
+                  export EIM_FILE_PATH="../test-bin/eim"
+                  export EIM_VERSION="eim ${{ env.CLI_TAG }}"
+                  export IDF_VERSION="${{ env.IDF_TAG }}"                 
+                  cd tests
+                  npm ci
+                  npm run test --file=mirrors-test
 
             # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -110,35 +149,62 @@ jobs:
                   $STRIPPED_TAG = $LATEST_TAG -replace '^v', ''
                   echo "CLI_TAG=$STRIPPED_TAG" | Out-File -FilePath $env:GITHUB_ENV -Append
 
+            - name: Fetch latest IDF release
+              if: runner.os == 'Windows'
+              run: |
+                  $json = Invoke-RestMethod -Uri "https://dl.espressif.com/dl/esp-idf/idf_versions.json"
+                  $IDF_TAG = $json.VERSIONS[1].name
+                  echo "IDF_TAG=$IDF_TAG" | Out-File -FilePath $env:GITHUB_ENV -Append
+                  echo "Latest IDF for default install: $IDF_TAG"
+
             - name: Extract artifact (Windows)
               if: runner.os == 'Windows'
               run: |
                   mkdir -p test-bin
                   7z x ./artifacts/eim-${{ inputs.run_id }}-${{ matrix.package_name }}/eim.zip -otest-bin
                   # 7z x ./test-bin/eim.zip -otest-bin
 
-            - name: Print powershell and windows version (Windows)
-              if: runner.os == 'Windows'
-              run: |
-                  $PSVersionTable
-                  [System.Environment]::OSVersion.Version
-
             - name: Run prerequisites test script (Windows)
+              continue-on-error: true
               if: runner.os == 'Windows'
               run: |
                   $env:LOG_TO_FILE="true"
-                  .\tests\run_pre_test.ps1 "..\test-bin\eim.exe" "eim ${{ env.CLI_TAG }}"
+                  $env:EIM_FILE_PATH = "..\test-bin\eim.exe"
+                  $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}"
+                  Set-Location -Path "./tests"
+                  Expand-Archive node_modules.zip
+                  npm run pre-test
+                  npm run pre-install
 
             - name: Install dependencies (Windows)
-              if: runner.os == 'windows'
+              if: |
+                  runner.os == 'windows'
               run: |
                   choco install ninja -y
 
-            - name: Run IDF installation and post install test script (Windows)
-              if: runner.os == 'Windows'
+            - name: Run IDF basic install test script (Windows)
+              continue-on-error: true
+              if: |
+                  runner.os == 'Windows'
+              run: |
+                  $env:LOG_TO_FILE="true"
+                  $env:EIM_FILE_PATH = "..\test-bin\eim.exe"
+                  $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}"
+                  $env:IDF_VERSION = "${{ env.IDF_TAG }}"                  
+                  Set-Location -Path "./tests"
+                  npm run test-win --file=basic-test
+
+            - name: Run IDF extended install test script (Windows)
+              continue-on-error: true
+              if: |
+                  runner.os == 'Windows'
               run: |
                   $env:LOG_TO_FILE="true"
-                  .\tests\run_test.ps1 "..\test-bin\eim.exe" "eim ${{ env.CLI_TAG }}"
+                  $env:EIM_FILE_PATH = "..\test-bin\eim.exe"
+                  $env:EIM_VERSION = "eim ${{ env.CLI_TAG }}"
+                  $env:IDF_VERSION = "${{ env.IDF_TAG }}"                  
+                  Set-Location -Path "./tests"
+                  npm run test-win --file=extended-test
 
             # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -148,13 +214,7 @@ jobs:
               with:
                   name: test-results-${{ matrix.package_name }}-${{matrix.run_on}}.zip
                   path: |
-                      ./tests/results-pre-test.xml
-                      ./tests/results-pre-install-test.xml
-                      ./tests/results-default-test.xml
-                      ./tests/results-variation1-test.xml
-                      ./tests/results-non-interactive-test.xml
-                      ./tests/results-CNMirror-test.xml
-                      ./tests/results-CNMirror2-test.xml
+                      ./tests/results-*.json
                       ./tests/test.log
 
     publish-test-results:
@@ -164,13 +224,11 @@ jobs:
         if: always()
 
         steps:
-            - name: Download Artifacts
-              uses: actions/download-artifact@v4
-              with:
-                  path: ./artifacts
-
             - name: Publish Test Results
-              uses: EnricoMi/publish-unit-test-result-action@v2
+              uses: dorny/test-reporter@v1
               with:
-                  action_fail: true
-                  files: "./artifacts/**/*.xml"
+                  artifact: /test-results-(.*)/
+                  name: "Tests $1"
+                  path: "*.json"
+                  reporter: mocha-json
+                  fail-on-empty: "true"

tests/README.md

@@ -104,49 +104,31 @@ Install ESP-IDF pre-requisites
 
 Navigate to the idf-im-cli folder, where the repository was cloned.
 
-The scripts should be executed passing as arguments the path to the `eim` application and the version of the file being tested.
+The test runs are split into multiple files, each of them with an associated JSON test script and a NPM run script for each access.  
+The tests relies on environmental variables for the information below, the test scripts also have default values associated to this variables in case
+environmental variables are not found.
 
-#### Windows
+EIM_FILE_PATH -> Specify the path to the EIM application -> default value Windows: `$USERPROFILE\eim-cli\` Linux/MacOS: `$HOME/eim-cli/`  
+EIM_VERSION -> Version of the EIM application being tested -> default value "eim 0.1.6"  
+IDF_VERSION -> The latest released version of ESP-IDF, used on express installation by EIM -> default "v5.4"
 
-Open Powershell, and enable script execution:  
-`Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass`
+Option variables
 
-Prerequisites test can be executed by running:  
-`.\tests\run_pre_test.ps1 "<PATH TO EIM.EXE>" "<Version being tested>"`  
-Default arguments are:  
-`.\tests\run_pre_test.ps1 "$env:USERPROFILE\eim-cli\eim.exe" "idf-im-cli 0.1.5"`
+LOG_TO_FILE="true" -> Enable logs saved to text file on the execution folder -> default false  
+DEBUG="true" -> Enable debug level messages generated by the test scripts -> default false
 
-To execute tests on windows, use the script  
-`.\tests\run_test.ps1 "<PATH TO EIM.EXE>" "<Version being tested>"`  
-Default arguments are:  
-`.\tests\run_test.ps1 "$env:USERPROFILE\eim-cli\eim.exe" "idf-im-cli 0.1.5"`
+To modify the test parameters, modify the json files located at `/src/tests/runs/suites`
+then execute the tests by running the test npm script passing the test script file name as argument:
 
-#### Linux
+`npm run test --file=basic_test`
+`npm run test --file=extended_test`
+`npm run test --file=mirrors-test`
 
-(if needed) Give execution permission to the test script  
-`chmod +x ./tests/run_test.sh`
+> For Windows use `test-win`
 
-Prerequisites test can be executed by running:  
-`. ./tests/run_pre_test.sh "<PATH TO EIM>" "<Version being tested>"`  
-Default arguments are:  
-`. ./tests/run_pre_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"`
+The test for prerequisites test can be executed to check the detection of missing prerequisites (before they are installed in the system) by running:
 
-To execute tests on linux, use the script:  
-`. ./tests/run_test.sh "<PATH TO EIM>" "<Version being tested>"`  
-Default arguments are:  
-`. ./tests/run_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"`
-
-#### MacOS
-
-Prerequisites test can be executed by running:  
-`. ./tests/run_pre_test.sh "<PATH TO EIM>" "<Version being tested>"`  
-Default arguments are:  
-`. ./tests/run_pre_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"`
-
-To execute tests on linux, use the script:  
-`. ./tests/run_test.sh "<PATH TO EIM>" "<Version being tested>"`  
-Default arguments are:  
-`. ./tests/run_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"`
+`npm run pre-test`
 
 # Installation Manager Usage
 

tests/eim_config.toml

@@ -0,0 +1,16 @@
+path = "/home/fabricio/.espressif"
+idf_path = "/home/fabricio/.espressif/v5.4/esp-idf"
+esp_idf_json_path = "/home/fabricio/.espressif/tools"
+tool_download_folder_name = "dist"
+tool_install_folder_name = "tools"
+target = ["all"]
+idf_versions = ["v5.4"]
+tools_json_file = "tools/tools.json"
+idf_tools_path = "tools/idf_tools.py"
+config_file_save_path = "eim_config.toml"
+non_interactive = true
+wizard_all_questions = false
+mirror = "https://github.com"
+idf_mirror = "https://github.com"
+recurse_submodules = false
+install_all_prerequisites = false

tests/package.json

@@ -9,12 +9,9 @@
         "winston": "^3.15.0"
     },
     "scripts": {
-        "pre-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-pre-test.xml script/prerequisites.test.js",
-        "pre-install": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-pre-install-test.xml script/prerequisitesInstallRunner.test.js",
-        "default-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-default-test.xml runs/defaultInstall.test.js",
-        "variation1-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-variation1-test.xml runs/variation1.test.js",
-        "non-interactive-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-non-interactive-test.xml runs/nonInteractiveInstall.test.js",
-        "cnrunner1-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-CNMirror-test.xml runs/CNmirrors.test.js",
-        "cnrunner2-test": "mocha --exit --bail --reporter mocha-junit-reporter --reporter-options mochaFile=./results-CNmirrors2-test.xml runs/CNmirrors2.test.js"
+        "pre-test": "mocha --exit --reporter json --reporter-options output=./results-pre-requisites-test.json script/prerequisites.test.js",
+        "pre-install": "mocha --exit --bail --reporter json --reporter-options output=./results-pre-install-test.json script/prerequisitesInstallRunner.test.js",
+        "test": "export JSON_FILENAME=$npm_config_file && mocha --exit --reporter json --reporter-options output=./results-$JSON_FILENAME.json runs/testRun.test.js",
+        "test-win": "powershell -Command \"$env:JSON_FILENAME=$env:npm_config_file; mocha --exit --reporter json --reporter-options output=./results-$env:JSON_FILENAME.json runs/testRun.test.js\""
     }
 }