[EIM-65] [Test] Add alternative mirrors test (#73)

Add test failure description Add non interactive test Add alternative mirror testing Update README file Update macOS runner version to mac-13 Removing terminal process kill method
espressif · Dec 5, 2024 · 82a8773 · 82a8773
1 parent fbcc575
commit 82a8773
Show file tree

Hide file tree

Showing 23 changed files with 1,054 additions and 354 deletions.
diff --git a/.github/workflows/build_rust.yaml b/.github/workflows/build_rust.yaml
@@ -27,7 +27,7 @@ jobs:
             package_name: windows-x64
           - os: macos-latest
             package_name: macos-aarch64
-          - os: macos-12
+          - os: macos-13
             package_name: macos-x64
 
     steps:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -20,10 +20,16 @@ jobs:
                 include:
                     - os: ubuntu-latest
                       package_name: linux-x64
+                      run_on: GitHub
                     - os: windows-latest
                       package_name: windows-x64
+                      run_on: GitHub
                     - os: macos-latest
                       package_name: macos-aarch64
+                      run_on: GitHub
+                    - os: ubuntu-latest
+                      package_name: linux-x64
+                      run_on: CNRunner
 
         steps:
             - name: Checkout repository
@@ -63,12 +69,12 @@ jobs:
               run: |
                   chmod +x ./test-bin/eim
 
-            - name: Run prerequisites test script (non-Windows)
-              if: runner.os != 'Windows'
+            - name: Run prerequisites test script (non-Windows), skip for CNRunner
+              if: runner.os != 'Windows' && matrix.run_on != 'CNRunner'
               run: |
                   export LOG_TO_FILE="true"
                   chmod +x ./tests/run_pre_test.sh
-                  . ./tests/run_pre_test.sh "../test-bin/eim" "idf-im-cli ${{ env.CLI_TAG }}"
+                  . ./tests/run_pre_test.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}"
 
             - name: Install dependencies (Ubuntu)
               if: runner.os == 'Linux'
@@ -81,11 +87,18 @@ jobs:
                   brew install cmake ninja dfu-util
 
             - name: Run IDF installation and post install test script (non-Windows)
-              if: runner.os != 'Windows'
+              if: runner.os != 'Windows' && matrix.run_on != 'CNRunner'
               run: |
                   export LOG_TO_FILE="true"
                   chmod +x ./tests/run_test.sh
-                  . ./tests/run_test.sh "../test-bin/eim" "idf-im-cli ${{ env.CLI_TAG }}"
+                  . ./tests/run_test.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}"
+
+            - name: Run IDF installation from alternative mirrors in mainland China
+              if: matrix.run_on == 'CNRunner'
+              run: |
+                  export LOG_TO_FILE="true"
+                  chmod +x ./tests/run_cnrunner.sh
+                  . ./tests/run_cnrunner.sh "../test-bin/eim" "eim ${{ env.CLI_TAG }}"
 
             # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -114,7 +127,7 @@ jobs:
               if: runner.os == 'Windows'
               run: |
                   $env:LOG_TO_FILE="true"
-                  .\tests\run_pre_test.ps1 "..\test-bin\eim.exe" "idf-im-cli ${{ env.CLI_TAG }}"
+                  .\tests\run_pre_test.ps1 "..\test-bin\eim.exe" "eim ${{ env.CLI_TAG }}"
 
             - name: Install dependencies (Windows)
               if: runner.os == 'windows'
@@ -125,19 +138,23 @@ jobs:
               if: runner.os == 'Windows'
               run: |
                   $env:LOG_TO_FILE="true"
-                  .\tests\run_test.ps1 "..\test-bin\eim.exe" "idf-im-cli ${{ env.CLI_TAG }}"
+                  .\tests\run_test.ps1 "..\test-bin\eim.exe" "eim ${{ env.CLI_TAG }}"
 
             # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
             - name: Upload test results
               uses: actions/upload-artifact@v4
               if: always()
               with:
-                  name: test-results-${{ matrix.package_name }}.zip
+                  name: test-results-${{ matrix.package_name }}-${{matrix.run_on}}.zip
                   path: |
                       ./tests/results-pre-test.xml
+                      ./tests/results-pre-install-test.xml
                       ./tests/results-default-test.xml
                       ./tests/results-variation1-test.xml
+                      ./tests/results-non-interactive-test.xml
+                      ./tests/results-CNMirror-test.xml
+                      ./tests/results-CNMirror2-test.xml
                       ./tests/test.log
 
     publish-test-results:

diff --git a/tests/README.md b/tests/README.md
@@ -11,22 +11,29 @@ All tests are developed in Node.js using Chain and Mocha as test libraries in co
 
 On the test machine, the first step is to copy the testing artifacts. The location of the artifacts can be set using environment variable, or the test will look for the `eim` file in the default location:
 
-Windows: $USERPROFILE\eim-cli\
-Linux/MacOS: $HOME/eim-cli/
+Windows: `$USERPROFILE\eim-cli\`  
+Linux/MacOS: `$HOME/eim-cli/`
 
 ### Windows
 
 Install chocolatey package manager:
-https://docs.chocolatey.org/en-us/choco/setup/
-Run this command with administrator priviledges.
+
+> https://docs.chocolatey.org/en-us/choco/setup/
+
+Run this command with administrator privileges.
+
 `Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))`
 
 Install Node.js:
-https://nodejs.org/en/download/prebuilt-installer/current
-`choco install nodejs-lts --version="22.11.0" -y`
+
+> https://nodejs.org/en/download/prebuilt-installer/current
+
+`choco install nodejs-lts --version="20.18.1" -y`
 
 Install git:
-https://git-scm.com/download/win
+
+> https://git-scm.com/download/win
+
 `choco install git.install -y`
 
 Clone the public repository:
@@ -36,99 +43,114 @@ Clone the public repository:
 ### Linux:
 
 Install Git and curl and build-essential packages
+
 `sudo apt install -y git curl build-essential`
+
 `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash`
 
 Start a new terminal (to load nvm)
-`nvm install 22`
+
+`nvm install 20`
 
 Clone the public repository:
+
 `git clone https://github.com/espressif/idf-im-cli.git`
 
-> At his point test for prerequisits can be run, the remaining tests requires the pre-requisites to be installed.
+> **At his point test for prerequisites can be run, the remaining tests requires the pre-requisites to be installed.**
 
 Install ESP-IDF pre-requisites
-https://docs.espressif.com/projects/esp-idf/en/v5.3.1/esp32/get-started/linux-macos-setup.html
+
+> https://docs.espressif.com/projects/esp-idf/en/v5.3.1/esp32/get-started/linux-macos-setup.html
+
 `sudo apt install git cmake ninja-build wget flex bison gperf ccache libffi-dev libssl-dev dfu-util libusb-dev python3 python3-venv python3-pip`
 
 ### MacOS
 
 Install homebrew package manager if not already installed:
-https://brew.sh/
+
+> https://brew.sh/
+
 `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"`
 
 Install node.js
-https://nodejs.org/en/download/package-manager
-`brew install node@22`
-`echo 'export PATH="/usr/local/opt/node@22/bin:$PATH"' >> ~/.zshrc`
+
+> https://nodejs.org/en/download/package-manager
+
+`brew install node@20`
+
+`echo 'export PATH="/usr/local/opt/node@20/bin:$PATH"' >> ~/.zshrc`
 
 > This requires to restart the terminal in order to load Node.JS
 
 Install git
-https://git-scm.com/downloads/mac
+
+> https://git-scm.com/downloads/mac
+
 `brew install git`
 
 Clone the public repository:
+
 `git clone https://github.com/espressif/idf-im-cli.git`
 
-> At his point test for prerequisites can be run, the remaining tests requires the pre-requisites to be installed.
+> **At his point test for prerequisites can be run, the remaining tests requires the pre-requisites to be installed.**
 
 Install ESP-IDF pre-requisites
-https://docs.espressif.com/projects/esp-idf/en/v5.3.1/esp32/get-started/linux-macos-setup.html
+
+> https://docs.espressif.com/projects/esp-idf/en/v5.3.1/esp32/get-started/linux-macos-setup.html
+
 `brew install cmake ninja dfu-util`
 
 ## Commands summary
 
 Navigate to the idf-im-cli folder, where the repository was cloned.
+
 The scripts should be executed passing as arguments the path to the `eim` application and the version of the file being tested.
 
 #### Windows
 
-Open Powershell, and enable script execution:
+Open Powershell, and enable script execution:  
 `Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass`
 
-Prerequisites test can be executed by running:
-`.\tests\run_pre_test.ps1 "<PATH TO EIM.EXE>" "<Version being tested>"`
-Default arguments are:
-`.\tests\run_pre_test.ps1 "$env:USERPROFILE\eim-cli\eim.exe" "idf-im-cli 0.1.4"`
+Prerequisites test can be executed by running:  
+`.\tests\run_pre_test.ps1 "<PATH TO EIM.EXE>" "<Version being tested>"`  
+Default arguments are:  
+`.\tests\run_pre_test.ps1 "$env:USERPROFILE\eim-cli\eim.exe" "idf-im-cli 0.1.5"`
 
-To execute tests on windows, use the script
-`.\tests\run_test.ps1 "<PATH TO EIM.EXE>" "<Version being tested>"`
-Default arguments are:
-`.\tests\run_test.ps1 "$env:USERPROFILE\eim-cli\eim.exe" "idf-im-cli 0.1.4"`
+To execute tests on windows, use the script  
+`.\tests\run_test.ps1 "<PATH TO EIM.EXE>" "<Version being tested>"`  
+Default arguments are:  
+`.\tests\run_test.ps1 "$env:USERPROFILE\eim-cli\eim.exe" "idf-im-cli 0.1.5"`
 
 #### Linux
 
-(if needed) Give execution permission to the test script
+(if needed) Give execution permission to the test script  
 `chmod +x ./tests/run_test.sh`
 
-Prerequisites test can be executed by running:
-`. ./tests/run_pre_test.sh "<PATH TO EIM>" "<Version being tested>"`
-Default arguments are:
-`. ./tests/run_pre_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.4"`
+Prerequisites test can be executed by running:  
+`. ./tests/run_pre_test.sh "<PATH TO EIM>" "<Version being tested>"`  
+Default arguments are:  
+`. ./tests/run_pre_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"`
 
-To execute tests on linux, use the script:
-`. ./tests/run_test.sh "<PATH TO EIM>" "<Version being tested>"`
-Default arguments are:
-`. ./tests/run_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.4"`
+To execute tests on linux, use the script:  
+`. ./tests/run_test.sh "<PATH TO EIM>" "<Version being tested>"`  
+Default arguments are:  
+`. ./tests/run_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"`
 
 #### MacOS
 
-Prerequisites test can be executed by running:
-`. ./tests/run_pre_test.sh "<PATH TO EIM>" "<Version being tested>"`
-Default arguments are:
-`. ./tests/run_pre_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.3"`
-
-To execute tests on linux, use the script:
-`. ./tests/run_test.sh "<PATH TO EIM>" "<Version being tested>"`
-Default arguments are:
-`. ./tests/run_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.3"`
+Prerequisites test can be executed by running:  
+`. ./tests/run_pre_test.sh "<PATH TO EIM>" "<Version being tested>"`  
+Default arguments are:  
+`. ./tests/run_pre_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"`
 
-<TODO>
+To execute tests on linux, use the script:  
+`. ./tests/run_test.sh "<PATH TO EIM>" "<Version being tested>"`  
+Default arguments are:  
+`. ./tests/run_test.sh "$HOME/eim-cli/eim" "idf-im-cli 0.1.5"`
 
 # Installation Manager Usage
 
-##Application arguments
+## Application arguments
 
 ```
 Options:
@@ -166,14 +188,14 @@ Options:
           Should the installer attempt to install all missing prerequisites (default false). This flag only affects Windows platforms as we do not offer prerequisites for other platforms.
           [possible values: true, false]
       --config-file-save-path <CONFIG_FILE_SAVE_PATH>
-          if set, the installer will as it's very last move save the configuration to the specified file path. This file can than be used to repeat the instalation with the same settings.
+          if set, the installer will as it's very last move save the configuration to the specified file path. This file can than be used to repeat the installation with the same settings.
   -h, --help
           Print help (see a summary with '-h')
   -V, --version
           Print version
 ```
 
-## Example config file:
+## Example config file
 
 file config.toml (Linux)
 
@@ -225,15 +247,26 @@ idf_mirror = "https://github.com"
 
 ## References
 
-Packages required by EIM:
+Alternative Mirrors:
 
-Windows: eim should be able to perform all requirements installation
+IDF:  
+https://github.com  
+https://jihulab.com/esp-mirror
 
-Linux: sudo apt install git cmake ninja-build wget flex bison gperf ccache libffi-dev libssl-dev dfu-util libusb-dev python3 python3-venv python3-pip
+Tools:  
+https://github.com  
+https://dl.espressif.com/github_assets  
+https://dl.espressif.cn/github_assets
 
-MacOS:
-Install homebrew and load the application to the terminal profile
+Packages required by EIM:
 
-`/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"`
+Windows:  
+`eim should be able to perform all requirements installation`
+
+Linux:  
+`sudo apt install git cmake ninja-build wget flex bison gperf ccache libffi-dev libssl-dev dfu-util libusb-dev python3 python3-venv python3-pip`
 
-Then run: brew install dfu-util cmake ninja python3
+MacOS:  
+Install homebrew and load the application to the terminal profile  
+`/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"`  
+Then run: `brew install dfu-util cmake ninja python3`