E2E test #28
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E test | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| cases: | |
| description: 'specific cases to be excuted. Sample: ["./aad/a.tests.ts", "./bot/b.tests.ts"]. Set empty to run all cases' | |
| required: false | |
| schedule: | |
| - cron: "0 22 * * *" | |
| push: | |
| branches: | |
| - renovate/** | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| cases: ${{ steps.schedule-cases.outputs.cases || steps.dispatch-cases.outputs.cases || steps.pr-cases.outputs.cases || steps.push-cases.outputs.cases }} | |
| env: | |
| AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }} | |
| AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }} | |
| M365_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| M365_TENANT_ID: ${{ secrets.TEST_TENANT_ID_2 }} | |
| CI_ENABLED: "true" | |
| M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME }} | |
| AZURE_DEVOPS_EXT_PAT: ${{ secrets.ADO_PAT }} | |
| steps: | |
| - name: Checkout (dev) | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/checkout@v3 | |
| - name: Checkout (main) | |
| if: ${{ github.event_name == 'push' }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: main | |
| - name: Setup node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Setup legacy-peer-deps | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| npm config set legacy-peer-deps true | |
| - name: Setup project | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 5 | |
| command: | | |
| npm run setup-metrics | |
| - name: Package CLI | |
| working-directory: packages/cli | |
| run: | | |
| rm -f *.tgz | |
| npm pack | |
| mv *.tgz teamsfx-cli.tgz | |
| - name: Upload to artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Teamsfx-cli | |
| path: ./packages/cli/teamsfx-cli.tgz | |
| - name: List cases for schedule | |
| id: schedule-cases | |
| if: ${{ github.event_name == 'schedule' }} | |
| working-directory: packages/cli/tests/e2e | |
| run: | | |
| cases=`find . -wholename "*.tests.ts" | jq -Rsc '[split("\n") | .[]| select(.!="")]'` | |
| echo "cases=$cases" >> $GITHUB_OUTPUT | |
| - name: List cases for dispatch | |
| id: dispatch-cases | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| working-directory: packages/cli/tests/e2e | |
| run: | | |
| inputCases='${{ github.event.inputs.cases }}' | |
| if [ -z "$inputCases" ]; then | |
| allCases=`find . -wholename "*.tests.ts" | jq -Rsc '[split("\n") | .[]| select(.!="")]'` | |
| echo "cases=$allCases" >> $GITHUB_OUTPUT | |
| else | |
| echo "cases=$inputCases" >> $GITHUB_OUTPUT | |
| fi | |
| - name: List cases for push | |
| id: push-cases | |
| if: ${{ github.event_name == 'push' }} | |
| working-directory: packages/cli/tests/e2e | |
| run: | | |
| cases=`find ./bot ./frontend -path "*.tests.ts" -and \ | |
| -not '(' -path '*/*.dotnet.tests.ts' -or -path '*/Blazor*' ')' \ | |
| | jq -Rsc '[split("\n") | .[]| select(.!="")]'` | |
| echo "cases=$cases" >> $GITHUB_OUTPUT | |
| - name: E2E Test clean | |
| working-directory: packages/cli | |
| run: | | |
| npm run test:e2e:clean | |
| execute-case: | |
| if: ${{ needs.setup.outputs.cases }} | |
| env: | |
| AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }} | |
| AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }} | |
| M365_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| M365_TENANT_ID: ${{ secrets.TEST_TENANT_ID_2 }} | |
| CI_ENABLED: "true" | |
| M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME }} | |
| AZURE_DEVOPS_EXT_PAT: ${{ secrets.ADO_PAT }} | |
| TEAMSFX_DEBUG_TEMPLATE: "true" | |
| NODE_ENV: "development" | |
| TEAMSFX_AAD_DEPLOY_ONLY: "true" | |
| SIDELOADING_SERVICE_ENDPOINT: ${{ secrets.SIDELOADING_SERVICE_ENDPOINT }} | |
| SIDELOADING_SERVICE_SCOPE: ${{ secrets.SIDELOADING_SERVICE_SCOPE }} | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cases: ${{ fromJson(needs.setup.outputs.cases) }} | |
| name: ${{ matrix.cases }} | |
| steps: | |
| - name: Checkout (dev) | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/checkout@v3 | |
| - name: Checkout (main) | |
| if: ${{ github.event_name == 'push' }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: main | |
| - name: Setup node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Setup Azure Functions Core Tools For Linux | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg | |
| sudo mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg | |
| sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-$(lsb_release -cs)-prod $(lsb_release -cs) main" > /etc/apt/sources.list.d/dotnetdev.list' | |
| sudo apt-get update | |
| sudo apt-get install azure-functions-core-tools-4 | |
| which func | |
| func --version | |
| - name: Setup legacy-peer-deps | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| npm config set legacy-peer-deps true | |
| - name: Setup .net | |
| uses: actions/setup-dotnet@v2 | |
| with: | |
| dotnet-version: 6.0.x | |
| - name: Setup project | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 5 | |
| command: | | |
| npm run setup | |
| - name: Link CLI | |
| run: | | |
| npm link --force | |
| - name: print system info | |
| run: | | |
| lscpu | |
| - name: Download samples(daily) | |
| if: github.event_name == 'schedule' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: OfficeDev/TeamsFx-Samples | |
| ref: dev | |
| path: packages/cli/tests/e2e/resource | |
| - name: Download samples(rc) | |
| if: github.event_name == 'workflow_dispatch' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: OfficeDev/TeamsFx-Samples | |
| ref: v3 | |
| path: packages/cli/tests/e2e/resource | |
| - name: Download samples(main) | |
| if: github.event_name == 'push' && startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') == false | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: OfficeDev/TeamsFx-Samples | |
| ref: main | |
| path: packages/cli/tests/e2e/resource | |
| - name: Download samples from another repo | |
| if: startsWith(matrix.cases, './samples/') && contains(matrix.cases, 'ProactiveMessage') | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: OfficeDev/Microsoft-Teams-Samples | |
| ref: main | |
| path: packages/cli/tests/e2e/resource | |
| - name: run test | |
| working-directory: packages/cli/tests/e2e | |
| run: | | |
| file=`find . -wholename "${{ matrix.cases }}"` | |
| if [ -z "$file" ]; then | |
| echo "can't find target case in $file" | |
| exit 1 | |
| else | |
| npx mocha --reporter mochawesome --timeout 1200000 $file | |
| fi | |
| - name: get report name | |
| id: get-report-name | |
| if: ${{ always() }} | |
| run: | | |
| name="${{ matrix.cases }}" | |
| name="${name//'.tests.ts'/}" | |
| name="${name//.\//}" | |
| name="${name//\//_}" | |
| echo "name=$name" >> $GITHUB_OUTPUT | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ github.event_name != 'schedule' || success() || (failure() && github.run_attempt >= 5) }} | |
| with: | |
| name: ${{ steps.get-report-name.outputs.name }} | |
| path: | | |
| ./packages/cli/tests/e2e/mochawesome-report/mochawesome.json | |
| tear-down: | |
| if: ${{ always() }} | |
| needs: execute-case | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| AZURE_ACCOUNT_OBJECT_ID: ${{ secrets.TEST_USER_OBJECT_ID }} | |
| AZURE_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.TEST_SUBSCRIPTION_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.TEST_TENANT_ID }} | |
| M365_ACCOUNT_NAME: ${{ secrets.TEST_USER_NAME }} | |
| M365_ACCOUNT_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} | |
| M365_TENANT_ID: ${{ secrets.TEST_TENANT_ID_2 }} | |
| CI_ENABLED: "true" | |
| M365_ACCOUNT_COLLABORATOR: ${{ secrets.TEST_COLLABORATOR_USER_NAME }} | |
| AZURE_DEVOPS_EXT_PAT: ${{ secrets.ADO_PAT }} | |
| steps: | |
| - name: Checkout (dev) | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| uses: actions/checkout@v3 | |
| - name: Checkout (main) | |
| if: ${{ github.event_name == 'push' }} | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: main | |
| - name: Setup node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Setup legacy-peer-deps | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| npm config set legacy-peer-deps true | |
| - name: Setup project | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 5 | |
| command: | | |
| npm run setup | |
| - name: E2E Test clean | |
| working-directory: packages/cli | |
| run: | | |
| npm run test:e2e:clean | |
| rerun: | |
| needs: tear-down | |
| if: ${{ (github.event_name == 'schedule' || github.event_name == 'push') && failure() && github.run_attempt < 5 }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: rerun | |
| run: | | |
| curl \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/${{ github.repository }}/actions/workflows/rerun.yml/dispatches \ | |
| -d '{"ref":"${{ github.ref_name }}","inputs":{"run_id":"${{ github.run_id }}", "max_attempts":"5"}}' | |
| report: | |
| needs: execute-case | |
| if: ${{ github.event_name == 'schedule' && (success() || (failure() && github.run_attempt >= 5)) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Install Dateutils | |
| run: | | |
| sudo apt install dateutils | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: ~/artifacts | |
| - name: List jobs | |
| id: list-jobs | |
| working-directory: packages/cli | |
| run: | | |
| page=1 | |
| jobs="[]" | |
| while : | |
| do | |
| url=https://api.github.com/repos/OfficeDev/TeamsFx/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs\?per_page\=100\&page\=$page | |
| resp=`curl -H "Accept: application/vnd.github.v3+json" -u:${{ secrets.GITHUB_TOKEN }} $url` | |
| new_jobs=`echo $resp | jq -cr '.jobs'` | |
| jobs=`jq -cr --slurp 'add' <(echo "$jobs") <(echo "$new_jobs")` | |
| has_next=`curl -I -H "Accept: application/vnd.github.v3+json" -u:${{ secrets.GITHUB_TOKEN }} $url | grep -Fi "link:" | grep "rel=\"last\"" || true` | |
| if [ -z "$has_next" ]; then | |
| break | |
| fi | |
| page=$((page+1)) | |
| done | |
| cases=`echo $jobs| jq -r '.[] | select(.name | contains("tests.ts")) | .name'` | |
| passed=0 | |
| failed=0 | |
| skipped=0 | |
| passed_lists="" | |
| failed_lists="" | |
| skipped_lists="" | |
| emails="[email protected];" | |
| while IFS= read -r case; | |
| do | |
| if [ -z "$case" ]; then | |
| continue | |
| fi | |
| file="tests/e2e/$case" | |
| elegant_path="${file//.\//}" | |
| started_at=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .steps[] | select(.name == "run test") | .started_at'` | |
| completed_at=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .steps[] | select(.name == "run test") | .completed_at'` | |
| duration=`dateutils.ddiff $started_at $completed_at -f "%Mm %Ss"` | |
| email="" | |
| if grep -q "@author" $file; then | |
| email=`grep '@author' $file | grep -i -o '[A-Z0-9._%+-]\+@[A-Z0-9.-]\+\.[A-Z]\{2,4\}'` | |
| fi | |
| author="" | |
| if [ -z "$email" ]; then | |
| author="N/A" | |
| else | |
| author="<a href=\\\"mailto:$email\\\"><span>$email</span></a>" | |
| fi | |
| url=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .html_url'` | |
| url="<a href=\\\"$url\\\">$elegant_path</a>" | |
| target_type="TS/JS" | |
| if [[ $case == *".dotnet."* ]]; then | |
| target_type=".NET" | |
| fi | |
| name="${case//'.tests.ts'/}" | |
| name="${name//.\//}" | |
| name="${name//\//_}" | |
| job_id=`echo $jobs | jq --arg case $case -r '.[] | select(.name == $case ) | .id'` | |
| report_file=`find ~/artifacts -wholename "*${name}*/mochawesome.json"` | |
| if [ ! -z "$report_file" ]; then | |
| echo "Found the $report_file with $job_id and $name" | |
| tests=`cat $report_file | jq -cr '[ .. | objects | with_entries(select(.key=="tests")) | select(. != {}) | select(.tests | type=="array") ] | map(.tests | .[]) | .[]'` | |
| lable="" | |
| while IFS= read -r test; | |
| do | |
| name=`echo $test | jq -cr .fullTitle` | |
| duration=`echo $test | jq -cr .duration` | |
| if [[ ! -z `echo $test | jq 'select(.pass==true)'` ]]; then | |
| passed=$((passed+1)) | |
| label="<span style=\\\"background-color:#2aa198;color:white;font-weight:bold;\\\">PASSED</span>" | |
| elif [[ ! -z `echo $test | jq 'select(.fail==true)'` ]]; then | |
| failed=$((failed+1)) | |
| label="<span style=\\\"background-color: #dc322f;color:white;font-weight:bold;\\\">FAILED</span>" | |
| if [[ ! -z "$email" && ! "$emails" == *"$email"* ]]; then | |
| emails="$emails;$email" | |
| fi | |
| elif [[ ! -z `echo $test | jq 'select(.skipped==true)'` || ! -z `echo $test | jq 'select(.pending==true)'` ]]; then | |
| skipped=$((skipped+1)) | |
| label="<span style=\\\"background-color: #b58900;color:white;font-weight:bold;\\\">SKIPPED</span>" | |
| fi | |
| row="<tr> <td style=\\\"text-align: left;\\\">$url</td> <td style=\\\"text-align: left;\\\">$name</td> <td style=\\\"text-align: center;\\\">$target_type</td> <td style=\\\"text-align: center;\\\">$label</td> <td style=\\\"text-align: center;\\\">$author</td> <td>$((duration/1000)) sec</td> </tr>" | |
| if [[ ! -z `echo $test | jq 'select(.pass==true)'` ]]; then | |
| passed_lists="$passed_lists $row" | |
| elif [[ ! -z `echo $test | jq 'select(.fail==true)'` ]]; then | |
| failed_lists="$failed_lists $row" | |
| elif [[ ! -z `echo $test | jq 'select(.skipped==true)'` || ! -z `echo $test | jq 'select(.pending==true)'` ]]; then | |
| skipped_lists="$skipped_lists $row" | |
| fi | |
| done <<< $tests | |
| else | |
| echo "Failed to find the $report_file with $job_id and $name" | |
| fi | |
| done <<< $cases | |
| body="<table class=\\\"w3-table w3-striped w3-bordered\\\"> <tr> <th>PATH</th> <th>CASE</th> <th>TARGET TYPE</th> <th>STATUS</th> <th>AUTHOR</th> <th>DURATION</th> </tr> $failed_lists $skipped_lists $passed_lists </table> <br />" | |
| total=$((passed+failed+skipped)) | |
| subject="TeamsFx E2E V3 Test Report ($passed/$total Passed, $failed/$total Failed, $skipped/$total Skipped)" | |
| if [ $failed -gt 0 ]; then | |
| subject="[FAILED] $subject" | |
| else | |
| subject="[PASSED] $subject" | |
| fi | |
| echo "body=$body" >> $GITHUB_OUTPUT | |
| echo "to=$emails" >> $GITHUB_OUTPUT | |
| echo "subject=$subject" >> $GITHUB_OUTPUT | |
| - name: Send E-mail to the whole team | |
| uses: satak/webrequest-action@master | |
| with: | |
| url: https://prod-30.eastus.logic.azure.com:443/workflows/9aa865da96054bd89749c2d4ce68df8e/triggers/manual/paths/invoke?api-version=2016-10-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=uIoB06NUHSnKoZeWjNDI9t4rrhbTBKxJNiBtDoMRSQs | |
| method: POST | |
| payload: | | |
| { | |
| "to": "${{ steps.list-jobs.outputs.to }}", | |
| "body": "${{ steps.list-jobs.outputs.body }}", | |
| "subject": "${{ steps.list-jobs.outputs.subject }}", | |
| "apiKey": "${{ secrets.MAIL_API_KEY }}" | |
| } |