From b8479bb2ac14a1e850a10cc0b274ef1b69d7a229 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Thu, 9 Jan 2025 11:24:33 +0100 Subject: [PATCH 01/11] Update timeout --- .github/workflows/end-to-end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 61ae9d72c3..63a4043c96 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -69,7 +69,7 @@ jobs: name: End to end Tests needs: build-end-to-end-tests runs-on: macos-15 - timeout-minutes: 90 + timeout-minutes: 120 strategy: matrix: test-tag: [release, privacy, securityTest, adClick] From 731879953e5b75383ea387aebc66eaed4f1e7482 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Thu, 9 Jan 2025 11:56:11 +0100 Subject: [PATCH 02/11] Switch to use mobile dev action and manually parse the results --- .github/workflows/end-to-end.yml | 38 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 63a4043c96..2cb5d004b8 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -69,7 +69,7 @@ jobs: name: End to end Tests needs: build-end-to-end-tests runs-on: macos-15 - timeout-minutes: 120 + timeout-minutes: 150 strategy: matrix: test-tag: [release, privacy, securityTest, adClick] @@ -87,13 +87,39 @@ jobs: name: duckduckgo-ios-app path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app - - name: Install Maestro - run: | - export MAESTRO_VERSION=1.39.1; curl -Ls "https://get.maestro.mobile.dev" | bash - - name: End to End tests + id: upload + uses: mobile-dev-inc/action-maestro-cloud@v1.9.7 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ secrets.ROBIN_PROJECT_KEY }} + name: ${{ matrix.test-tag }}_${{ github.sha }} + timeout: 120 + app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app + workspace: .maestro + include-tags: ${{ matrix.test-tag }} + env: ONBOARDING_COMPLETED=true + ios-version: 17 + + - name: Process Robin Results + if: always() run: | - export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.ROBIN_API_KEY }} --project-id ${{ secrets.ROBIN_PROJECT_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=${{ matrix.test-tag }} DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/ + # Parse the JSON string + RESULTS='${{ steps.upload.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}' + + # Check if any flow was not successful + NON_SUCCESS_FLOWS=$(echo $RESULTS | jq '[.[] | select(.status != "SUCCESS")]') + NON_SUCCESS_COUNT=$(echo $NON_SUCCESS_FLOWS | jq 'length') + + if [ "$NON_SUCCESS_COUNT" -gt 0 ]; then + echo "Error: Some flows were not successful" + echo "Details of non-successful flows:" + echo $NON_SUCCESS_FLOWS | jq '.' + exit 1 + else + echo "All flows were successful" + fi + shell: bash # notify-failure: # name: Notify on failure From 215304b9e9630adb055b16cff215d2d8f4983799 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Fri, 10 Jan 2025 14:56:52 +0100 Subject: [PATCH 03/11] Increase timeout --- .github/workflows/end-to-end.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 2cb5d004b8..29c11f07d3 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -69,7 +69,7 @@ jobs: name: End to end Tests needs: build-end-to-end-tests runs-on: macos-15 - timeout-minutes: 150 + timeout-minutes: 240 strategy: matrix: test-tag: [release, privacy, securityTest, adClick] @@ -94,7 +94,7 @@ jobs: api-key: ${{ secrets.ROBIN_API_KEY }} project-id: ${{ secrets.ROBIN_PROJECT_KEY }} name: ${{ matrix.test-tag }}_${{ github.sha }} - timeout: 120 + timeout: 240 app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app workspace: .maestro include-tags: ${{ matrix.test-tag }} From 9a741ec66732aa425f04df59c88bd7207c39e6c3 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Fri, 10 Jan 2025 14:58:08 +0100 Subject: [PATCH 04/11] Update result parsing script --- .github/workflows/end-to-end.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 29c11f07d3..0d54acab21 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -107,14 +107,26 @@ jobs: # Parse the JSON string RESULTS='${{ steps.upload.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}' + # Check if RESULTS is empty or null + if [ -z "$RESULTS" ]; then + echo "Error: No results found" + exit 1 + fi + # Check if any flow was not successful - NON_SUCCESS_FLOWS=$(echo $RESULTS | jq '[.[] | select(.status != "SUCCESS")]') - NON_SUCCESS_COUNT=$(echo $NON_SUCCESS_FLOWS | jq 'length') - + NON_SUCCESS_FLOWS=$(echo "$RESULTS" | jq -r '[.[] | select(.status != "SUCCESS")]') + NON_SUCCESS_COUNT=$(echo "$NON_SUCCESS_FLOWS" | jq -r 'length') + + # Check if NON_SUCCESS_COUNT is a number + if ! [[ "$NON_SUCCESS_COUNT" =~ ^[0-9]+$ ]]; then + echo "Error: Invalid non-success count" + exit 1 + fi + if [ "$NON_SUCCESS_COUNT" -gt 0 ]; then echo "Error: Some flows were not successful" echo "Details of non-successful flows:" - echo $NON_SUCCESS_FLOWS | jq '.' + echo "$NON_SUCCESS_FLOWS" | jq '.' exit 1 else echo "All flows were successful" From 5dad5261177945e2704b33735f308ea951934612 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Fri, 10 Jan 2025 14:58:31 +0100 Subject: [PATCH 05/11] Decrease timeout for testing --- .github/workflows/end-to-end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 0d54acab21..70e9f2d687 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -94,7 +94,7 @@ jobs: api-key: ${{ secrets.ROBIN_API_KEY }} project-id: ${{ secrets.ROBIN_PROJECT_KEY }} name: ${{ matrix.test-tag }}_${{ github.sha }} - timeout: 240 + timeout: 5 app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app workspace: .maestro include-tags: ${{ matrix.test-tag }} From a025a2487f89fba277616999ff87ca763fd2f9b0 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Fri, 10 Jan 2025 15:55:22 +0100 Subject: [PATCH 06/11] Add debug prints --- .github/workflows/end-to-end.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 70e9f2d687..a2534a64ad 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -104,6 +104,12 @@ jobs: - name: Process Robin Results if: always() run: | + echo "Debug: MAESTRO_CLOUD_UPLOAD_STATUS = " + echo "${{ steps.upload.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }}" + + echo "Debug: MAESTRO_CLOUD_FLOW_RESULTS = " + echo "${{ steps.upload.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}" + # Parse the JSON string RESULTS='${{ steps.upload.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}' From a501626c0725f3a46dc8ba81f9ecee4d4546304b Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Mon, 13 Jan 2025 16:25:44 +0100 Subject: [PATCH 07/11] Temporarily use forked action --- .github/workflows/end-to-end.yml | 40 +------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index a2534a64ad..14a849e1e0 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -89,7 +89,7 @@ jobs: - name: End to End tests id: upload - uses: mobile-dev-inc/action-maestro-cloud@v1.9.7 + uses: loremattei/action-maestro-cloud@fix-timeout-handling with: api-key: ${{ secrets.ROBIN_API_KEY }} project-id: ${{ secrets.ROBIN_PROJECT_KEY }} @@ -100,44 +100,6 @@ jobs: include-tags: ${{ matrix.test-tag }} env: ONBOARDING_COMPLETED=true ios-version: 17 - - - name: Process Robin Results - if: always() - run: | - echo "Debug: MAESTRO_CLOUD_UPLOAD_STATUS = " - echo "${{ steps.upload.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }}" - - echo "Debug: MAESTRO_CLOUD_FLOW_RESULTS = " - echo "${{ steps.upload.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}" - - # Parse the JSON string - RESULTS='${{ steps.upload.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}' - - # Check if RESULTS is empty or null - if [ -z "$RESULTS" ]; then - echo "Error: No results found" - exit 1 - fi - - # Check if any flow was not successful - NON_SUCCESS_FLOWS=$(echo "$RESULTS" | jq -r '[.[] | select(.status != "SUCCESS")]') - NON_SUCCESS_COUNT=$(echo "$NON_SUCCESS_FLOWS" | jq -r 'length') - - # Check if NON_SUCCESS_COUNT is a number - if ! [[ "$NON_SUCCESS_COUNT" =~ ^[0-9]+$ ]]; then - echo "Error: Invalid non-success count" - exit 1 - fi - - if [ "$NON_SUCCESS_COUNT" -gt 0 ]; then - echo "Error: Some flows were not successful" - echo "Details of non-successful flows:" - echo "$NON_SUCCESS_FLOWS" | jq '.' - exit 1 - else - echo "All flows were successful" - fi - shell: bash # notify-failure: # name: Notify on failure From f9b988c96c37ed3d7296f95d21011fb53b86afce Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Tue, 14 Jan 2025 10:08:46 +0100 Subject: [PATCH 08/11] Fail on timeout --- .github/workflows/end-to-end.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 14a849e1e0..03b22db96e 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -100,6 +100,7 @@ jobs: include-tags: ${{ matrix.test-tag }} env: ONBOARDING_COMPLETED=true ios-version: 17 + strict-success: true # notify-failure: # name: Notify on failure From 72ea7630e04138803d588b97a7f34f2e44b5b15e Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Tue, 14 Jan 2025 10:52:18 +0100 Subject: [PATCH 09/11] Restore timeout setting --- .github/workflows/end-to-end.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 03b22db96e..8f350b8bc9 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -69,7 +69,7 @@ jobs: name: End to end Tests needs: build-end-to-end-tests runs-on: macos-15 - timeout-minutes: 240 + timeout-minutes: 300 strategy: matrix: test-tag: [release, privacy, securityTest, adClick] @@ -94,7 +94,7 @@ jobs: api-key: ${{ secrets.ROBIN_API_KEY }} project-id: ${{ secrets.ROBIN_PROJECT_KEY }} name: ${{ matrix.test-tag }}_${{ github.sha }} - timeout: 5 + timeout: 300 app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app workspace: .maestro include-tags: ${{ matrix.test-tag }} From 8144d2ac0899287b998df455ed5ea118fbcd5421 Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 15 Jan 2025 11:52:13 +0100 Subject: [PATCH 10/11] Update Robin action configuration --- .github/workflows/end-to-end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 8f350b8bc9..0845d1e9ba 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -100,7 +100,7 @@ jobs: include-tags: ${{ matrix.test-tag }} env: ONBOARDING_COMPLETED=true ios-version: 17 - strict-success: true + fail-on-timeout: true # notify-failure: # name: Notify on failure From 584b923e9aa4a530c4f715948e7fd31131db465f Mon Sep 17 00:00:00 2001 From: Lorenzo Mattei Date: Wed, 15 Jan 2025 12:01:43 +0100 Subject: [PATCH 11/11] Switch Robin project key to be a variable --- .github/workflows/ios-end-to-end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ios-end-to-end.yml b/.github/workflows/ios-end-to-end.yml index 0341201eb6..09bc34f33a 100644 --- a/.github/workflows/ios-end-to-end.yml +++ b/.github/workflows/ios-end-to-end.yml @@ -92,7 +92,7 @@ jobs: uses: loremattei/action-maestro-cloud@fix-timeout-handling with: api-key: ${{ secrets.ROBIN_API_KEY }} - project-id: ${{ secrets.ROBIN_PROJECT_KEY }} + project-id: ${{ vars.ROBIN_PROJECT_KEY }} name: ${{ matrix.test-tag }}_${{ github.sha }} timeout: 300 app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app