Merge pull request #3851 from neos/task/use-saucectl

TASK: Use saucectl with configuration

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2.0
+version: 2.1
 
 aliases:
   - &workspace_root ~/neos-ui-workspace
@@ -40,6 +40,9 @@ aliases:
     paths:
       - .
 
+orbs:
+  gh: circleci/[email protected]
+
 jobs:
   checkout:
     docker:
@@ -89,9 +92,20 @@ jobs:
           MYSQL_ROOT_PASSWORD: not_a_real_password
     working_directory: *workspace_root
     steps:
+      - checkout
       - attach_workspace: *attach_workspace
       - restore_cache: *restore_app_cache
 
+      - gh/install
+      - run:
+          name: Login to GitHub
+          command: |
+            echo $AUTH_TOKEN_GITHUB | gh auth login --with-token
+      - run:
+          name: Install Sauce Connect
+          command: |
+            curl -L -o sauce-connect.deb https://saucelabs.com/downloads/sauce-connect/5.1.3/sauce-connect_5.1.3.linux_amd64.deb
+            sudo dpkg -i sauce-connect.deb
       - run: rm -rf /home/circleci/app/Packages/Application/Neos.Neos.Ui
       - run: cd /home/circleci/app/Packages/Application && mv ~/neos-ui-workspace Neos.Neos.Ui
       - run: |
@@ -108,46 +122,37 @@ jobs:
       - run: curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash
       - run: chmod +x ~/.nvm/nvm.sh
       - run:
+          name: Start Sauce Connect
+          background: true
+          command: sc run --username ${SAUCE_USERNAME} --access-key ${SAUCE_ACCESS_KEY} --tunnel-name "circleci-tunnel" --region "us-west-1" --proxy-localhost allow
+      - run:
+          name: Define target branch
+          command: |
+            TARGET_BRANCH=$(gh pr view $CIRCLE_PULL_REQUEST --json baseRefName --jq '.baseRefName')
+            echo "Target Branch: $TARGET_BRANCH"
+            # Save the variable to BASH_ENV to be able to access it in the next steps
+            echo "export TARGET_BRANCH=$TARGET_BRANCH" >> $BASH_ENV
+      - run:
+          name: Use target branch
+          command: |
+            echo "Using target branch: $TARGET_BRANCH"
+      - run:
+          name: Prepare and run e2e tests
           no_output_timeout: 30m
           command: |
             export NVM_DIR="$HOME/.nvm"
             [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
             cd /home/circleci/app/Packages/Application/Neos.Neos.Ui
             nvm install
             nvm use
-            make test-e2e-saucelabs > /home/circleci/app/Data/Logs/AcceptanceTesting.log
+            make test-e2e-saucelabs
       - store_artifacts:
           path: /home/circleci/app/Data/Logs
       - persist_to_workspace:
           root: /home/circleci/app/Data/Logs
           paths:
             - .
 
-  post-acceptance-tests-recordings:
-    environment:
-      FLOW_CONTEXT: Production
-    docker:
-      - image: cimg/php:8.2-node
-
-    steps:
-      - attach_workspace:
-          at: /home/circleci/app
-      - run:
-          name: Install GitHub CLI and jq
-          command: |
-            type -p curl >/dev/null || (sudo apt update && sudo apt install curl -y)
-            curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
-            && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
-            && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
-            && sudo apt update \
-            && sudo apt install gh -y
-      - run:
-          name: Run Script
-          command: |
-            JOB_IDS=$(cat /home/circleci/app/AcceptanceTesting.log | grep -o 'https://app.saucelabs.com/tests/[a-zA-Z0-9]\+' | sed 's/.*\///')
-            echo "Job IDs: $JOB_IDS"
-            /home/circleci/app/Build/comment-acceptance-tests.sh "$JOB_IDS" "$(basename "$CIRCLE_PULL_REQUEST")"
-
   php-unittests:
     environment:
       FLOW_CONTEXT: Production
@@ -186,9 +191,6 @@ workflows:
       - e2e:
           requires:
             - build_flow_app
-      - post-acceptance-tests-recordings:
-          requires:
-            - e2e
       - php-unittests:
           requires:
             - build_flow_app

.sauce/config1Dimension.yml

@@ -0,0 +1,54 @@
+apiVersion: v1alpha
+kind: testcafe
+showConsoleLog: true
+sauce:
+  region: us-west-1
+  concurrency: 1 # Controls how many suites are executed at the same time.
+  retries: 3
+  metadata:
+    tags:
+      - e2e
+      - $TARGET_BRANCH
+    build: $TARGET_BRANCH
+  tunnel:
+    name: "circleci-tunnel"
+testcafe:
+  version: 3.6.2
+# Controls what files are available in the context of a test run (unless explicitly excluded by .sauceignore).
+rootDir: ./
+suites:
+  - name: "One dimensional Tests in Firefox on Windows"
+    browserName: "firefox"
+    src:
+      - "Tests/IntegrationTests/Fixtures/1Dimension/*.e2e.js"
+    platformName: "Windows 10"
+    screenResolution: "1280x1024"
+    smartRetry:
+      failedOnly: true
+  - name: "One dimensional Tests in Firefox on MacOS"
+    browserName: "firefox"
+    src:
+      - "Tests/IntegrationTests/Fixtures/1Dimension/*.e2e.js"
+    platformName: "macOS 13"
+    screenResolution: "1440x900"
+    smartRetry:
+      failedOnly: true
+npm:
+  dependencies:
+    - testcafe-react-selectors
+
+# Controls what artifacts to fetch when the suites have finished.
+artifacts:
+  download:
+    match:
+      - neosui-two-dimensional-test-report.json
+      - console.log
+      - sauce-test-report.json
+    when: always
+    allAttempts: true
+    directory: ../../Data/Logs/saucelabs-artifacts/
+
+reporters:
+  json:
+    enabled: true
+    filename: neosui-one-dimensional-test-report.json

.sauce/config2Dimension.yml

@@ -0,0 +1,53 @@
+apiVersion: v1alpha
+kind: testcafe
+sauce:
+  region: us-west-1
+  concurrency: 1 # Controls how many suites are executed at the same time.
+  retries: 3
+  metadata:
+    tags:
+      - e2e
+      - $TARGET_BRANCH
+    build: $TARGET_BRANCH
+  tunnel:
+    name: "circleci-tunnel"
+testcafe:
+  version: 3.6.2
+# Controls what files are available in the context of a test run (unless explicitly excluded by .sauceignore).
+rootDir: ./
+suites:
+  - name: "Two dimensional Tests in Firefox on Windows"
+    browserName: "firefox"
+    src:
+      - "Tests/IntegrationTests/Fixtures/2Dimension/*.e2e.js"
+    platformName: "Windows 10"
+    screenResolution: "1280x1024"
+    smartRetry:
+      failedOnly: true
+  - name: "Two dimensional Tests in Firefox on MacOS"
+    browserName: "firefox"
+    src:
+      - "Tests/IntegrationTests/Fixtures/2Dimension/*.e2e.js"
+    platformName: "macOS 13"
+    screenResolution: "1440x900"
+    smartRetry:
+      failedOnly: true
+npm:
+  dependencies:
+    - testcafe-react-selectors
+
+# Controls what artifacts to fetch when the suites have finished.
+artifacts:
+  download:
+    match:
+      - neosui-two-dimensional-test-report.json
+      - console.log
+      - sauce-test-report.json
+    when: always
+    allAttempts: true
+    directory: ../../Data/Logs/saucelabs-artifacts/
+
+reporters:
+  json:
+    enabled: true
+    filename: neosui-two-dimensional-test-report.json

.sauceignore

@@ -0,0 +1,12 @@
+# This file instructs saucectl to not package any files mentioned here.
+.git/
+.github/
+.DS_Store
+.hg/
+.vscode/
+.idea/
+.gitignore
+.hgignore
+.gitlab-ci.yml
+.npmrc
+*.gif

Makefile

@@ -108,7 +108,7 @@ test:
 
 ## Executes integration tests on saucelabs.
 test-e2e-saucelabs:
-	bash Tests/IntegrationTests/e2e.sh "saucelabs:Firefox@latest:Windows 10"
+	bash Tests/IntegrationTests/e2e-saucelabs.sh
 
 ## Executes integration tests locally.
 test-e2e:

Tests/IntegrationTests/e2e-saucelabs.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Check if saucectl is installed
+if ! command -v saucectl &> /dev/null; then
+    echo "saucectl is not installed. Installing saucectl..."
+    # Install saucectl via npm (assuming npm is installed)
+    npm install -g saucectl
+fi
+
+# Check if SAUCE_USERNAME and SAUCE_ACCESS_KEY are set
+if [[ -n "$SAUCE_USERNAME" && -n "$SAUCE_ACCESS_KEY" ]]; then
+    echo "SAUCE_USERNAME and SAUCE_ACCESS_KEY are set. Configuring saucectl..."
+    #saucectl configure
+else
+    echo "SAUCE_USERNAME or SAUCE_ACCESS_KEY is not set. Skipping saucectl configuration."
+    exit 1
+fi
+
+cd ../../..
+
+rm -rf DummyDistributionPackages || true
+mv DistributionPackages DummyDistributionPackages
+mkdir DistributionPackages
+
+ln -s "../Packages/Application/Neos.Neos.Ui/Tests/IntegrationTests/SharedNodeTypesPackage" DistributionPackages/Neos.TestNodeTypes
+
+for fixture in Packages/Application/Neos.Neos.Ui/Tests/IntegrationTests/Fixtures/*/; do
+    echo "$fixture"
+    dimension=$(basename "$fixture")
+    echo "$dimension"
+
+    ln -s "../${fixture}SitePackage" DistributionPackages/Neos.TestSite
+
+    # TODO: optimize this
+    composer reinstall neos/test-nodetypes
+    composer reinstall neos/test-site
+    # make sure neos is installed even if patching led to the removal (bug)
+    composer update neos/neos-development-collection
+    ./flow flow:cache:flush --force
+    ./flow flow:cache:warmup
+    ./flow configuration:show --path Neos.ContentRepository.contentDimensions
+
+    if ./flow site:list | grep -q 'Node name'; then
+        ./flow site:prune '*'
+    fi
+    ./flow site:import --package-key=Neos.TestSite
+    ./flow resource:publish
+
+    cd Packages/Application/Neos.Neos.Ui
+    saucectl run --config .sauce/config${dimension}.yml
+    cd ../../..
+    rm -f DistributionPackages/Neos.TestSite
+
+done
+
+rm -rf DistributionPackages
+mv DummyDistributionPackages DistributionPackages