Bisect behat

README.md

@@ -47,6 +47,7 @@ You also need to set several environment variables, depending on your testing re
 | `BEHAT_SUITE`          | A behat suite, usually pointing to a Moodle theme       | Empty               | The theme to test with Behat. Valid options are `default` (meaning site default), and `classic` for 3.7 upwards. |
 | `RUNCOUNT`             | INTEGER                                                 | 1                   | Used to perform tests in a loop. Use with caution and always with tags. |
 |`MOODLE_CONFIG`         | JSON STRING                                             | Empty               | Custom Modle config to use during the execution. For example, if you want to set `$CFG->noreplyaddress = '[email protected]';`, the value of this variable should be `{"noreplyaddress":"[email protected]"}`. |
+| `GOOD_COMMIT` and `BAD_COMMIT`       | A couple of git references (commits, tags, branches supported) | Empty               | When both are defined, both the `phpunit` and the `behat` jobs will run an automated `git bisect` session to determine which commit broke the test. Note that, when running bisect sessions it's highly recommended to restrict the tests by some criteria (filter, tag, name, ...)|
 | `MOBILE_VERSION`       | `latest`, `next`                                        | Empty               | The Moodle app version to use when executing behat @app tests. |
 | `PLUGINSTOINSTALL`     | gitrepoplugin1\|gitfolderplugin1\|gitbranchplugin1;gitrepoplugin2\|gitfolderplugin2 | Empty | External plugins to install.<br/>The following information is needed for each plugin: gitrepo (mandatory), folder (mandatory) and branch (optional).<br/>The plugin fields should be separated by "\|" and each plugin should be separated using ";".<br/>Example: "https://github.com/moodlehq/moodle-local_mobile.git\|local/mobile\|MOODLE_37_STABLE;[email protected]:jleyva/moodle-block_configurablereports.git\|blocks/configurable_reports" |
 | `PLUGINSDIR`           | /path/to/your/plugins                                   | WORKSPACE/plugins   | The location of the plugins checkout. |

runner/main/jobtypes/behat/behat.sh

@@ -84,6 +84,10 @@ function behat_to_summary() {
     echo "== BEHAT_TIMING_FILENAME: ${BEHAT_TIMING_FILENAME}"
     echo "== BEHAT_INCREASE_TIMEOUT: ${BEHAT_INCREASE_TIMEOUT}"
     echo "== MOODLE_CONFIG: ${MOODLE_CONFIG}"
+    if [[ -n "${GOOD_COMMIT}" ]] || [[ -n "${BAD_COMMIT}" ]]; then
+        echo "== GOOD_COMMIT: ${GOOD_COMMIT}"
+        echo "== BAD_COMMIT: ${BAD_COMMIT}"
+    fi
     echo "== MOBILE_VERSION: ${MOBILE_VERSION}"
     echo "== MOBILE_APP_PORT: ${MOBILE_APP_PORT}"
     echo "== PLUGINSTOINSTALL: ${PLUGINSTOINSTALL}"
@@ -134,7 +138,7 @@ function behat_check() {
     verify_modules $(behat_modules)
 
     # These env variables must be set for the job to work.
-    verify_env UUID WORKSPACE SHAREDDIR ENVIROPATH WEBSERVER
+    verify_env UUID WORKSPACE SHAREDDIR ENVIROPATH WEBSERVER GOOD_COMMIT BAD_COMMIT
 }
 
 # Behat job type init.
@@ -162,10 +166,30 @@ function behat_config() {
         print_warning "BEHAT_PARALLEL is not a number or it's 0, setting it to 1."
         BEHAT_PARALLEL=1
     fi
+
+    # If GOOD_COMMIT and BAD_COMMIT are set, it means that we are going to run a bisect
+    # session, so we need to enable FULLGIT (to get access to complete repository clone).
+    if [[ -n "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]]; then
+        FULLGIT="yes"
+        # Also, we don't want to allow repetitions, parallels, reruns or timing in the bisect session.
+        RUNCOUNT=1
+        BEHAT_PARALLEL=1
+        BEHAT_RERUNS=0
+        BEHAT_TIMING_FILENAME=
+    fi
 }
 
-# Behat job setup.
+# Behat job type setup.
 function behat_setup() {
+    # If both GOOD_COMMIT and BAD_COMMIT are not set, we are going to run a normal session.
+    # (for bisect sessions we don't have to setup the environment).
+    if [[ -z "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then
+        behat_setup_normal
+    fi
+}
+
+# Behat job type setup for normal mode.
+function behat_setup_normal() {
     # If there is a timing filename configured, look for it within the workspace/timing folder.
     # And copy it to the shared folder that the docker-php container will use.
     if [[ -n "${BEHAT_TIMING_FILENAME}" ]]; then
@@ -184,33 +208,47 @@ function behat_setup() {
     echo
     echo ">>> startsection Initialising Behat environment at $(date)<<<"
     echo "============================================================================"
+    local initcmd
+    behat_initcmd initcmd # By nameref.
+
+    echo "Running: ${initcmd[*]}"
+
+    docker exec -t -u www-data "${WEBSERVER}" "${initcmd[@]}"
+    echo "============================================================================"
+    echo ">>> stopsection <<<"
+}
+
+# Returns (by nameref) an array with the command needed to init the Behat site.
+function behat_initcmd() {
+    local -n cmd=$1
     # We need to determine the init suite to use.
     local initsuite=""
     if [[ -n "$BEHAT_SUITE" ]]; then
         initsuite="-a=${BEHAT_SUITE}"
     fi
-    # Setup server folder permissions.
-    docker exec -t "${WEBSERVER}" bash -c 'chown -R www-data:www-data /var/www/*'
+
     # Build the complete init command.
-    local cmd=(
+    cmd=(
         php admin/tool/behat/cli/init.php
         "${initsuite}"
         -j="${BEHAT_PARALLEL}"
         --axe
     )
-
-    echo "Running: ${cmd[*]}"
-
-    docker exec -t -u www-data "${WEBSERVER}" \
-        "${cmd[@]}"
-
-    echo "============================================================================"
-    echo ">>> stopsection <<<"
 }
 
 # Behat job type run.
 function behat_run() {
-    # Run the job type.
+    # If both GOOD_COMMIT and BAD_COMMIT are not set, we are going to run a normal session.
+    if [[ -z "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then
+        behat_run_normal
+    else
+        # If GOOD_COMMIT and BAD_COMMIT are set, we are going to run a bisect session.
+        behat_run_bisect
+    fi
+}
+
+# PHPUnit job tye run for normal mode.
+function behat_run_normal() {    # Run the job type.
     echo
     if [[ RUNCOUNT -gt 1 ]]; then
         echo ">>> startsection Starting ${RUNCOUNT} Behat main runs at $(date) <<<"
@@ -298,6 +336,77 @@ function behat_run() {
     done
 }
 
+# Behat job tye run for bisect mode.
+function behat_run_bisect() {
+    # Run the job type.
+    echo
+    echo ">>> startsection Starting Behat bisect session at $(date) <<<"
+    echo "=== Good commit: ${GOOD_COMMIT}"
+    echo "=== Bad commit: ${BAD_COMMIT}"
+    echo "============================================================================"
+    # Start the bisect session.
+    docker exec -t -u www-data "${WEBSERVER}" \
+        git bisect start "${BAD_COMMIT}" "${GOOD_COMMIT}"
+
+    # Build the int command.
+    local initcmd
+    behat_initcmd initcmd # By nameref.
+
+    # Build the run command.
+    local runcmd
+    behat_main_command runcmd # By nameref.
+
+    # Prepare the commands to be injected into shell script (not needed for direct execution, only when injecting).
+    # (any element having spaces/ampersands needs to be explicitly quoted, --tags and --name are the usual suspects).
+    for i in "${!runcmd[@]}"; do
+        # Only if the element is an name=value option.
+        if [[ ${runcmd[$i]} == --*=* ]]; then
+            # Split the option in name and value.
+            local name="${runcmd[$i]%%=*}"
+            local value="${runcmd[$i]#*=}"
+            # If the value has spaces, quote it.
+            if [[ ${value} == *" "* ]]; then
+                runcmd[$i]="${name}=\"${value}\""
+            fi
+            # If the value has ampersand, quote it.
+            if [[ ${value} == *"&"* ]]; then
+                runcmd[$i]="${name}=\"${value}\""
+            fi
+        fi
+    done
+
+    # Generate the bisect.sh script that we are going to use to run the behat bisect session.
+    # (it runs both init and run commands together).
+    docker exec -i -u www-data "${WEBSERVER}" \
+        bash -c "cat > bisect.sh" <<- EOF
+			#!/bin/bash
+			${initcmd[@]} >/dev/null 2>&1; ${runcmd[@]}
+			exitcode=\$?
+			echo "============================================================================"
+			exit \$exitcode
+			EOF
+
+    # Run the bisect session.
+    echo "============================================================================"
+    docker exec -u www-data "${WEBSERVER}" \
+        git bisect run bash bisect.sh
+    EXITCODE=$?
+
+    # Print the bisect logs, for the records.
+    echo
+    echo "============================================================================"
+    echo "Bisect logs and reset:"
+    docker exec -u www-data "${WEBSERVER}" \
+        git bisect log
+
+    # Finish the bisect session.
+    docker exec -u www-data "${WEBSERVER}" \
+        git bisect reset
+
+    echo "============================================================================"
+    echo ">>> stopsection <<<"
+}
+
 # Behat job type teardown.
 function behat_teardown() {
     # Need to copy the updated timing file back to the workspace.

runner/main/jobtypes/phpunit/phpunit.sh

@@ -132,18 +132,6 @@ function phpunit_config() {
 
 # PHPUnit job type setup.
 function phpunit_setup() {
-    # If one of GOOD_COMMIT or BAD_COMMIT are not set, but the other is, error out.
-    if [[ -n "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then
-        exit_error "GOOD_COMMIT is set but BAD_COMMIT is not set."
-    fi
-    if [[ -z "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]]; then
-        exit_error "BAD_COMMIT is set but GOOD_COMMIT is not set."
-    fi
-    # If both GOOD_COMMIT and BAD_COMMIT are set and they are the same, error out.
-    if [[ -n "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]] && [[ "${GOOD_COMMIT}" == "${BAD_COMMIT}" ]]; then
-        exit_error "GOOD_COMMIT and BAD_COMMIT are set, but they are the same."
-    fi
-
     # If both GOOD_COMMIT and BAD_COMMIT are not set, we are going to run a normal session.
     # (for bisect sessions we don't have to setup the environment).
     if [[ -z "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then
@@ -251,6 +239,13 @@ function phpunit_run_bisect() {
         git bisect run bash bisect.sh
     EXITCODE=$?
 
+    # Print the bisect logs, for the records.
+    echo
+    echo "============================================================================"
+    echo "Bisect logs and reset:"
+    docker exec -u www-data "${WEBSERVER}" \
+        git bisect log
+
     # Finish the bisect session.
     docker exec -u www-data "${WEBSERVER}" \
         git bisect reset

runner/main/modules/git/git.sh

@@ -47,3 +47,19 @@ function git_config() {
         GIT_COMMIT=$(cd "${CODEDIR}" && git rev-parse HEAD)
     fi
 }
+
+# Git module setup.
+function git_setup() {
+    # If one of GOOD_COMMIT or BAD_COMMIT are not set, but the other is, error out.
+    if [[ -n "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then
+        exit_error "GOOD_COMMIT is set but BAD_COMMIT is not set."
+    fi
+    if [[ -z "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]]; then
+        exit_error "BAD_COMMIT is set but GOOD_COMMIT is not set."
+    fi
+
+    # If both GOOD_COMMIT and BAD_COMMIT are set and they are the same, error out.
+    if [[ -n "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]] && [[ "${GOOD_COMMIT}" == "${BAD_COMMIT}" ]]; then
+        exit_error "GOOD_COMMIT and BAD_COMMIT are set, but they are the same."
+    fi
+}

test/behat_bisect_test.bats

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+#
+# This file is part of the Moodle Continuous Integration Project.
+#
+# Moodle is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Moodle is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Moodle.  If not, see <https://www.gnu.org/licenses/>.
+
+# Run the tests for the behat job type (bisect mode)
+
+setup() {
+    load 'helpers/common'
+    _common_setup
+}
+
+teardown() {
+    load 'helpers/common'
+    _common_teardown
+}
+
+@test "Behat bisect tests: run a known 4.3 regression (MDL-77991)" {
+    # Set all the required variables.
+    JOBTYPE="behat"
+    PHP_VERSION="8.2"
+    DBTYPE="pgsql"
+    CODEDIR="${MOODLE_CI_RUNNER_GITDIR}"
+    BEHAT_TAGS="@gradereport_grader&&@core&&~@mod_assign"
+    BEHAT_NAME="all the user to expand all of them at once"
+    BEHAT_PARALLEL=3
+    BEHAT_RERUNS=3
+    BEHAT_TIMING_FILENAME="wont_be_used.json"
+    GOOD_COMMIT="ce131c354aaebe40340e4f05334164d585087361"
+    BAD_COMMIT="567f4c0669e2a37f65d37c2950ccc76cb77a1484"
+
+    # Checkout
+    run git_moodle_checkout v4.3.2
+    assert_success
+
+    # Run the job
+    run launch_runner
+    assert_success
+    assert_output --partial "== JOBTYPE: behat"
+    assert_output --partial "== Moodle branch (version.php): 403"
+    assert_output --partial "== PHP version: 8.2"
+    assert_output --partial "== DBTYPE: pgsql"
+    assert_output --partial "== BEHAT_NAME: all the user to expand all of them at once"
+    assert_output --partial "== BEHAT_PARALLEL: 1"
+    assert_output --partial "== BEHAT_RERUNS: 0"
+    refute_output --partial "wont_be_used.json"
+    assert_output --partial "== GOOD_COMMIT: ce131c354aaebe40340e4f05334164d585087361"
+    assert_output --partial "== BAD_COMMIT: 567f4c0669e2a37f65d37c2950ccc76cb77a1484"
+    assert_output --partial "Setting up docker-selenium module..."
+    assert_output --partial "Bisecting:"
+    assert_output --partial "1be10f4249868e1bf1e9b44ba71b559c23e0cd06 is the first bad commit"
+    assert_output --partial "Merge branch 'MDL-77991' of https://github.com/Chocolate-lightning/moodle"
+    assert_output --partial "132 files changed, 3805 insertions(+), 2650 deletions(-)"
+    assert_output --partial "Bisect logs and reset:"
+    assert_output --partial "Exporting all docker logs for UUID"
+    assert_output --partial "Stopping and removing all docker containers"
+    assert_output --partial "== Exit code: 0"
+}
+
+@test "Behat bisect tests: only GOOD_COMMIT specified" {
+    # Set all the required variables.
+    JOBTYPE="phpunit"
+    PHP_VERSION="8.3"
+    DBTYPE="pgsql"
+    CODEDIR="${MOODLE_CI_RUNNER_GITDIR}"
+    PHPUNIT_FILTER="test_enrol_user_sees_own_courses"
+    GOOD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7"
+
+    # Checkout
+    run git_moodle_checkout v4.3.2
+    assert_success
+
+    # Run the job
+    run launch_runner
+    assert_failure
+    assert_output --partial "ERROR: GOOD_COMMIT is set but BAD_COMMIT is not set."
+}
+
+@test "Behat bisect tests: only BAD_COMMIT specified" {
+    # Set all the required variables.
+    JOBTYPE="phpunit"
+    PHP_VERSION="8.3"
+    DBTYPE="pgsql"
+    CODEDIR="${MOODLE_CI_RUNNER_GITDIR}"
+    PHPUNIT_FILTER="test_enrol_user_sees_own_courses"
+    BAD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7"
+
+    # Checkout
+    run git_moodle_checkout v4.3.2
+    assert_success
+
+    # Run the job
+    run launch_runner
+    assert_failure
+    assert_output --partial "ERROR: BAD_COMMIT is set but GOOD_COMMIT is not set."
+}
+
+@test "Behat bisect tests: same GOOD and BAD commits specified" {
+    # Set all the required variables.
+    JOBTYPE="phpunit"
+    PHP_VERSION="8.3"
+    DBTYPE="pgsql"
+    CODEDIR="${MOODLE_CI_RUNNER_GITDIR}"
+    GOOD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7"
+    BAD_COMMIT="${GOOD_COMMIT}"
+
+    # Checkout
+    run git_moodle_checkout v4.3.2
+    assert_success
+
+    # Run the job
+    run launch_runner
+    assert_failure
+    assert_output --partial "ERROR: GOOD_COMMIT and BAD_COMMIT are set, but they are the same."
+}