diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 67420d24..4057aec2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,10 @@ This project adheres to [Semantic Versioning](http://semver.org/). The format of this change log follows the advice given at [Keep a CHANGELOG](http://keepachangelog.com). ## [Unreleased] +### Added +- Added support for the `--tags` and `--name` options to the `behat` command. +- Added support for the `--configure`, `--testsuite` and `--filter` options to the `phpunit` command. + ### Changed - ACTION SUGGESTED: If you are using GitHub Actions, it's recomended to use `!cancelled()` instead of `always()` for moodle-plugin-ci tests. Adding a final step that always returns failure when the workflow is cancelled will ensure that cancelled workflows are not marked as successful. For a working example, please reference the updated `gha.dist.yml` file. - ACTION SUGGESTED: For some (unknown) reason, Travis environments with PHP 8.2 have started to fail with error: diff --git a/docs/CLI.md b/docs/CLI.md index b7bb9c47..13e86ae6 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -241,7 +241,7 @@ Run Behat on a plugin ### Usage -* `behat [-m|--moodle MOODLE] [-p|--profile PROFILE] [--suite SUITE] [--start-servers] [--auto-rerun AUTO-RERUN] [--dump] [--] ` +* `behat [-m|--moodle MOODLE] [-p|--profile PROFILE] [--suite SUITE] [--tags TAGS] [--name NAME] [--start-servers] [--auto-rerun AUTO-RERUN] [--dump] [--] ` Run Behat on a plugin @@ -269,7 +269,7 @@ Path to Moodle #### `--profile|-p` -Behat profile to use +Behat profile option to use * Accept value: yes * Is value required: yes @@ -279,7 +279,7 @@ Behat profile to use #### `--suite` -Behat suite to use (Moodle theme) +Behat suite option to use (Moodle theme) * Accept value: yes * Is value required: yes @@ -287,6 +287,26 @@ Behat suite to use (Moodle theme) * Is negatable: no * Default: `'default'` +#### `--tags` + +Behat tags option to use. If not set, defaults to the component name + +* Accept value: yes +* Is value required: yes +* Is multiple: no +* Is negatable: no +* Default: `''` + +#### `--name` + +Behat name option to use + +* Accept value: yes +* Is value required: yes +* Is multiple: no +* Is negatable: no +* Default: `''` + #### `--start-servers` Start Selenium and PHP servers @@ -1955,7 +1975,7 @@ Run PHPUnit on a plugin ### Usage -* `phpunit [-m|--moodle MOODLE] [--coverage-text] [--coverage-clover] [--coverage-pcov] [--coverage-xdebug] [--coverage-phpdbg] [--fail-on-incomplete] [--fail-on-risky] [--fail-on-skipped] [--fail-on-warning] [--testdox] [--] ` +* `phpunit [-m|--moodle MOODLE] [-c|--configuration CONFIGURATION] [--testsuite TESTSUITE] [--filter FILTER] [--testdox] [--coverage-text] [--coverage-clover] [--coverage-pcov] [--coverage-xdebug] [--coverage-phpdbg] [--fail-on-incomplete] [--fail-on-risky] [--fail-on-skipped] [--fail-on-warning] [--] ` Run PHPUnit on a plugin @@ -1981,6 +2001,46 @@ Path to Moodle * Is negatable: no * Default: `'.'` +#### `--configuration|-c` + +PHPUnit configuration XML file (relative to plugin directory) + +* Accept value: yes +* Is value required: yes +* Is multiple: no +* Is negatable: no +* Default: `NULL` + +#### `--testsuite` + +PHPUnit testsuite option to use (must exist in the configuration file being used) + +* Accept value: yes +* Is value required: yes +* Is multiple: no +* Is negatable: no +* Default: `NULL` + +#### `--filter` + +PHPUnit filter option to use + +* Accept value: yes +* Is value required: yes +* Is multiple: no +* Is negatable: no +* Default: `NULL` + +#### `--testdox` + +Enable testdox formatter + +* Accept value: no +* Is value required: no +* Is multiple: no +* Is negatable: no +* Default: `false` + #### `--coverage-text` Generate and print code coverage report in text format @@ -2071,16 +2131,6 @@ Treat tests with warnings as failures * Is negatable: no * Default: `false` -#### `--testdox` - -Enable testdox formatter - -* Accept value: no -* Is value required: no -* Is multiple: no -* Is negatable: no -* Default: `false` - #### `--help|-h` Display help for the given command. When no command is given display help for the list command diff --git a/src/Command/BehatCommand.php b/src/Command/BehatCommand.php index 94171b36..aeeecd99 100644 --- a/src/Command/BehatCommand.php +++ b/src/Command/BehatCommand.php @@ -58,8 +58,11 @@ protected function configure(): void parent::configure(); $this->setName('behat') - ->addOption('profile', 'p', InputOption::VALUE_REQUIRED, 'Behat profile to use', 'default') - ->addOption('suite', null, InputOption::VALUE_REQUIRED, 'Behat suite to use (Moodle theme)', 'default') + ->addOption('profile', 'p', InputOption::VALUE_REQUIRED, 'Behat profile option to use', 'default') + ->addOption('suite', null, InputOption::VALUE_REQUIRED, 'Behat suite option to use (Moodle theme)', 'default') + ->addOption('tags', null, InputOption::VALUE_REQUIRED, 'Behat tags option to use. ' . + 'If not set, defaults to the component name', '') + ->addOption('name', null, InputOption::VALUE_REQUIRED, 'Behat name option to use', '') ->addOption('start-servers', null, InputOption::VALUE_NONE, 'Start Selenium and PHP servers') ->addOption('auto-rerun', null, InputOption::VALUE_REQUIRED, 'Number of times to rerun failures', 2) ->addOption('dump', null, InputOption::VALUE_NONE, 'Print contents of Behat failure HTML files') @@ -89,14 +92,19 @@ protected function execute(InputInterface $input, OutputInterface $output): int $cmd = [ 'php', 'admin/tool/behat/cli/run.php', - '--tags=@' . $this->plugin->getComponent(), '--profile=' . $input->getOption('profile'), '--suite=' . $input->getOption('suite'), + '--tags=' . ($input->getOption('tags') ?: '@' . $this->plugin->getComponent()), '--auto-rerun=' . $input->getOption('auto-rerun'), '--verbose', '-vvv', ]; + $name = $input->getOption('name'); + if (!empty($name) && is_string($name)) { + $cmd[] = '--name=\'' . $name . '\''; + } + if ($output->isDecorated()) { $cmd[] = '--colors'; } diff --git a/src/Command/PHPUnitCommand.php b/src/Command/PHPUnitCommand.php index 602bb91b..7e4728bb 100644 --- a/src/Command/PHPUnitCommand.php +++ b/src/Command/PHPUnitCommand.php @@ -29,6 +29,20 @@ protected function configure(): void $this->setName('phpunit') ->setDescription('Run PHPUnit on a plugin') + ->addOption( + 'configuration', + 'c', + InputOption::VALUE_REQUIRED, + 'PHPUnit configuration XML file (relative to plugin directory)' + ) + ->addOption( + 'testsuite', + null, + InputOption::VALUE_REQUIRED, + 'PHPUnit testsuite option to use (must exist in the configuration file being used)' + ) + ->addOption('filter', null, InputOption::VALUE_REQUIRED, 'PHPUnit filter option to use') + ->addOption('testdox', null, InputOption::VALUE_NONE, 'Enable testdox formatter') ->addOption('coverage-text', null, InputOption::VALUE_NONE, 'Generate and print code coverage report in text format') ->addOption('coverage-clover', null, InputOption::VALUE_NONE, 'Generate code coverage report in Clover XML format') ->addOption('coverage-pcov', null, InputOption::VALUE_NONE, 'Use the pcov extension to calculate code coverage') @@ -37,8 +51,7 @@ protected function configure(): void ->addOption('fail-on-incomplete', null, InputOption::VALUE_NONE, 'Treat incomplete tests as failures') ->addOption('fail-on-risky', null, InputOption::VALUE_NONE, 'Treat risky tests as failures') ->addOption('fail-on-skipped', null, InputOption::VALUE_NONE, 'Treat skipped tests as failures') - ->addOption('fail-on-warning', null, InputOption::VALUE_NONE, 'Treat tests with warnings as failures') - ->addOption('testdox', null, InputOption::VALUE_NONE, 'Enable testdox formatter'); + ->addOption('fail-on-warning', null, InputOption::VALUE_NONE, 'Treat tests with warnings as failures'); } protected function initialize(InputInterface $input, OutputInterface $output): void @@ -80,6 +93,28 @@ protected function execute(InputInterface $input, OutputInterface $output): int private function resolveOptions(InputInterface $input): array { $options = []; + + if ($input->getOption('configuration')) { + $options[] = [ + '--configuration', + $this->plugin->directory . '/' . $input->getOption('configuration'), + ]; + } + + if ($input->getOption('testsuite')) { + $options[] = [ + '--testsuite', + $input->getOption('testsuite'), + ]; + } + + if ($input->getOption('filter')) { + $options[] = [ + '--filter', + $input->getOption('filter'), + ]; + } + if ($this->supportsCoverage() && $input->getOption('coverage-text')) { $options[] = [ '--coverage-text', @@ -103,16 +138,25 @@ private function resolveOptions(InputInterface $input): array ]; } } - if (is_file($this->plugin->directory . '/phpunit.xml')) { - $options[] = [ - '--configuration', - $this->plugin->directory, - ]; - } else { - $options[] = [ - '--testsuite', - $this->plugin->getComponent(), - ]; + + // Only can set configuration or testsuite here (auto) if the former has not been set via command line option. + if (!$input->getOption('configuration')) { + // Use default configuration (phpunit.xml) only if it exists. + if (is_file($this->plugin->directory . '/phpunit.xml')) { + $options[] = [ + '--configuration', + $this->plugin->directory . '/phpunit.xml', + ]; + } else { + // Fallback to try to use the best testsuite potentially available. + // Only can set automatic testsuite if it has not been passed via command line option. + if (!$input->getOption('testsuite')) { + $options[] = [ + '--testsuite', + $this->plugin->getComponent() . '_testsuite', // This is our best guess. + ]; + } + } } return array_merge(...$options); // Merge all options into a single array. diff --git a/tests/Command/BehatCommandTest.php b/tests/Command/BehatCommandTest.php index a971d5ea..88751df4 100644 --- a/tests/Command/BehatCommandTest.php +++ b/tests/Command/BehatCommandTest.php @@ -21,7 +21,7 @@ class BehatCommandTest extends MoodleTestCase { - protected function executeCommand($pluginDir = null, $moodleDir = null): CommandTester + protected function executeCommand($pluginDir = null, $moodleDir = null, array $cmdOptions = []): CommandTester { if ($pluginDir === null) { $pluginDir = $this->pluginDir; @@ -38,10 +38,15 @@ protected function executeCommand($pluginDir = null, $moodleDir = null): Command $application->add($command); $commandTester = new CommandTester($application->find('behat')); - $commandTester->execute([ - 'plugin' => $pluginDir, - '--moodle' => $moodleDir, - ]); + $cmdOptions = array_merge( + [ + 'plugin' => $pluginDir, + '--moodle' => $moodleDir, + ], + $cmdOptions + ); + $commandTester->execute($cmdOptions); + $this->lastCmd = $command->execute->lastCmd; // We need this for assertions against the command run. return $commandTester; } @@ -50,6 +55,28 @@ public function testExecute() { $commandTester = $this->executeCommand(); $this->assertSame(0, $commandTester->getStatusCode()); + $this->assertMatchesRegularExpression('/php.*admin.tool.behat.cli.run/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--profile=default.*--suite=default/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--tags=@local_ci/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--verbose.*-vvv/', $this->lastCmd); + } + + public function testExecuteWithTags() + { + $commandTester = $this->executeCommand(null, null, ['--tags' => '@tag1&&@tag2']); + $this->assertSame(0, $commandTester->getStatusCode()); + $this->assertMatchesRegularExpression('/--tags=@tag1&&@tag2/', $this->lastCmd); + $this->assertDoesNotMatchRegularExpression('/--tags=@local_ci/', $this->lastCmd); + } + + public function testExecuteWithName() + { + $featureName = 'With "double quotes" and \'single quotes\''; + // Note that everything is escaped for shell execution, plus own regexp quoting. + $expectedName = preg_quote(escapeshellarg("--name='$featureName'")); + $commandTester = $this->executeCommand(null, null, ['--name' => $featureName]); + $this->assertSame(0, $commandTester->getStatusCode()); + $this->assertMatchesRegularExpression("/{$expectedName}/", $this->lastCmd); } public function testExecuteNoFeatures() @@ -70,6 +97,7 @@ public function testExecuteNoPlugin() public function testExecuteNoMoodle() { $this->expectException(\InvalidArgumentException::class); + // TODO: Check what's happening here. moodleDir should be the 2nd parameter, but then the test fails. $this->executeCommand($this->moodleDir . '/no/moodle'); } } diff --git a/tests/Command/PHPUnitCommandTest.php b/tests/Command/PHPUnitCommandTest.php index 9e550e22..959a3a34 100644 --- a/tests/Command/PHPUnitCommandTest.php +++ b/tests/Command/PHPUnitCommandTest.php @@ -21,7 +21,7 @@ class PHPUnitCommandTest extends MoodleTestCase { - protected function executeCommand($pluginDir = null, $moodleDir = null): CommandTester + protected function executeCommand($pluginDir = null, $moodleDir = null, array $cmdOptions = []): CommandTester { if ($pluginDir === null) { $pluginDir = $this->pluginDir; @@ -37,10 +37,15 @@ protected function executeCommand($pluginDir = null, $moodleDir = null): Command $application->add($command); $commandTester = new CommandTester($application->find('phpunit')); - $commandTester->execute([ - 'plugin' => $pluginDir, - '--moodle' => $moodleDir, - ]); + $cmdOptions = array_merge( + [ + 'plugin' => $pluginDir, + '--moodle' => $moodleDir, + ], + $cmdOptions + ); + $commandTester->execute($cmdOptions); + $this->lastCmd = $command->execute->lastCmd; // We need this for assertions against the command run. return $commandTester; } @@ -49,6 +54,50 @@ public function testExecute() { $commandTester = $this->executeCommand(); $this->assertSame(0, $commandTester->getStatusCode()); + $this->assertMatchesRegularExpression('/vendor.bin.phpunit/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--testsuite.*local_ci_testsuite/', $this->lastCmd); + $this->assertDoesNotMatchRegularExpression('/--configuration.*local\/ci/', $this->lastCmd); + } + + public function testExecuteWithCustomPHPUnitXMLFile() + { + $commandTester = $this->executeCommand(null, null, ['--configuration' => 'some_config.xml']); + $this->assertSame(0, $commandTester->getStatusCode()); + $this->assertMatchesRegularExpression('/vendor.bin.phpunit/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--configuration.*.*local\/ci\/some_config.xml/', $this->lastCmd); + $this->assertDoesNotMatchRegularExpression('/--configuration.*local\/ci\/phpunit.xml/', $this->lastCmd); + $this->assertDoesNotMatchRegularExpression('/--testsuite.*local_ci_testsuite/', $this->lastCmd); + } + + public function testExecuteWithGeneratedPHPUnitXMLFile() + { + $fs = new Filesystem(); + $fs->touch($this->pluginDir . '/phpunit.xml'); + $commandTester = $this->executeCommand(); + $this->assertSame(0, $commandTester->getStatusCode()); + $this->assertMatchesRegularExpression('/vendor.bin.phpunit/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--configuration.*local\/ci\/phpunit.xml/', $this->lastCmd); + $this->assertDoesNotMatchRegularExpression('/--testsuite.*local_ci_testsuite/', $this->lastCmd); + } + + public function testExecuteWithTestSuite() + { + $commandTester = $this->executeCommand(null, null, ['--testsuite' => 'some_testsuite']); + $this->assertSame(0, $commandTester->getStatusCode()); + $this->assertMatchesRegularExpression('/vendor.bin.phpunit/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--testsuite.*some_testsuite/', $this->lastCmd); + $this->assertDoesNotMatchRegularExpression('/--configuration.*local\/ci/', $this->lastCmd); + $this->assertDoesNotMatchRegularExpression('/--testsuite.*local_ci_testsuite/', $this->lastCmd); + } + + public function testExecuteWithFilter() + { + $commandTester = $this->executeCommand(null, null, ['--filter' => 'some_filter']); + $this->assertSame(0, $commandTester->getStatusCode()); + $this->assertMatchesRegularExpression('/vendor.bin.phpunit/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--filter.*some_filter/', $this->lastCmd); + $this->assertMatchesRegularExpression('/--testsuite.*local_ci_testsuite/', $this->lastCmd); + $this->assertDoesNotMatchRegularExpression('/--configuration.*local\/ci/', $this->lastCmd); } public function testExecuteNoTests() @@ -70,6 +119,7 @@ public function testExecuteNoPlugin() public function testExecuteNoMoodle() { $this->expectException(\InvalidArgumentException::class); + // TODO: Check what's happening here. moodleDir should be the 2nd parameter, but then the test fails. $this->executeCommand($this->moodleDir . '/no/moodle'); } } diff --git a/tests/Fake/Process/DummyExecute.php b/tests/Fake/Process/DummyExecute.php index 2353318a..1f9a3501 100644 --- a/tests/Fake/Process/DummyExecute.php +++ b/tests/Fake/Process/DummyExecute.php @@ -18,8 +18,9 @@ class DummyExecute extends Execute { public string $returnOutput = ''; + public string $lastCmd = ''; // We need this for assertions against the command run. - private function getMockProcess($cmd) + private function getMockProcess(string $cmd) { $process = \Mockery::mock('Symfony\Component\Process\Process'); $process->shouldReceive( @@ -43,22 +44,12 @@ private function getMockProcess($cmd) public function run($cmd, ?string $error = null): Process { - if ($cmd instanceof Process) { - // Get the command line from process. - $cmd = $cmd->getCommandLine(); - } - - return $this->helper->run($this->output, $this->getMockProcess($cmd), $error); + return $this->helper->run($this->output, $this->getMockProcess($this->getCommandLine($cmd)), $error); } public function mustRun($cmd, ?string $error = null): Process { - if ($cmd instanceof Process) { - // Get the command line from process. - $cmd = $cmd->getCommandLine(); - } - - return $this->helper->mustRun($this->output, $this->getMockProcess($cmd), $error); + return $this->helper->mustRun($this->output, $this->getMockProcess($this->getCommandLine($cmd)), $error); } public function runAll(array $processes): void @@ -73,15 +64,29 @@ public function mustRunAll(array $processes): void public function passThrough(array $commandline, ?string $cwd = null, ?float $timeout = null): Process { - return $this->passThroughProcess($this->getMockProcess($commandline)); + return $this->getMockProcess($this->getCommandLine($commandline)); } public function passThroughProcess(Process $process): Process { - if ($process instanceof \Mockery\MockInterface) { - return $process; + return $this->getMockProcess($this->getCommandLine($process)); + } + + /** + * Helper function to get the command line from a Process object or array. + * + * @param Process|array $cmd the command to run and its arguments listed as different entities + * + * @return string the command to run in a shell + */ + private function getCommandLine($cmd): string + { + if (is_array($cmd)) { + $this->lastCmd = (new Process($cmd))->getCommandLine(); + } else { + $this->lastCmd = $cmd->getCommandLine(); } - return $this->getMockProcess($process->getCommandLine()); + return $this->lastCmd; } } diff --git a/tests/MoodleTestCase.php b/tests/MoodleTestCase.php index 5275bef5..00e269e3 100644 --- a/tests/MoodleTestCase.php +++ b/tests/MoodleTestCase.php @@ -16,6 +16,7 @@ class MoodleTestCase extends FilesystemTestCase { protected string $moodleDir; protected string $pluginDir; + protected string $lastCmd = ''; // We need this for assertions against the command run. protected function setUp(): void { @@ -23,6 +24,7 @@ protected function setUp(): void $this->moodleDir = $this->tempDir; $this->pluginDir = $this->tempDir . '/local/ci'; + $this->lastCmd = ''; $this->fs->mirror(__DIR__ . '/Fixture/moodle', $this->moodleDir); $this->fs->mirror(__DIR__ . '/Fixture/moodle-local_ci', $this->pluginDir);