Skip to content

Commit

Permalink
Capture stderr of test via temp file and output it in test results [C…
Browse files Browse the repository at this point in the history
…loses #420] (#438)
  • Loading branch information
MartinMystikJonas authored and dg committed Aug 29, 2022
1 parent 0480187 commit f2ca728
Show file tree
Hide file tree
Showing 25 changed files with 195 additions and 53 deletions.
1 change: 1 addition & 0 deletions src/Runner/CliTester.php
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public function run(): ?int

if ($this->options['--info']) {
$job = new Job(new Test(__DIR__ . '/info.php'), $this->interpreter);
$job->setTempDirectory($this->options['--temp']);
$job->run();
echo $job->getTest()->stdout;
return null;
Expand Down
40 changes: 20 additions & 20 deletions src/Runner/Job.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,7 @@ class Job
/** waiting time between process activity check in microseconds */
public const RUN_USLEEP = 10000;

public const
RUN_ASYNC = 1,
RUN_COLLECT_ERRORS = 2;
public const RUN_ASYNC = 1;

/** @var Test */
private $test;
Expand All @@ -46,8 +44,8 @@ class Job
/** @var resource|null */
private $stdout;

/** @var resource|null */
private $stderr;
/** @var string|null */
private $stderrFile;

/** @var int */
private $exitCode = self::CODE_NONE;
Expand All @@ -74,6 +72,14 @@ public function __construct(Test $test, PhpInterpreter $interpreter, ?array $env
}


public function setTempDirectory(?string $path): void
{
$this->stderrFile = $path === null
? null
: $path . DIRECTORY_SEPARATOR . 'Job.pid-' . getmypid() . '.' . uniqid() . '.stderr';
}


public function setEnvironmentVariable(string $name, string $value): void
{
$this->envVars[$name] = $value;
Expand All @@ -88,7 +94,7 @@ public function getEnvironmentVariable(string $name): string

/**
* Runs single test.
* @param int $flags self::RUN_ASYNC | self::RUN_COLLECT_ERRORS
* @param int $flags self::RUN_ASYNC
*/
public function run(int $flags = 0): void
{
Expand All @@ -110,7 +116,7 @@ public function run(int $flags = 0): void
[
['pipe', 'r'],
['pipe', 'w'],
['pipe', 'w'],
$this->stderrFile ? ['file', $this->stderrFile, 'w'] : ['pipe', 'w'],
],
$pipes,
dirname($this->test->getFile()),
Expand All @@ -122,19 +128,15 @@ public function run(int $flags = 0): void
putenv($name);
}

[$stdin, $this->stdout, $stderr] = $pipes;
[$stdin, $this->stdout] = $pipes;
fclose($stdin);
if ($flags & self::RUN_COLLECT_ERRORS) {
$this->stderr = $stderr;
} else {
fclose($stderr);

if (isset($pipes[2])) {
fclose($pipes[2]);
}

if ($flags & self::RUN_ASYNC) {
stream_set_blocking($this->stdout, false); // on Windows does not work with proc_open()
if ($this->stderr) {
stream_set_blocking($this->stderr, false);
}
} else {
while ($this->isRunning()) {
usleep(self::RUN_USLEEP); // stream_select() doesn't work with proc_open()
Expand All @@ -153,9 +155,6 @@ public function isRunning(): bool
}

$this->test->stdout .= stream_get_contents($this->stdout);
if ($this->stderr) {
$this->test->stderr .= stream_get_contents($this->stderr);
}

$status = proc_get_status($this->proc);
if ($status['running']) {
Expand All @@ -165,8 +164,9 @@ public function isRunning(): bool
$this->duration += microtime(true);

fclose($this->stdout);
if ($this->stderr) {
fclose($this->stderr);
if ($this->stderrFile) {
$this->test->stderr .= file_get_contents($this->stderrFile);
unlink($this->stderrFile);
}

$code = proc_close($this->proc);
Expand Down
1 change: 1 addition & 0 deletions src/Runner/Output/ConsolePrinter.php
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ public function finish(Test $test): void

$title = ($test->title ? "$test->title | " : '') . substr($test->getSignature(), strlen($this->baseDir));
$message = ' ' . str_replace("\n", "\n ", trim((string) $test->message)) . "\n\n";
$message = preg_replace('/^ $/m', '', $message);
if ($test->getResult() === Test::FAILED) {
$this->buffer .= Dumper::color('red', "-- FAILED: $title") . "\n$message";
} elseif ($test->getResult() === Test::SKIPPED && $this->displaySkipped) {
Expand Down
9 changes: 9 additions & 0 deletions src/Runner/Test.php
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,15 @@ public function getDuration(): ?float
}


/**
* Full output (stdout + stderr)
*/
public function getOutput(): string
{
return $this->stdout . ($this->stderr ? "\nSTDERR:\n" . $this->stderr : '');
}


/**
* @return static
*/
Expand Down
9 changes: 6 additions & 3 deletions src/Runner/TestHandler.php
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,9 @@ public function initiate(string $file): void

foreach ($tests as $test) {
$this->runner->prepareTest($test);
$this->runner->addJob(new Job($test, $php, $this->runner->getEnvironmentVariables()));
$job = new Job($test, $php, $this->runner->getEnvironmentVariables());
$job->setTempDirectory($this->tempDir);
$this->runner->addJob($job);
}
}

Expand Down Expand Up @@ -196,10 +198,11 @@ private function initiateTestCase(Test $test, $foo, PhpInterpreter $interpreter)

if ($methods === null) {
$job = new Job($test->withArguments(['method' => TestCase::ListMethods]), $interpreter, $this->runner->getEnvironmentVariables());
$job->setTempDirectory($this->tempDir);
$job->run();

if (in_array($job->getExitCode(), [Job::CODE_ERROR, Job::CODE_FAIL, Job::CODE_SKIP], true)) {
return $test->withResult($job->getExitCode() === Job::CODE_SKIP ? Test::SKIPPED : Test::FAILED, $job->getTest()->stdout);
return $test->withResult($job->getExitCode() === Job::CODE_SKIP ? Test::SKIPPED : Test::FAILED, $job->getTest()->getOutput());
}

$stdout = $job->getTest()->stdout;
Expand Down Expand Up @@ -245,7 +248,7 @@ private function assessExitCode(Job $job, $code): ?Test
$message = $job->getExitCode() !== Job::CODE_FAIL
? "Exited with error code {$job->getExitCode()} (expected $code)"
: '';
return $job->getTest()->withResult(Test::FAILED, trim($message . "\n" . $job->getTest()->stdout));
return $job->getTest()->withResult(Test::FAILED, trim($message . "\n" . $job->getTest()->getOutput()));
}

return null;
Expand Down
3 changes: 2 additions & 1 deletion tests/Runner/Job.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ require __DIR__ . '/../bootstrap.php';
test(function () {
$test = (new Test('Job.test.phptx'))->withArguments(['one', 'two' => 1])->withArguments(['three', 'two' => 2]);
$job = new Job($test, createInterpreter());
$job->run($job::RUN_COLLECT_ERRORS);
$job->setTempDirectory(Tester\Helpers::prepareTempDir(sys_get_temp_dir()));
$job->run();

Assert::false($job->isRunning());
Assert::same($test, $job->getTest());
Expand Down
5 changes: 3 additions & 2 deletions tests/Runner/Runner.multiple-fails.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ $interpreter = createInterpreter()
$runner = new Runner($interpreter);
$runner->paths[] = __DIR__ . '/multiple-fails/*.phptx';
$runner->outputHandlers[] = $logger = new Logger;
$runner->setTempDirectory(Tester\Helpers::prepareTempDir(sys_get_temp_dir()));
$runner->run();

Assert::match(
Expand Down Expand Up @@ -86,8 +87,8 @@ Assert::same(Test::FAILED, $logger->results['testcase-pre-fail.phptx'][0]);

Assert::match(
defined('PHPDBG_VERSION')
? '%A%Parse error: %a% in %a%testcase-syntax-error.phptx on line %d%'
: 'Parse error: %a% in %a%testcase-syntax-error.phptx on line %d%',
? '%A%Parse error: %a% in %a%testcase-syntax-error.phptx on line %d%%A?%'
: 'Parse error: %a% in %a%testcase-syntax-error.phptx on line %d%%A?%',
trim($logger->results['testcase-syntax-error.phptx'][1])
);
Assert::same(Test::FAILED, $logger->results['testcase-syntax-error.phptx'][0]);
Expand Down
1 change: 1 addition & 0 deletions tests/Runner/Test.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ test(function () {
Assert::null($test->message);
Assert::same('', $test->stdout);
Assert::same('', $test->stderr);
Assert::same('', $test->getOutput());
Assert::same('some/Test.phpt', $test->getFile());
Assert::same([], $test->getArguments());
Assert::same('some/Test.phpt', $test->getSignature());
Expand Down
28 changes: 24 additions & 4 deletions tests/RunnerOutput/OutputHandlers.expect.console.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,24 @@ F.sF.sFsF.s
line
stdout.Failed:

in %a%01-basic.fail.phptx(7) Tester\Assert::fail('');
in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail('');

STDERR:
Multi
line
stderr.

-- FAILED: Title for output handlers | 02-title.fail.phptx
Multi
line
stdout.Failed:

in %a%02-title.fail.phptx(11) Tester\Assert::fail('');
in %a%02-title.fail.phptx(%d%) Tester\Assert::fail('');

STDERR:
Multi
line
stderr.

-- FAILED: 03-message.fail.phptx
Multi
Expand All @@ -23,14 +33,24 @@ F.sF.sFsF.s
line
message.

in %a%03-message.fail.phptx(7) Tester\Assert::fail("Multi\nline\nmessage.");
in %a%03-message.fail.phptx(%d%) Tester\Assert::fail("Multi\nline\nmessage.");

STDERR:
Multi
line
stderr.

-- FAILED: 04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
Multi
line
stdout.Failed:

in %a%04-args.fail.phptx(11) Tester\Assert::fail('');
in %a%04-args.fail.phptx(%d%) Tester\Assert::fail('');

STDERR:
Multi
line
stderr.


FAILURES! (11 tests, 4 failures, 4 skipped, %a% seconds)
42 changes: 31 additions & 11 deletions tests/RunnerOutput/OutputHandlers.expect.consoleWithSkip.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,43 +7,63 @@ F.sF.sFsF.s
line
stdout.Failed:

in %a%01-basic.fail.phptx(7) Tester\Assert::fail('');
in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail('');

-- Skipped: %a?%01-basic.skip.phptx
STDERR:
Multi
line
stderr.

-- Skipped: 01-basic.skip.phptx


-- FAILED: Title for output handlers | 02-title.fail.phptx
-- FAILED: Title for output handlers | %a?%02-title.fail.phptx
Multi
line
stdout.Failed:

in %a%02-title.fail.phptx(11) Tester\Assert::fail('');
in %a%02-title.fail.phptx(%d%) Tester\Assert::fail('');

STDERR:
Multi
line
stderr.

-- Skipped: Title for output handlers | 02-title.skip.phptx
-- Skipped: Title for output handlers | %a?%02-title.skip.phptx


-- FAILED: 03-message.fail.phptx
-- FAILED: %a?%03-message.fail.phptx
Multi
line
stdout.Failed: Multi
line
message.

in %a%03-message.fail.phptx(7) Tester\Assert::fail("Multi\nline\nmessage.");
in %a%03-message.fail.phptx(%d%) Tester\Assert::fail("Multi\nline\nmessage.");

-- Skipped: 03-message.skip.phptx
STDERR:
Multi
line
stderr.

-- Skipped: %a?%03-message.skip.phptx
Multi
line
message.

-- FAILED: 04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
-- FAILED: %a?%04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
Multi
line
stdout.Failed:

in %a%04-args.fail.phptx(11) Tester\Assert::fail('');
in %a%04-args.fail.phptx(%d%) Tester\Assert::fail('');

STDERR:
Multi
line
stderr.

-- Skipped: 04-args.skip.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
-- Skipped: %a?%04-args.skip.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
Multi
line
message.
Expand Down
28 changes: 24 additions & 4 deletions tests/RunnerOutput/OutputHandlers.expect.jUnit.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@
line
stdout.Failed:
in %a%01-basic.fail.phptx(7) Tester\Assert::fail('');"/>
in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail('');
STDERR:
Multi
line
stderr."/>
</testcase>
<testcase classname="%a%01-basic.pass.phptx" name="%a%01-basic.pass.phptx"/>
<testcase classname="%a%01-basic.skip.phptx" name="%a%01-basic.skip.phptx">
Expand All @@ -17,7 +22,12 @@ in %a%01-basic.fail.phptx(7) Tester\Assert::fail('');"/>
line
stdout.Failed:
in %a%02-title.fail.phptx(11) Tester\Assert::fail('');"/>
in %a%02-title.fail.phptx(%d%) Tester\Assert::fail('');
STDERR:
Multi
line
stderr."/>
</testcase>
<testcase classname="%a%02-title.pass.phptx" name="%a%02-title.pass.phptx"/>
<testcase classname="%a%02-title.skip.phptx" name="%a%02-title.skip.phptx">
Expand All @@ -30,7 +40,12 @@ stdout.Failed: Multi
line
message.
in %a%03-message.fail.phptx(7) Tester\Assert::fail(&quot;Multi\nline\nmessage.&quot;);"/>
in %a%03-message.fail.phptx(%d%) Tester\Assert::fail(&quot;Multi\nline\nmessage.&quot;);
STDERR:
Multi
line
stderr."/>
</testcase>
<testcase classname="%a%03-message.skip.phptx" name="%a%03-message.skip.phptx">
<skipped/>
Expand All @@ -40,7 +55,12 @@ in %a%03-message.fail.phptx(7) Tester\Assert::fail(&quot;Multi\nline\nmessage.&q
line
stdout.Failed:
in %a%04-args.fail.phptx(11) Tester\Assert::fail('');"/>
in %a%04-args.fail.phptx(%d%) Tester\Assert::fail('');
STDERR:
Multi
line
stderr."/>
</testcase>
<testcase classname="%a%04-args.pass.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini" name="%a%04-args.pass.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini"/>
<testcase classname="%a%04-args.skip.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini" name="%a%04-args.skip.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini">
Expand Down
Loading

0 comments on commit f2ca728

Please sign in to comment.