Skip to content

Commit

Permalink
Capture output on tests and benches
Browse files Browse the repository at this point in the history
  • Loading branch information
@udoprog
udoprog committed Dec 1, 2021
1 parent bd1d09f commit 14e6ec4
Show file tree
Hide file tree
Showing 8 changed files with 208 additions and 60 deletions.
70 changes: 62 additions & 8 deletions crates/rune-cli/src/benches.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ use rune::compile::{Item, Meta};
use rune::runtime::{Function, Unit, Value};
use rune::termcolor::StandardStream;
use rune::{Any, Context, ContextError, Hash, Module, Sources};
use rune_modules::capture_io::CaptureIo;
use std::fmt;
use std::io::Write;
use std::sync::Arc;
use std::time::Instant;
Expand Down Expand Up @@ -46,6 +48,7 @@ pub(crate) async fn run(
o: &mut StandardStream,
args: &Flags,
context: &Context,
io: Option<&CaptureIo>,
unit: Arc<Unit>,
sources: &Sources,
fns: &[(Hash, Meta)],
Expand All @@ -58,18 +61,35 @@ pub(crate) async fn run(
let mut any_error = false;

for (hash, meta) in fns {
let item = &meta.item.item;
let mut bencher = Bencher::default();

if let Err(error) = vm.call(*hash, (&mut bencher,)) {
writeln!(o, "Error in benchmark `{}`", meta.item.item)?;
writeln!(o, "{}: Error in benchmark", item)?;
error.emit(o, sources)?;
any_error = true;

if let Some(io) = io {
writeln!(o, "-- output --")?;
io.drain_into(&mut *o)?;
writeln!(o, "-- end output --")?;
}

continue;
}

let multiple = bencher.fns.len() > 1;

for (i, f) in bencher.fns.iter().enumerate() {
if let Err(e) = bench_fn(o, i, &meta.item.item, args, f) {
writeln!(o, "Error running benchmark iteration: {}", e)?;
if let Err(e) = bench_fn(o, i, item, args, f, multiple) {
writeln!(o, "{}: Error in bench iteration: {}", item, e)?;

if let Some(io) = io {
writeln!(o, "-- output --")?;
io.drain_into(&mut *o)?;
writeln!(o, "-- end output --")?;
}

any_error = true;
}
}
Expand All @@ -88,6 +108,7 @@ fn bench_fn(
item: &Item,
args: &Flags,
f: &Function,
multiple: bool,
) -> anyhow::Result<()> {
for _ in 0..args.warmup {
let value = f.call::<_, Value>(())?;
Expand Down Expand Up @@ -117,10 +138,43 @@ fn bench_fn(
/ len;
let stddev = variance.sqrt();

writeln!(
o,
"bench {}#{}: mean={:.2}ns, stddev={:.2}, iterations={}",
item, i, average, stddev, args.iterations,
)?;
let format = Format {
average: average as u128,
stddev: stddev as u128,
iterations,
};

if multiple {
writeln!(o, "bench {}#{}: {}", item, i, format)?;
} else {
writeln!(o, "bench {}: {}", item, format)?;
}

Ok(())
}

struct Format {
average: u128,
stddev: u128,
iterations: usize,
}

impl fmt::Display for Format {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"mean={:.2}, stddev={:.2}, iterations={}",
Time(self.average),
Time(self.stddev),
self.iterations
)
}
}

struct Time(u128);

impl fmt::Display for Time {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}ns", self.0)
}
}
27 changes: 25 additions & 2 deletions crates/rune-cli/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ use anyhow::Result;
use rune::compile::ParseOptionError;
use rune::termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
use rune::{Context, ContextError, Options};
use rune_modules::capture_io::CaptureIo;
use std::error::Error;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
Expand Down Expand Up @@ -151,6 +152,23 @@ impl SharedFlags {

Ok(context)
}

/// Setup a context that captures output.
fn context_with_capture(&self, io: &CaptureIo) -> Result<Context, ContextError> {
let mut context = rune_modules::with_config(false)?;

context.install(&rune_modules::capture_io::module(io)?)?;

if self.experimental {
context.install(&rune_modules::experiments::module(true)?)?;
}

if self.test {
context.install(&benches::test_module()?)?;
}

Ok(context)
}
}

#[derive(Debug, Clone, StructOpt)]
Expand Down Expand Up @@ -180,6 +198,7 @@ impl Args {
// Command-specific override defaults.
match &self.cmd {
Command::Test(_) | Command::Check(_) => {
options.debug_info(true);
options.test(true);
options.bytecode(false);
}
Expand Down Expand Up @@ -362,29 +381,33 @@ async fn run_path(
match &args.cmd {
Command::Check(flags) => check::run(o, flags, options, path),
Command::Test(flags) => {
let context = flags.shared.context()?;
let io = rune_modules::capture_io::CaptureIo::new();
let context = flags.shared.context_with_capture(&io)?;

let load = loader::load(o, &context, args, options, path, visitor::Attribute::Test)?;

tests::run(
o,
flags,
&context,
Some(&io),
load.unit,
&load.sources,
&load.functions,
)
.await
}
Command::Bench(flags) => {
let context = flags.shared.context()?;
let io = rune_modules::capture_io::CaptureIo::new();
let context = flags.shared.context_with_capture(&io)?;

let load = loader::load(o, &context, args, options, path, visitor::Attribute::Bench)?;

benches::run(
o,
flags,
&context,
Some(&io),
load.unit,
&load.sources,
&load.functions,
Expand Down
116 changes: 69 additions & 47 deletions crates/rune-cli/src/tests.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
use crate::{ExitCode, SharedFlags};
use anyhow::Result;
use rune::compile::Meta;
use rune::runtime::{Unit, Value, Vm, VmError};
use rune::termcolor::StandardStream;
use rune::{Context, Hash, Sources};
use rune_modules::capture_io::CaptureIo;
use std::io::Write;
use std::sync::Arc;
use std::time::Instant;
Expand All @@ -25,15 +27,16 @@ pub(crate) struct Flags {
#[derive(Debug)]
enum FailureReason {
Crash(VmError),
ReturnedNone,
ReturnedErr(Value),
ReturnedNone { output: Box<[u8]> },
ReturnedErr { output: Box<[u8]>, error: Value },
}

#[derive(Debug)]
struct TestCase<'a> {
hash: Hash,
meta: &'a Meta,
outcome: Option<FailureReason>,
buf: Vec<u8>,
}

impl<'a> TestCase<'a> {
Expand All @@ -42,88 +45,106 @@ impl<'a> TestCase<'a> {
hash,
meta,
outcome: None,
buf: Vec::new(),
}
}

fn start(&self, o: &mut StandardStream, quiet: bool) -> Result<(), std::io::Error> {
if quiet {
return Ok(());
async fn execute(
&mut self,
o: &mut StandardStream,
vm: &mut Vm,
quiet: bool,
io: Option<&CaptureIo>,
) -> Result<bool> {
if !quiet {
write!(o, "Test {:30} ", self.meta.item.item)?;
}

write!(o, "Test {:30} ", self.meta.item.item)
}

async fn execute(&mut self, vm: &mut Vm) -> Result<bool, VmError> {
let result = match vm.execute(self.hash, ()) {
Err(err) => Err(err),
Ok(mut execution) => execution.async_complete().await,
};

if let Some(io) = io {
let _ = io.drain_into(&mut self.buf);
}

self.outcome = match result {
Err(e) => Some(FailureReason::Crash(e)),
Ok(v) => match v {
Value::Result(result) => match result.take()? {
Ok(..) => None,
Err(err) => Some(FailureReason::ReturnedErr(err)),
Err(error) => Some(FailureReason::ReturnedErr {
output: self.buf.as_slice().into(),
error,
}),
},
Value::Option(option) => match *option.borrow_ref()? {
Some(..) => None,
None => Some(FailureReason::ReturnedNone),
None => Some(FailureReason::ReturnedNone {
output: self.buf.as_slice().into(),
}),
},
_ => None,
},
};

Ok(self.outcome.is_none())
}

fn end(&self, o: &mut StandardStream, quiet: bool) -> Result<(), std::io::Error> {
if quiet {
match &self.outcome {
Some(FailureReason::Crash(_)) => {
write!(o, "F")
Some(FailureReason::Crash { .. }) => {
write!(o, "F")?;
}
Some(FailureReason::ReturnedErr(_)) => {
write!(o, "f")
Some(FailureReason::ReturnedErr { .. }) => {
write!(o, "f")?;
}
Some(FailureReason::ReturnedNone) => {
write!(o, "n")
Some(FailureReason::ReturnedNone { .. }) => {
write!(o, "n")?;
}
None => {
write!(o, ".")?;
}
None => write!(o, "."),
}
} else {
match &self.outcome {
Some(FailureReason::Crash(_)) => {
writeln!(o, "failed")
Some(FailureReason::Crash { .. }) => {
writeln!(o, "failed")?;
}
Some(FailureReason::ReturnedErr { .. }) => {
writeln!(o, "returned error")?;
}
Some(FailureReason::ReturnedErr(_)) => {
writeln!(o, "returned error")
Some(FailureReason::ReturnedNone { .. }) => {
writeln!(o, "returned none")?;
}
Some(FailureReason::ReturnedNone) => {
writeln!(o, "returned none")
None => {
writeln!(o, "passed")?;
}
None => writeln!(o, "passed"),
}
}

self.buf.clear();
Ok(self.outcome.is_none())
}

fn emit(&self, o: &mut StandardStream, sources: &Sources) -> Result<(), std::io::Error> {
if self.outcome.is_none() {
return Ok(());
}
match self.outcome.as_ref().unwrap() {
FailureReason::Crash(err) => {
writeln!(o, "----------------------------------------")?;
writeln!(o, "Test: {}\n", self.meta.item.item)?;
err.emit(o, sources).expect("failed writing diagnostics");
}
FailureReason::ReturnedNone => {}
FailureReason::ReturnedErr(e) => {
writeln!(o, "----------------------------------------")?;
writeln!(o, "Test: {}\n", self.meta.item.item)?;
writeln!(o, "Error: {:?}\n", e)?;
fn emit(&self, o: &mut StandardStream, sources: &Sources) -> Result<()> {
if let Some(outcome) = &self.outcome {
match outcome {
FailureReason::Crash(err) => {
writeln!(o, "----------------------------------------")?;
writeln!(o, "Test: {}\n", self.meta.item.item)?;
err.emit(o, sources)?;
}
FailureReason::ReturnedNone { .. } => {}
FailureReason::ReturnedErr { output, error, .. } => {
writeln!(o, "----------------------------------------")?;
writeln!(o, "Test: {}\n", self.meta.item.item)?;
writeln!(o, "Error: {:?}\n", error)?;
writeln!(o, "-- output --")?;
o.write_all(output)?;
writeln!(o, "-- end of output --")?;
}
}
}

Ok(())
}
}
Expand All @@ -132,6 +153,7 @@ pub(crate) async fn run(
o: &mut StandardStream,
flags: &Flags,
context: &Context,
io: Option<&CaptureIo>,
unit: Arc<Unit>,
sources: &Sources,
fns: &[(Hash, Meta)],
Expand All @@ -154,11 +176,11 @@ pub(crate) async fn run(
for test in &mut cases {
executed_count += 1;

test.start(o, flags.quiet)?;
let success = test.execute(&mut vm).await?;
test.end(o, flags.quiet)?;
let success = test.execute(o, &mut vm, flags.quiet, io).await?;

if !success {
failure_count += 1;

if !flags.no_fail_fast {
break;
}
Expand Down
Loading

0 comments on commit 14e6ec4

Please sign in to comment.