From cef3276521ba005fa30af46130dba2836347b1ae Mon Sep 17 00:00:00 2001 From: Riley-Kilgore Date: Mon, 16 Sep 2024 08:29:40 -0700 Subject: [PATCH] Add json flag for check command --- crates/aiken-lsp/src/server/lsp_project.rs | 1 + crates/aiken-project/src/lib.rs | 29 ++- crates/aiken-project/src/options.rs | 2 + crates/aiken-project/src/telemetry.rs | 233 ++++++++++++++++----- crates/aiken-project/src/watch.rs | 49 +++-- crates/aiken/src/cmd/blueprint/address.rs | 2 +- crates/aiken/src/cmd/blueprint/apply.rs | 2 +- crates/aiken/src/cmd/blueprint/hash.rs | 2 +- crates/aiken/src/cmd/blueprint/policy.rs | 2 +- crates/aiken/src/cmd/build.rs | 4 +- crates/aiken/src/cmd/check.rs | 9 +- crates/aiken/src/cmd/docs.rs | 2 +- crates/aiken/src/cmd/export.rs | 2 +- 13 files changed, 248 insertions(+), 91 deletions(-) diff --git a/crates/aiken-lsp/src/server/lsp_project.rs b/crates/aiken-lsp/src/server/lsp_project.rs index 9ab0367fe..7ba55adf4 100644 --- a/crates/aiken-lsp/src/server/lsp_project.rs +++ b/crates/aiken-lsp/src/server/lsp_project.rs @@ -38,6 +38,7 @@ impl LspProject { PropertyTest::DEFAULT_MAX_SUCCESS, Tracing::silent(), None, + false, ); self.project.restore(checkpoint); diff --git a/crates/aiken-project/src/lib.rs b/crates/aiken-project/src/lib.rs index f2fa4f87a..caa245b91 100644 --- a/crates/aiken-project/src/lib.rs +++ b/crates/aiken-project/src/lib.rs @@ -197,11 +197,13 @@ where uplc: bool, tracing: Tracing, env: Option, + json: bool, ) -> Result<(), Vec> { let options = Options { code_gen_mode: CodeGenMode::Build(uplc), tracing, env, + json, }; self.compile(options) @@ -225,7 +227,7 @@ where let mut modules = self.parse_sources(self.config.name.clone())?; - self.type_check(&mut modules, Tracing::silent(), None, false)?; + self.type_check(&mut modules, Tracing::silent(), None, false, false)?; let destination = destination.unwrap_or_else(|| self.root.join("docs")); @@ -267,6 +269,7 @@ where property_max_success: usize, tracing: Tracing, env: Option, + json: bool, ) -> Result<(), Vec> { let options = Options { tracing, @@ -282,6 +285,7 @@ where property_max_success, } }, + json: json, }; self.compile(options) @@ -343,6 +347,7 @@ where root: self.root.clone(), name: self.config.name.to_string(), version: self.config.version.clone(), + json: options.json, }); let env = options.env.as_deref(); @@ -353,7 +358,7 @@ where let mut modules = self.parse_sources(self.config.name.clone())?; - self.type_check(&mut modules, options.tracing, env, true)?; + self.type_check(&mut modules, options.tracing, env, true, options.json)?; match options.code_gen_mode { CodeGenMode::Build(uplc_dump) => { @@ -400,7 +405,8 @@ where self.collect_tests(verbose, match_tests, exact_match, options.tracing)?; if !tests.is_empty() { - self.event_listener.handle_event(Event::RunningTests); + self.event_listener + .handle_event(Event::RunningTests { json: options.json }); } let tests = self.run_tests(tests, seed, property_max_success); @@ -427,8 +433,11 @@ where }) .collect(); - self.event_listener - .handle_event(Event::FinishedTests { seed, tests }); + self.event_listener.handle_event(Event::FinishedTests { + seed, + tests, + json: options.json, + }); if !errors.is_empty() { Err(errors) @@ -629,7 +638,11 @@ where Ok(blueprint) } - fn with_dependencies(&mut self, parsed_packages: &mut ParsedModules) -> Result<(), Vec> { + fn with_dependencies( + &mut self, + parsed_packages: &mut ParsedModules, + json: bool, + ) -> Result<(), Vec> { let manifest = deps::download(&self.event_listener, &self.root, &self.config)?; for package in manifest.packages { @@ -640,6 +653,7 @@ where root: lib.clone(), name: package.name.to_string(), version: package.version.clone(), + json, }); self.read_package_source_files(&lib.join("lib"))?; @@ -828,10 +842,11 @@ where tracing: Tracing, env: Option<&str>, validate_module_name: bool, + json: bool, ) -> Result<(), Vec> { let our_modules: BTreeSet = modules.keys().cloned().collect(); - self.with_dependencies(modules)?; + self.with_dependencies(modules, json)?; for name in modules.sequence(&our_modules)? { if let Some(module) = modules.remove(&name) { diff --git a/crates/aiken-project/src/options.rs b/crates/aiken-project/src/options.rs index c15517063..e90d82802 100644 --- a/crates/aiken-project/src/options.rs +++ b/crates/aiken-project/src/options.rs @@ -4,6 +4,7 @@ pub struct Options { pub code_gen_mode: CodeGenMode, pub tracing: Tracing, pub env: Option, + pub json: bool, } impl Default for Options { @@ -12,6 +13,7 @@ impl Default for Options { code_gen_mode: CodeGenMode::NoOp, tracing: Tracing::silent(), env: None, + json: false, } } } diff --git a/crates/aiken-project/src/telemetry.rs b/crates/aiken-project/src/telemetry.rs index cf946294f..49a65bcbc 100644 --- a/crates/aiken-project/src/telemetry.rs +++ b/crates/aiken-project/src/telemetry.rs @@ -6,6 +6,7 @@ use aiken_lang::{ test_framework::{PropertyTestResult, TestResult, UnitTestResult}, }; use owo_colors::{OwoColorize, Stream::Stderr}; +use serde_json::json; use std::{collections::BTreeMap, fmt::Display, path::PathBuf}; use uplc::machine::cost_model::ExBudget; @@ -18,6 +19,7 @@ pub enum Event { name: String, version: String, root: PathBuf, + json: bool, }, BuildingDocumentation { name: String, @@ -37,10 +39,13 @@ pub enum Event { name: String, path: PathBuf, }, - RunningTests, + RunningTests { + json: bool, + }, FinishedTests { seed: u32, tests: Vec>, + json: bool, }, WaitingForBuildDirLock, ResolvingPackages { @@ -81,17 +86,20 @@ impl EventListener for Terminal { name, version, root, + json, } => { - eprintln!( - "{} {} {} ({})", - " Compiling" - .if_supports_color(Stderr, |s| s.bold()) - .if_supports_color(Stderr, |s| s.purple()), - name.if_supports_color(Stderr, |s| s.bold()), - version, - root.display() - .if_supports_color(Stderr, |s| s.bright_blue()) - ); + if !json { + eprintln!( + "{} {} {} ({})", + " Compiling" + .if_supports_color(Stderr, |s| s.bold()) + .if_supports_color(Stderr, |s| s.purple()), + name.if_supports_color(Stderr, |s| s.bold()), + version, + root.display() + .if_supports_color(Stderr, |s| s.bright_blue()) + ); + } } Event::BuildingDocumentation { name, @@ -166,53 +174,70 @@ impl EventListener for Terminal { name.if_supports_color(Stderr, |s| s.bright_blue()), ); } - Event::RunningTests => { - eprintln!( - "{} {}\n", - " Testing" - .if_supports_color(Stderr, |s| s.bold()) - .if_supports_color(Stderr, |s| s.purple()), - "...".if_supports_color(Stderr, |s| s.bold()) - ); + Event::RunningTests { json } => { + if !json { + eprintln!( + "{} {}\n", + " Testing" + .if_supports_color(Stderr, |s| s.bold()) + .if_supports_color(Stderr, |s| s.purple()), + "...".if_supports_color(Stderr, |s| s.bold()) + ); + } } - Event::FinishedTests { seed, tests } => { + Event::FinishedTests { seed, tests, json } => { let (max_mem, max_cpu, max_iter) = find_max_execution_units(&tests); - for (module, results) in &group_by_module(&tests) { - let title = module - .if_supports_color(Stderr, |s| s.bold()) - .if_supports_color(Stderr, |s| s.blue()) - .to_string(); + if json { + let json_output = serde_json::json!({ + "seed": seed, + "modules": group_by_module(&tests).iter().map(|(module, results)| { + serde_json::json!({ + "name": module, + "tests": results.iter().map(|r| fmt_test_json(r, max_mem, max_cpu, max_iter)).collect::>(), + "summary": fmt_test_summary_json(results) + }) + }).collect::>(), + "summary": fmt_overall_summary_json(&tests) + }); + println!("{}", serde_json::to_string_pretty(&json_output).unwrap()); + } else { + for (module, results) in &group_by_module(&tests) { + let title = module + .if_supports_color(Stderr, |s| s.bold()) + .if_supports_color(Stderr, |s| s.blue()) + .to_string(); - let tests = results - .iter() - .map(|r| fmt_test(r, max_mem, max_cpu, max_iter, true)) - .collect::>() - .join("\n"); - - let seed_info = if results - .iter() - .any(|t| matches!(t, TestResult::PropertyTestResult { .. })) - { - format!( - "with {opt}={seed} → ", - opt = "--seed".if_supports_color(Stderr, |s| s.bold()), - seed = format!("{seed}").if_supports_color(Stderr, |s| s.bold()) - ) - } else { - String::new() - }; - - let summary = format!("{}{}", seed_info, fmt_test_summary(results, true)); - println!( - "{}\n", - pretty::indent( - &pretty::open_box(&title, &tests, &summary, |border| border - .if_supports_color(Stderr, |s| s.bright_black()) - .to_string()), - 4 - ) - ); + let tests = results + .iter() + .map(|r| fmt_test(r, max_mem, max_cpu, max_iter, true)) + .collect::>() + .join("\n"); + + let seed_info = if results + .iter() + .any(|t| matches!(t, TestResult::PropertyTestResult { .. })) + { + format!( + "with {opt}={seed} → ", + opt = "--seed".if_supports_color(Stderr, |s| s.bold()), + seed = format!("{seed}").if_supports_color(Stderr, |s| s.bold()) + ) + } else { + String::new() + }; + + let summary = format!("{}{}", seed_info, fmt_test_summary(results, true)); + println!( + "{}\n", + pretty::indent( + &pretty::open_box(&title, &tests, &summary, |border| border + .if_supports_color(Stderr, |s| s.bright_black()) + .to_string()), + 4 + ) + ); + } } } Event::ResolvingPackages { name } => { @@ -488,7 +513,105 @@ fn fmt_test_summary(tests: &[&TestResult], styled: bool) -> String { ) } -fn group_by_module(results: &Vec>) -> BTreeMap>> { +fn fmt_test_json( + result: &TestResult, + max_mem: usize, + max_cpu: usize, + max_iter: usize, +) -> serde_json::Value { + let mut test = json!({ + "name": result.title(), + "status": if result.is_success() { "PASS" } else { "FAIL" }, + }); + + match result { + TestResult::UnitTestResult(UnitTestResult { + spent_budget, + assertion, + test: unit_test, + .. + }) => { + test["execution_units"] = json!({ + "memory": spent_budget.mem, + "cpu": spent_budget.cpu, + }); + if !result.is_success() { + if let Some(assertion) = assertion { + test["assertion"] = json!({ + "message": assertion.to_string(Stderr, false), + "expected_to_fail": matches!(unit_test.on_test_failure, OnTestFailure::SucceedEventually | OnTestFailure::SucceedImmediately), + }); + } + } + } + TestResult::PropertyTestResult(PropertyTestResult { + iterations, + labels, + counterexample, + .. + }) => { + test["iterations"] = json!(iterations); + test["labels"] = json!(labels); + test["counterexample"] = match counterexample { + Ok(Some(expr)) => json!(Formatter::new().expr(expr, false).to_pretty_string(60)), + Ok(None) => json!(null), + Err(err) => json!({"error": err.to_string()}), + }; + } + } + + if !result.traces().is_empty() { + test["traces"] = json!(result.traces()); + } + + test +} + +fn fmt_test_summary_json(tests: &[&TestResult]) -> serde_json::Value { + let total = tests.len(); + let passed = tests.iter().filter(|t| t.is_success()).count(); + let failed = total - passed; + + json!({ + "total": total, + "passed": passed, + "failed": failed, + }) +} + +fn fmt_overall_summary_json(tests: &[TestResult]) -> serde_json::Value { + let total = tests.len(); + let passed = tests.iter().filter(|t| t.is_success()).count(); + let failed = total - passed; + + let modules = group_by_module(tests); + let module_count = modules.len(); + + let (max_mem, max_cpu, max_iter) = find_max_execution_units(tests); + + json!({ + "total_tests": total, + "passed_tests": passed, + "failed_tests": failed, + "module_count": module_count, + "max_execution_units": { + "memory": max_mem, + "cpu": max_cpu, + }, + "max_iterations": max_iter, + "modules": modules.into_iter().map(|(module, results)| { + json!({ + "name": module, + "tests": results.iter().map(|r| fmt_test_json(r, max_mem, max_cpu, max_iter)).collect::>(), + "summary": fmt_test_summary_json(&results), + }) + }).collect::>(), + }) +} + +fn group_by_module( + results: &[TestResult], +) -> BTreeMap>> { let mut modules = BTreeMap::new(); for r in results { let xs: &mut Vec<&TestResult<_, _>> = modules.entry(r.module().to_string()).or_default(); diff --git a/crates/aiken-project/src/watch.rs b/crates/aiken-project/src/watch.rs index 91da47c4f..6810e3438 100644 --- a/crates/aiken-project/src/watch.rs +++ b/crates/aiken-project/src/watch.rs @@ -88,7 +88,12 @@ pub fn default_filter(evt: &Event) -> bool { } } -pub fn with_project(directory: Option<&Path>, deny: bool, mut action: A) -> miette::Result<()> +pub fn with_project( + directory: Option<&Path>, + deny: bool, + json: bool, + mut action: A, +) -> miette::Result<()> where A: FnMut(&mut Project) -> Result<(), Vec>, { @@ -116,36 +121,38 @@ where let warning_count = warnings.len(); - for warning in &warnings { - warning.report() - } + if !json { + for warning in &warnings { + warning.report() + } + + if let Err(errs) = build_result { + for err in &errs { + err.report() + } - if let Err(errs) = build_result { - for err in &errs { - err.report() + eprintln!( + "{}", + Summary { + check_count: project.checks_count, + warning_count, + error_count: errs.len(), + } + ); + + return Err(ExitFailure::into_report()); } eprintln!( "{}", Summary { check_count: project.checks_count, - warning_count, - error_count: errs.len(), + error_count: 0, + warning_count } ); - - return Err(ExitFailure::into_report()); } - eprintln!( - "{}", - Summary { - check_count: project.checks_count, - error_count: 0, - warning_count - } - ); - if warning_count > 0 && deny { Err(ExitFailure::into_report()) } else { @@ -239,7 +246,7 @@ where .if_supports_color(Stderr, |s| s.bold()) .if_supports_color(Stderr, |s| s.purple()), ); - with_project(directory, false, &mut action).unwrap_or(()) + with_project(directory, false, false, &mut action).unwrap_or(()) } } } diff --git a/crates/aiken/src/cmd/blueprint/address.rs b/crates/aiken/src/cmd/blueprint/address.rs index df54879e1..f29ead1df 100644 --- a/crates/aiken/src/cmd/blueprint/address.rs +++ b/crates/aiken/src/cmd/blueprint/address.rs @@ -33,7 +33,7 @@ pub fn exec( mainnet, }: Args, ) -> miette::Result<()> { - with_project(directory.as_deref(), false, |p| { + with_project(directory.as_deref(), false, false, |p| { let title = module.as_ref().map(|m| { format!( "{m}{}", diff --git a/crates/aiken/src/cmd/blueprint/apply.rs b/crates/aiken/src/cmd/blueprint/apply.rs index 247eb857f..4127cf719 100644 --- a/crates/aiken/src/cmd/blueprint/apply.rs +++ b/crates/aiken/src/cmd/blueprint/apply.rs @@ -47,7 +47,7 @@ pub fn exec( validator, }: Args, ) -> miette::Result<()> { - with_project(None, false, |p| { + with_project(None, false, false, |p| { let title = module.as_ref().map(|m| { format!( "{m}{}", diff --git a/crates/aiken/src/cmd/blueprint/hash.rs b/crates/aiken/src/cmd/blueprint/hash.rs index c73708900..f5410584b 100644 --- a/crates/aiken/src/cmd/blueprint/hash.rs +++ b/crates/aiken/src/cmd/blueprint/hash.rs @@ -23,7 +23,7 @@ pub fn exec( validator, }: Args, ) -> miette::Result<()> { - with_project(directory.as_deref(), false, |p| { + with_project(directory.as_deref(), false, false, |p| { let title = module.as_ref().map(|m| { format!( "{m}{}", diff --git a/crates/aiken/src/cmd/blueprint/policy.rs b/crates/aiken/src/cmd/blueprint/policy.rs index e23f118ca..080926d52 100644 --- a/crates/aiken/src/cmd/blueprint/policy.rs +++ b/crates/aiken/src/cmd/blueprint/policy.rs @@ -23,7 +23,7 @@ pub fn exec( validator, }: Args, ) -> miette::Result<()> { - with_project(directory.as_deref(), false, |p| { + with_project(directory.as_deref(), false, false, |p| { let title = module.as_ref().map(|m| { format!( "{m}{}", diff --git a/crates/aiken/src/cmd/build.rs b/crates/aiken/src/cmd/build.rs index fbe89fba2..8badfe768 100644 --- a/crates/aiken/src/cmd/build.rs +++ b/crates/aiken/src/cmd/build.rs @@ -79,10 +79,11 @@ pub fn exec( None => Tracing::All(trace_level), }, env.clone(), + false, ) }) } else { - with_project(directory.as_deref(), deny, |p| { + with_project(directory.as_deref(), deny, false, |p| { p.build( uplc, match filter_traces { @@ -90,6 +91,7 @@ pub fn exec( None => Tracing::All(trace_level), }, env.clone(), + false, ) }) }; diff --git a/crates/aiken/src/cmd/check.rs b/crates/aiken/src/cmd/check.rs index 864801832..ed5046db7 100644 --- a/crates/aiken/src/cmd/check.rs +++ b/crates/aiken/src/cmd/check.rs @@ -84,6 +84,10 @@ pub struct Args { /// [optional] #[clap(short, long, value_parser=trace_level_parser(), default_value_t=TraceLevel::Verbose, verbatim_doc_comment)] trace_level: TraceLevel, + + /// Output JSON (useful for scripting & automation) + #[clap(long)] + json: bool, } pub fn exec( @@ -100,6 +104,7 @@ pub fn exec( seed, max_success, env, + json, }: Args, ) -> miette::Result<()> { let mut rng = rand::thread_rng(); @@ -120,10 +125,11 @@ pub fn exec( None => Tracing::All(trace_level), }, env.clone(), + json, ) }) } else { - with_project(directory.as_deref(), deny, |p| { + with_project(directory.as_deref(), deny, json, |p| { p.check( skip_tests, match_tests.clone(), @@ -136,6 +142,7 @@ pub fn exec( None => Tracing::All(trace_level), }, env.clone(), + json, ) }) }; diff --git a/crates/aiken/src/cmd/docs.rs b/crates/aiken/src/cmd/docs.rs index fbc3ac182..b17c0c35a 100644 --- a/crates/aiken/src/cmd/docs.rs +++ b/crates/aiken/src/cmd/docs.rs @@ -38,7 +38,7 @@ pub fn exec( p.docs(destination.clone(), include_dependencies) }) } else { - with_project(directory.as_deref(), deny, |p| { + with_project(directory.as_deref(), deny, false, |p| { p.docs(destination.clone(), include_dependencies) }) }; diff --git a/crates/aiken/src/cmd/export.rs b/crates/aiken/src/cmd/export.rs index 6debdcd2b..9d18d4946 100644 --- a/crates/aiken/src/cmd/export.rs +++ b/crates/aiken/src/cmd/export.rs @@ -61,7 +61,7 @@ pub fn exec( trace_level, }: Args, ) -> miette::Result<()> { - with_project(directory.as_deref(), false, |p| { + with_project(directory.as_deref(), false, false, |p| { p.compile(Options::default())?; let export = p.export(