Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test framework #2019

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 92 additions & 87 deletions spec/ysh-stdlib-testing.test.sh
Original file line number Diff line number Diff line change
@@ -1,125 +1,130 @@
## our_shell: ysh
## oils_failures_allowed: 5

#### value.Expr test - positional test
#### Test framework

source --builtin testing.ysh
setglobal _test_use_color = false

echo 'parens'
test-expr (42 + 1)
echo

echo 'brackets'
test-expr [42 + 1]
echo

echo 'expr in parens'
test-expr (^[42 + 1])
echo

## STDOUT:
## END

#### value.Expr test - named test

source --builtin testing.ysh

echo 'parens'
test-named (n=42 + 1)
echo

echo 'brackets'
test-named [n=42 + 1]
echo

echo 'expr in parens'
test-named (n=^[42 + 1])
echo

echo 'no value'
test-named
echo

## STDOUT:
## END

#### assert builtin

source --builtin testing.ysh # get rid of this line later?

var x = 42

# how do you get the code string here?

assert [42 === x]
test-suite "three bad tests" {
test-case "the assertion is false" {
assert [1 > 0]
assert [1 > 1]
assert [1 > 2]
}

assert [42 < x]
test-case "there is an exception while evaluating the assertion" {
assert ["Superman" > "Batman"]
}

#assert [42 < x; fail_message='message']
test-case "there is an exception outside of any assertion" {
error "oops!"
}
}

#assert (^[(42 < x)], fail_message='passed message')
test-case "one good test case" {
assert [1 === 1]
assert [2 === 2]
}

# BUG
assert [42 < x, fail_message='passed message']
run-tests

## STDOUT:
begin three bad tests
test the assertion is false ...
assertion FAILED
test there is an exception while evaluating the assertion ...
assertion ERRORED: 10
test there is an exception outside of any assertion ...
ERROR: 10
end
test one good test case ... ok
3 / 4 tests failed
## END

#### ysh --tool test file
#### Test framework: nested test suites

cat >mytest.ysh <<EOF
echo hi
EOF

# which ysh
source --builtin testing.ysh
setglobal _test_use_color = false

# the test framework sets $SH to bin/ysh
# but ysh is already installed on this machine
test-case "A" { : }
test-suite "outer test suite" {
test-case "B" { : }
test-suite "first inner test suite" {
test-case "C" { : }
}
test-case "D" { : }
test-suite "second inner test suite" {
test-case "E" { : }
}
test-case "F" { : }
}
test-case "G" { : }

$SH --tool test mytest.ysh
run-tests

## STDOUT:
test A ... ok
begin outer test suite
test B ... ok
begin first inner test suite
test C ... ok
end
test D ... ok
begin second inner test suite
test E ... ok
end
test F ... ok
end
test G ... ok
7 tests succeeded
## END

# Hm can we do this entirely in user code, not as a builtin?

#### Describe Prototype
#### Test framework: stdout and stderr

source --builtin testing.ysh
setglobal _test_use_color = false

proc p {
echo STDOUT
echo STDERR >& 2
return 42
}

describe p {
test-case "that it prints to stdout and stderr" {
# each case changes to a clean directory?
#
# and each one is numbered?

it 'prints to stdout and stderr' {
try {
p > out 2>& err
}
assert (_status === 42)

cat out
cat err

# Oh man the here docs are still useful here because of 'diff' interface
# Multiline strings don't quite do it

diff out - <<< '''
STDOUT
'''

diff err - <<< '''
STDERR
'''
try {
p > out 2> err
}

assert [_status === 42]
assert [$(<out) === "STDOUT"]
assert [$(<err) === "STDERR"]
}

run-tests

## STDOUT:
TODO
test that it prints to stdout and stderr ... ok
1 tests succeeded
## END

# #### ysh --tool test file
#
# cat >mytest.ysh <<EOF
# echo hi
# EOF
#
# # which ysh
#
# # the test framework sets $SH to bin/ysh
# # but ysh is already installed on this machine
#
# $SH --tool test mytest.ysh
#
# ## STDOUT:
# ## END
#
# # Hm can we do this entirely in user code, not as a builtin?
269 changes: 178 additions & 91 deletions stdlib/testing.ysh
Original file line number Diff line number Diff line change
@@ -1,115 +1,202 @@
# testing.ysh
#
# Usage:
# source --builtin testing.sh
#
# func f(x) { return (x + 1) }
#
# describe foo {
# assert (43 === f(42))
# }
#
# if is-main {
# run-tests @ARGV # --filter
# }

module stdlib/testing || return 0

source --builtin args.ysh

proc assert ( ; cond ; fail_message='default fail message') {
echo 'hi from assert'

= cond

# I think this might be ready now?

var val = evalExpr(cond)

echo
echo 'value'
= val
pp line (val)

= fail_message

if (val) {
echo 'OK'
} else {
var m = evalExpr(fail_message)
echo "FAIL - this is where we extract the string - $m"
}
# TODO:
# - [x] Renaming a bit for "privacy"
# - [x] Renaming for public interface: test-case, test-suite, assert (subject to change)
# - [x] Rename to use "-"s for procs.
# - [x] Make test-case and test-suite take a word arg so they don't need parens
# - [x] error if you try to re-enter run-tests
# - [x] testing testing: assert fail, exn in assert, exn outside of assert
# - [-] check for both kinds of exceptions (_status and _error)
# - [x] tests for tests
# - [ ] question: naming for variables?
# - [ ] --tool test support (& spec test case for it)
# - [ ] Turn the assert expression into a string, to print if it fails
# - [ ] Use ctx instead of globals, wherever possible
# - [ ] Merge _status and _error. Every exception should set _error, with the
# status stored on _error.status. Use _error instead of the icky global
# _assertion_result

##########
# Colors #
##########

const _BOLD = $'\e[1m'
const _RED = $'\e[31m'
const _GREEN = $'\e[32m'
const _YELLOW = $'\e[33m'
const _PURPLE = $'\e[35m'
const _CYAN = $'\e[36m'
const _RESET = $'\e[0;0m'

func _red(text) {
if (_test_use_color) {
return ("${_BOLD}${_RED}${text}${_RESET}")
} else {
return (text)
}
}

proc test-assert {
var x = 42
assert [42 === x]
func _yellow(text) {
if (_test_use_color) {
return ("${_YELLOW}${text}${_RESET}")
} else {
return (text)
}
}

proc test-expr ( ; expr ) {
echo 'expr'
pp line (expr)
func _green(text) {
if (_test_use_color) {
return ("${_YELLOW}${text}${_RESET}")
} else {
return (text)
}
}

proc test-named ( ; ; n=^[99] ) {
echo 'n'
pp line (n)
func _cyan(text) {
if (_test_use_color) {
return ("${_YELLOW}${text}${_RESET}")
} else {
return (text)
}
}

# What happens when there are duplicate test IDs?
#
# Also I think filter by "$test_id/$case_id"
############
# Internal #
############

proc __it (case_id ; ; ; block) {
# This uses a clean directory
echo TODO
}

# is this accessible to users?
# It can contain a global list of things to run
var _testing_in_progress = false
var _test_suite_depth = 0
var _test_suite_stack = []
var _tests = {}

# Naming convention: a proc named 'describe' mutates a global named _describe?
# Or maybe _describe_list ?
var _num_test_fail = 0
var _num_test_succ = 0
var _test_use_color = true

var _describe_list = []
var _assertion_result = 0 # 0: no assert, 1: failed, 2: errored

proc describe (test_id ; ; ; block) {
echo describe
#= desc
proc _start-test-suite (; name) {
call _test_suite_stack->append(name)
}

# TODO:
# - need append
# - need ::
# _ _describe->append(cmd)
#
# Need to clean this up
# append (_describe, cmd) # does NOT work!
proc _end-test-suite {
call _test_suite_stack->pop()
}

call _describe_list->append(block)
proc _test-print-indented (msg) {
for _ in (0 .. _test_suite_depth) {
printf " "
}
echo "$msg"
}

proc Args {
echo TODO
proc _run-test-suite (; suite) {
for name, elem in (suite) {
if (type (elem) === "Dict") {
# It's another suite.
var begin = _cyan("begin")
_test-print-indented "$begin $name"
setglobal _test_suite_depth += 1;
_run-test-suite (elem)
setglobal _test_suite_depth -= 1;
var end = _cyan("end")
_test-print-indented "$end"
} else {
# It's a test case.
_run-test-case (name, elem)
}
}
}

# Problem: this creates a global variable?
Args (&spec) {
flag --filter 'Regex of test descriptions'
proc _run-test-case (; name, block) {
var test = _yellow("test")
for _ in (0 .. _test_suite_depth) {
printf " "
}
printf "$test $name ..."
setglobal _assertion_result = 0
try {
eval (block)
}
if (_status === 0) {
setglobal _num_test_succ += 1
var ok = _green(" ok")
printf "$ok"
printf '\n'
} else {
setglobal _num_test_fail += 1
printf '\n'
if (_assertion_result === 1) {
# An assertion failed.
var fail = _red("assertion FAILED")
_test-print-indented " $fail"
} elif (_assertion_result === 2) {
# There was an exception while evaluating an assertion.
var exn_in_assert = _red("assertion ERRORED:")
_test-print-indented " $exn_in_assert $_status"
} else {
# There was an exception in the test case outside of any `assert`.
var exn = _red("ERROR:")
_test-print-indented " $exn $_status"
}
}
}

proc run-tests {
var opt, i = parseArgs(spec, ARGV)
##########
# Public #
##########

# TODO:
# - parse --filter foo, which you can use eggex for!
proc test-suite (name ; ; ; block) {
_start-test-suite (name)
eval (block)
_end-test-suite
}

for cmd in (_describe) {
# TODO: print filename and 'describe' name?
proc test-case (name ; ; ; block) {
var test_suite = _tests
for suite_name in (_test_suite_stack) {
if (not (suite_name in test_suite)) {
setvar test_suite[suite_name] = {}
}
setvar test_suite = test_suite[suite_name]
}
setvar test_suite[name] = block
}

proc assert ( ; cond LAZY ) {
var success
try {
eval (cmd)
setvar success = evalExpr(cond)
}
if (_status !== 0) {
echo 'failed'
setglobal _assertion_result = 2
error "exception in assertion"
} elif (not success) {
setglobal _assertion_result = 1
error "assertion failed"
}
}

proc run-tests {
if (_testing_in_progress) {
error "Cannot run tests while testing is already in progress"
}
setglobal _testing_in_progress = true
setglobal _num_test_fail = 0
setglobal _num_test_succ = 0

_run-test-suite (_tests)
setglobal _tests = {}
setglobal _testing_in_progress = false

var total = _num_test_fail + _num_test_succ
if (total === 0) {
var na = _yellow("0 tests ran")
printf "$na"
printf '\n'
} elif (_num_test_fail === 0) {
var success = _green("$total tests succeeded")
printf "$success"
printf '\n'
} else {
var failure = _red("$_num_test_fail / $total tests failed")
printf "$failure"
printf '\n'
}
}
}