runt.toml

ver = "0.4.0"

##### Compilation Tests ######
## Test each pass in isolation. Gets the pass flags from a comment on the first line of the file
[[tests]]
name = "[core] passes"
paths = ["tests/passes/**/*.futil"]
# gets the pass flags a comment on the first line of the test file
cmd = """
flags="$(head -n 1 {} | cut -c 3-) -m file"
./target/debug/calyx {} $flags -l . \
  | sed 's/extern \".*\\(calyx\\/.*\\)\"/extern \"<ROOT>\\/\\1\"/'
"""

[[tests]]
name = "[core] parsing"
# Round-tripping from the compiler should not change anything.
paths = ["./tests/parsing/*.expect"]
cmd = """
./target/debug/calyx {} -m file -p none -l .
"""

[[tests]]
name = "[core] import"
# Round-tripping from the compiler should not change anything.
paths = ["./tests/import/a.futil"]
cmd = """
./target/debug/calyx {} -p none -l . \
  | sed 's/extern \".*\\(calyx\\/.*\\)\"/extern \"<ROOT>\\/\\1\"/'
"""

## Tests the error messages generated by the compiler. Runs passes for error
## checking.
[[tests]]
name = "[core] errors"
paths = [
  "tests/errors/*.futil",
  "tests/errors/papercut/*.futil",
  "tests/errors/parser/*.futil",
]
cmd = """
./target/debug/calyx {} -p well-formed -p papercut -p synthesis-papercut -l .
"""

[[tests]]
name = "[core] futil examples"
paths = ["examples/futil/*.futil"]
cmd = """
./target/debug/calyx {} -m file -d tdst -l .
"""

##### Backend Tests #####
[[tests]]
name = "[core] backends"
paths = ["tests/backend/**/*.futil"]
cmd = """
flags=$(head -n 1 {} | cut -c 3-)
./target/debug/calyx {} $flags -l .
"""

##### Frontend Tests #####
[[tests]]
name = "[frontend] dahlia"
paths = ["tests/frontend/dahlia/*.fuse"]
cmd = """
fud e {} --to calyx -q \
          -s calyx.exec './target/debug/calyx' \
        | tail -n+2
"""

[[tests]]
name = "[frontend] systolic array"
paths = ["tests/frontend/systolic/array-*.systolic"]
cmd = """
./frontends/systolic-lang/gen-systolic.py {}
"""

[[tests]]
name = "[frontend] NTT pipeline generation"
paths = ["tests/frontend/ntt-pipeline/*.txt"]
cmd = """
fud e --from ntt {} --to calyx -q
"""

[[tests]]
name = "[frontend] relay"
paths = ["tests/frontend/relay/*.relay"]
cmd = "python3 frontends/relay/relay_visitor.py {}"

[[tests]]
name = "[frontend] exp"
paths = ["tests/frontend/exp/*.txt"]
cmd = "python3 calyx-py/calyx/gen_exp.py {}"

[[tests]]
name = "[calyx-py] builder"
paths = ["calyx-py/test/*.py"]
cmd = """
python3 {}
"""

[[tests]]
name = "[calyx-py] correctness"
paths = ["calyx-py/test/correctness/*.py"]
cmd = """
name=$(basename {} .py) &&
dir=$(dirname {}) &&
python3 {} |\
 fud e --from calyx --to jq \
       --through verilog \
       --through dat \
       -s verilog.data "$dir/$name.data" \
       -s jq.expr ".memories" \
       -q
"""


##### Correctness Tests #####
## Tests that ensure that individual control
## constructs have correct behavior when compiled.
[[tests]]
name = "correctness dynamic"
paths = [
  "tests/correctness/*.futil",
  "tests/correctness/ref-cells/*.futil",
  "tests/correctness/sync/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s verilog.data {}.data \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags ' -d tdst' \
         -s verilog.cycle_limit 500 \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

[[tests]]
name = "correctness static timing"
paths = ["tests/correctness/*.futil", "tests/correctness/ref-cells/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

# Variants of the above but for `--nested` (non-ANF Verilog emission) mode.
[[tests]]
name = "correctness nested"
paths = ["tests/correctness/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s verilog.data {}.data \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags ' --nested' \
         -s verilog.cycle_limit 500 \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

# Tests to ensure that static compilation maintains its guarantees
# See: https://github.com/cucapra/calyx/discussions/1331
[[tests]]
name = "correctness static scheduling"
paths = ["tests/correctness/static/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags ' -x tdst:force -p all -d group2invoke' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         {} -q
"""
timeout = 120

# Tests to ensure that static compilation maintains its guarantees
# See: https://github.com/cucapra/calyx/discussions/1331 & https://github.com/cucapra/calyx/issues/1344
[[tests]]
name = "correctness static control"
paths = [
  "tests/correctness/static-control/*.futil",
  "tests/correctness/group-static-promotion/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags '-p all -d tdst -d group2invoke' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         {} -q
"""
timeout = 120

[[tests]]
name = "numeric types correctness and parsing"
paths = [
  "tests/correctness/numeric-types/parsing/*.futil",
  "tests/correctness/numeric-types/bitnum/*.futil",
  "tests/correctness/numeric-types/fixed-point/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through dat \
         --through verilog \
         -s verilog.data {}.data \
         -s jq.expr ".memories" \
         {} -q
"""

[[tests]]
name = "correctness test of static islands without static promotion"
paths = ["tests/correctness/static-islands/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through dat \
         --through verilog \
         -s verilog.data {}.data \
         -s calyx.flags "-d static-promotion" \
         -s jq.expr ".memories" \
         {} -q
"""

[[tests]]
name = "[frontend] tcam testing"
paths = ["tests/correctness/tcam/*.futil"]
cmd = """
fud exec --from calyx --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {}.data \
      -s jq.expr ".memories" \
      {} -q
"""

[[tests]]
name = "[frontend] systolic array PE correctness"
paths = ["tests/correctness/systolic/pe/*.systolic"]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through vcd_json \
      -s verilog.data {}.data \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.file {}.jq \
      {} -q
"""

[[tests]]
name = "[frontend] systolic array mmult static correctness"
paths = [
  "tests/correctness/systolic/mmult-inputs/*.data"
]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {} \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.expr ".memories" \
      {}_static.systolic -q
"""
expect_dir="tests/correctness/systolic/mmult-expect"

[[tests]]
name = "[frontend] systolic array mmult dynamic correctness"
paths = [
  "tests/correctness/systolic/mmult-inputs/*.data"
]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {} \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.expr ".memories" \
      {}_dynamic.systolic -q
"""
expect_dir="tests/correctness/systolic/mmult-expect"

[[tests]]
name = "[frontend] systolic array relu static correctness"
paths = [
  "tests/correctness/systolic/relu-inputs/*.data"
]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {} \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.expr ".memories" \
      {}_static.systolic -q
"""
expect_dir="tests/correctness/systolic/relu-expect"

[[tests]]
name = "[frontend] systolic array relu dynamic correctness"
paths = [
  "tests/correctness/systolic/relu-inputs/*.data"
]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {} \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.expr ".memories" \
      {}_dynamic.systolic -q
"""
expect_dir="tests/correctness/systolic/relu-expect"

[[tests]]
name = "[frontend] systolic array leaky relu correctness"
paths = [
  "tests/correctness/systolic/leaky-relu/*.systolic",
]
cmd = """
fud e --from systolic --to dat \
      --through verilog \
      -s verilog.data {}.data \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      {} -q
"""

[[tests]]
name = "[frontend] NTT pipeline correctness"
paths = ["tests/correctness/ntt-pipeline/*.txt"]
cmd = """
fud e --from ntt {} --through verilog --to dat -s verilog.data {}.data -q
"""

[[tests]]
name = "exp correctness"
paths = ["tests/correctness/exp/*.txt"]
cmd = """
python3 calyx-py/calyx/gen_exp.py {} |\
fud e --from calyx --to dat --through verilog -s verilog.data {}.data -q
"""

[[tests]]
name = "[frontend] mrxl correctness"
paths = ["frontends/mrxl/test/*.mrxl"]
expect_dir = "frontends/mrxl/test/expect/simulate_out/"
cmd = """
fud e --from mrxl {} \
      --to jq \
      --through verilog \
      --through dat \
      -s mrxl.data {}.data \
      -s jq.expr ".memories" \
      -q
"""

[[tests]]
name = "[frontend] relay correctness"
paths = ["tests/correctness/relay/*.relay"]
cmd = """
fud e -q {} --from relay --to jq \
            --through verilog \
            --through dat \
            -s verilog.data {}.data \
            -s jq.expr ".memories"
"""

## Tests errors that occur at runtime
[[tests]]
name = "runtime errors"
paths = ["tests/errors/runtime/*.futil"]
cmd = """
fud exec -s calyx.exec './target/debug/calyx' \
         -s verilog.cycle_limit 500 \
         --through verilog \
         {} --from calyx --to dat -q 2>&1 | \
         grep '%Error' | sed 's/%Error: [^:]*:[^:]*:/%Error:/'
"""

##### Examples #####

[[tests]]
name = "[examples] language tutorial"
paths = ["examples/tutorial/*.futil"]
cmd = """
fud e {} -s verilog.cycle_limit 500 \
         -s calyx.flags ' -d tdst' \
         --through verilog \
         -s verilog.data examples/tutorial/data.json \
         --from calyx --to dat -q
"""

[[tests]]
name = "[examples] sync primitives"
paths = ["examples/sync/*.futil"]
cmd = """
fud e {} -s verilog.cycle_limit 500 \
         -s calyx.flags ' -d tdst' \
         --through verilog \
         -s verilog.data {}.data \
         --from calyx --to dat -q
"""

[[tests]]
name = "[examples] memory by reference tutorial"
paths = ["examples/futil/memory-by-reference/*.futil"]
cmd = """
fud e {} -s verilog.cycle_limit 500 \
         -s calyx.flags ' -d tdst' \
         --through verilog \
         -s verilog.data {}.data \
         --from calyx --to dat -q
"""

[[tests]]
name = "[examples] dahlia examples"
paths = ["examples/dahlia/*.fuse"]
cmd = """
fud e {} -s verilog.cycle_limit 500 \
         --through verilog \
         -s verilog.data {}.data \
         --to dat -q
"""

##### Xilinx Tests ######
[[tests]]
name = "AXI wrapper generation"
paths = ["tests/xilinx/*.futil"]
expect_dir = "tests/xilinx/gen-verilog"
cmd = """
target/debug/calyx {} -b xilinx
"""

[[tests]]
name = "kernel.xml generation"
paths = ["tests/xilinx/*.futil"]
expect_dir = "tests/xilinx/gen-xml"
cmd = """
target/debug/calyx {} -b xilinx-xml
"""

[[tests]]
name = "Cocotb correctness tests"
paths = [
  "tests/xilinx/cocotb/dot-product/",
  "tests/xilinx/cocotb/vectorized-add/",
]
cmd = """
mkdir -p {}/hdl && \
dir_name=$(basename {}) && \
fud e -q --from calyx {}/${dir_name}.futil --to synth-verilog -o {}/hdl/main.sv &&\
  fud e -q --from calyx {}/${dir_name}.futil --to axi-wrapper -o {}/hdl/toplevel.v

cd {}/../ && \
make --silent -B TEST_PATH={} COCOTB_LOG_LEVEL=CRITICAL |\
    grep -E '^Output:' |\
    sed 's/Output://g' &&\
  make clean && \
  rm -f results.xml

"""