Generate static coverage reports through the firefly command line (#290)

* feat: Tried to parse artifacts on the client side

* feat: Added generate-report as kiele cmd arg

* feat: Tried to build the static report HTML template for Iele

* feat: Added static-report.html

* ci: Added test of generating report

* fix: Use python3 for kiele-generate-report.py script

* refactor: Added -o|--output to generate-report

.gitignore

@@ -8,3 +8,4 @@
 /.kprove*
 /.krun*
 result*
+.mypy_cache

Dockerfile

@@ -26,6 +26,7 @@ RUN    apt update         \
         pkg-config        \
         protobuf-compiler \
         python3           \
+        python3-pip       \
         zlib1g-dev
 
 RUN curl -sSL https://get.haskellstack.org/ | sh
@@ -35,6 +36,8 @@ RUN    apt-get update               \
     && apt-get upgrade --yes        \
     && apt-get install --yes nodejs
 
+RUN pip3 install dataclasses dacite
+
 ARG USER_ID=1000
 ARG GROUP_ID=1000
 RUN    groupadd --gid $GROUP_ID user                                        \

Jenkinsfile

@@ -38,6 +38,7 @@ pipeline {
             stage('Interactive')           { steps { sh 'make test-interactive'             } }
             stage('Node')                  { steps { sh 'make test-node -j4 TEST_PORT=9001' } }
             stage('Sourcemappings')        { steps { sh 'make test-sourcemap'               } }
+            stage('Report Generation')     { steps { sh 'make test-generate-report'         } }
           }
         }
       }

Makefile

@@ -62,7 +62,7 @@ export PATH:=$(IELE_BIN):$(PATH)
 .PHONY: all clean distclean libff protobuf coverage secp256k1 cryptopp \
         build build-interpreter build-vm build-check build-haskell build-node build-testnode build-assembler \
 		install install-interpreter install-vm install-kiele install-check uninstall \
-        split-tests split-vm-tests split-blockchain-tests test-node test-iele-coverage \
+        split-tests split-vm-tests split-blockchain-tests test-node test-iele-coverage test-generate-report \
         test-evm test-vm test-blockchain test-wellformed test-illformed test-bad-packet test-interactive test-sourcemap \
         test-iele test-iele-haskell test-iele-failing test-iele-slow test-iele-node assemble-iele-test test
 .SECONDARY:
@@ -153,7 +153,7 @@ TEST_PORT     = 10000
 TEST_ARGS     = --no-unparse
 TEST_DIR      = $(IELE_DIR)/tests
 
-test: split-tests test-vm test-iele test-iele-haskell test-iele-coverage test-wellformed test-illformed test-interactive test-node test-sourcemap
+test: split-tests test-vm test-iele test-iele-haskell test-iele-coverage test-wellformed test-illformed test-interactive test-node test-sourcemap test-generate-report
 
 split-tests: split-vm-tests split-blockchain-tests
 
@@ -252,6 +252,11 @@ test-node: TEST_PORT=9001
 test-node:
 	$(TEST_DIR)/node-test.sh $(MAKEFLAGS) --port $(TEST_PORT)
 
+report_tests := $(wildcard $(TEST_DIR)/reports/*/report.json)
+test-generate-report: $(report_tests:.json=.html)
+$(TEST_DIR)/reports/%/report.html: $(TEST_DIR)/reports/%/report.json
+	cd $(dir $@) && kiele generate-report report.json -o report.html
+
 $(TEST_DIR)/VMTests/%:  TEST_MODE     = VMTESTS
 %.iele.test-wellformed: TEST_SCHEDULE = DANSE
 %.iele.test-illformed:  TEST_SCHEDULE = DANSE
@@ -435,6 +440,8 @@ install_libs :=              \
     $(iele_interpreter_libs) \
     $(iele_haskell_libs)     \
     kore-json.py             \
+    kiele-generate-report.py \
+    static-report.html       \
     version
 
 $(IELE_RUNNER): $(IELE_DIR)/kiele
@@ -449,10 +456,20 @@ $(IELE_LIB)/kore-json.py: $(IELE_DIR)/kore-json.py
 	@mkdir -p $(dir $@)
 	$(INSTALL) $< $@
 
+$(IELE_LIB)/kiele-generate-report.py: $(IELE_DIR)/kiele-generate-report.py
+	@mkdir -p $(dir $@)
+	$(INSTALL) $< $@
+
+$(IELE_LIB)/static-report.html: $(IELE_DIR)/static-report.html
+	@mkdir -p $(dir $@)
+	$(INSTALL) $< $@
+
 $(INSTALL_BIN)/iele-interpreter: $(patsubst %, $(INSTALL_LIB)/%, $(iele_interpreter_libs))
 $(INSTALL_BIN)/iele-check:       $(patsubst %, $(INSTALL_LIB)/%, $(iele_check_libs))
 
 $(INSTALL_BIN)/kiele: $(INSTALL_LIB)/kore-json.py
+$(INSTALL_BIN)/kiele: $(INSTALL_LIB)/kiele-generate-report.py
+$(INSTALL_BIN)/kiele: $(INSTALL_LIB)/static-report.html
 $(INSTALL_BIN)/kiele: $(INSTALL_LIB)/version
 
 $(INSTALL_BIN)/%: $(IELE_BIN)/%

kiele

@@ -99,6 +99,24 @@ run_vm() {
     "${cmd[@]}"
 }
 
+run_generate_report() {
+    if [[ -f $(dirname $(which kiele))/static-report.html ]]; then
+        report_template=$(dirname $(which kiele))/static-report.html
+    else
+        report_template=$INSTALL_LIB/static-report.html
+    fi
+    output_path=""
+    while [[ $# -gt 0 ]]; do
+        arg="$1"
+        case $arg in
+            -o|--output) output_path="$2" ; shift 2 ;;
+            *) shift ;;
+        esac
+    done 
+    echo "Generating report"
+    echo $(kiele-generate-report.py $report_template $run_file $output_path) generated
+}
+
 # Main
 # ----
 
@@ -111,14 +129,16 @@ if [[ "$run_command" == 'help' ]] || [[ "$run_command" == '--help' ]] ; then
                $0 krun         [--backend (standalone|haskell|node|check)] <KIELE arg>* <pgm> <K arg>*
                $0 check                                                    <KIELE arg>* <pgm> <K arg>*
                $0 vm                                                       <KIELE arg>*
+               $0 generate-report                                          <Report json path> [-o|--output <Report output path>]
 
                $0 [help|--help|version|--version]
 
-           $0 assemble  : Translate IELE assembly <pgm to IELE bytecode
-           $0 interpret : Run given input JSON or IELE <pgm>
-           $0 krun      : Run given input <pgm> using 'krun'
-           $0 check     : Run the IELE well-formedness checker on <pgm>
-           $0 vm        : Run the KIELE VM server.
+           $0 assemble         : Translate IELE assembly <pgm to IELE bytecode
+           $0 interpret        : Run given input JSON or IELE <pgm>
+           $0 krun             : Run given input <pgm> using 'krun'
+           $0 check            : Run the IELE well-formedness checker on <pgm>
+           $0 vm               : Run the KIELE VM server.
+           $0 generate-report  : Generate the static HTML report
 
            $0 help    : Display this help message.
            $0 version : Display the versions of KIELE in use.
@@ -129,6 +149,8 @@ if [[ "$run_command" == 'help' ]] || [[ "$run_command" == '--help' ]] ; then
                  <K arg> is an argument you want to pass to K
                  <KIELE arg> is one of (--no-unparse|--debug|--host|--port|--mode|--coverage
                                        |--schedule|--module|--definition|--depth)
+                <Report json path> is the path to the report json file
+                <Report output path> is the output HTML report file path (default: \$uuid.html)
     "
     exit 0
 fi
@@ -187,10 +209,11 @@ if [[ "$run_command" != 'vm' ]]; then
 fi
 
 case "$run_command" in
-    assemble)  run_assemble  "$@" ;;
-    interpret) run_interpret "$@" ;;
-    krun)      run_krun      "$@" ;;
-    check)     run_check     "$@" ;;
-    vm)        run_vm        "$@" ;;
+    assemble)          run_assemble  "$@" ;;
+    interpret)         run_interpret "$@" ;;
+    krun)              run_krun      "$@" ;;
+    check)             run_check     "$@" ;;
+    vm)                run_vm        "$@" ;;
+    generate-report)   run_generate_report "$@";;
     *) $0 help ; fatal "Unknown command: $run_command" ;;
 esac