Cpu (#43)

* Get cpu baseline files in

* Update make taco-bench to be correct for gtest

* Add in cpu benchmarking functions ready for test on kiwi

* Add in CMakeLists.txt for taco-bench

* Forgot to add in main.cpp file for taco-bench

* Fix result initialization for baseline

* Fix result initialization for baseline suitesparse

* Fix numactl for kiwi and type in baseline.cpp

* Update to use 'GEN' as cmake build option

* Fix integer casting

* Fix some formats

* Use proper name when writing other tensor

* Fix datasetName to use tensorName

* Bump sam taco pointer to pull in commits from taco upstream

* Add in fixes for suitesparse cpu benchmarks

* Update makefile and cmake for taco-bench

* Add in fix for reading in other vector

* Add in some validation stuff

* add in diff checker

* Add in validation script

* Add in fixes for new GEN env variable

* Add in code changes for matrix generation for TTM and MTTKRP

* Fix tensor formats

* Add in validation script between sam and taco

* Enable mattransmul

* Fix code style

* Add in some debuggin statements

* Add in some debuggin statements

* Remove print statements and update dimensions for otherMat for mttkrp

* Add in code to get MTTKRP working

* Change default sparsity to lower for nell-1

* Fix validator script and some merge issues

* Add in suitesparse-bench/ and results-cpu/ to .gitignore

* Update permissions for checker file

* Update aggregator to work for new googlebench

* Remove .mtx from suitesparse list

* Update taco submodule and use a different transpose approach

* Add in some fixes for memory leaks with suitesparse spmm

* Remove print statements and fix some patching issues

* Change files to work for Lanka

* Add in changes to fix segfaults

* Fix some sspath stuff in suitesparse_runner.sh

* resume timing after validation output - otherwise dies in debug mode?

* add resumetiming to the frostt loop

Co-authored-by: Olivia W Hsu <[email protected]>
Co-authored-by: Maxwell Strange <[email protected]>
Co-authored-by: Maxwell Strange <[email protected]>

.gitignore

@@ -14,6 +14,10 @@ __pycache__/
 build/
 logs/
 results/
+results-cpu/
+suitesparse-bench/
 sam-outputs/
 compiler/benchmark/
-cmake-build-debug/
+cmake-build-*/
+
+*.out

.gitmodules

@@ -2,3 +2,6 @@
 	path = compiler/taco
 	url = https://github.com/weiya711/taco/
 	branch = sam
+[submodule "compiler/benchmark"]
+	path = compiler/benchmark
+	url = https://github.com/google/benchmark.git

Makefile

@@ -1,34 +1,110 @@
-ifeq ($(NEVA),ON)
+ifeq ($(BENCHES),)
+BENCHES=""
+endif
+
+# Set OPENMP=ON if compiling TACO with OpenMP support.
+ifeq ($(OPENMP),)
+OPENMP := "OFF"
+endif
+# Set NEVA=ON if compiling on the Stanford cluster (Neva or Kiwi).
+ifeq ($(NEVA),)
+NEVA := "OFF"
+endif
+# Set LANKA=ON if compiling on the MIT cluster (Lanka).
+ifeq ($(LANKA),)
+LANKA := "OFF"
+endif
+# Set GEN=ON if you would like to generate "other" tensors for performance into a file
+ifeq ($(GEN),)
+GEN := "OFF"
+endif
+
+benches_name := $(patsubst %.py,%,$(BENCHES))
+benches_name := $(subst /,_,$(benches_name))
+benches_name := $(subst *,_,$(benches_name))
+# Taco Specific Flags
+ifeq ($(TACO_OUT),)
+TACO_OUT := results-cpu/$(benches_name)_benches_$(shell date +%Y_%m_%d_%H%M%S).csv
+endif
+
+ifeq ("$(NEVA)","ON")
+	CMD := OMP_PROC_BIND=true LD_LIBRARY_PATH=compiler/build/lib/:$(LD_LIBRARY_PATH) numactl -C 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 -m 0 compiler/build/taco-bench $(BENCHFLAGS)
 	export SUITESPARSE_PATH=/nobackup/owhsu/sparse-datasets/suitesparse/
 	export FROSTT_PATH=/nobackup/owhsu/sparse-datasets/frostt/
 	export SUITESPARSE_FORMATTED_PATH=/nobackup/owhsu/sparse-datasets/suitesparse-formatted
 	export FROSTT_FORMATTED_TACO_PATH=/nobackup/owhsu/sparse-datasets/frostt-formatted/taco-tensor
 	export FROSTT_FORMATTED_PATH=/nobackup/owhsu/sparse-datasets/frostt-formatted
-endif 
+	export TACO_TENSOR_PATH=/nobackup/owhsu/sparse-datasets
+else ifeq ("$(LANKA)", "ON")
+	CMD := OMP_PROC_BIND=true LD_LIBRARY_PATH=compiler/build/lib/:$(LD_LIBRARY_PATH) numactl -C 0,2,4,6,8,10,24,26,28,30,32,34 -m 0 compiler/build/taco-bench $(BENCHFLAGS)
+	export SUITESPARSE_PATH=/data/scratch/changwan/florida_all
+	export FROSTT_PATH=/data/scratch/owhsu/datasets/frostt
+	export TACO_TENSOR_PATH=/data/scratch/owhsu/datasets
+else
+	CMD := LD_LIBRARY_PATH=compiler/build/lib/:$(LD_LIBRARY_PATH) compiler/build/taco-bench $(BENCHFLAGS)
+endif
 
 guard-%:
 	@ if [ "${${*}}" = "" ]; then \
         	echo "Environment variable $* not set"; \
         	exit 1; \
 	fi
 
+# ---- Run SAM python simulator stuff ----
 csv: 
 	scripts/pytest_suitesparse_with_benchmarks.sh
 
+run: submodules
+	./scripts/pytest_suitesparse.sh
+
 tests: sam 
 	python scripts/test_generating_code.py
 	make run
 
-run: submodules
-	./scripts/pytest_suitesparse.sh
+# ---- Build taco and make sam graphs ----
+.PHONY: sam
+sam: taco/build 
+	 cd compiler && bash -xe ./sam-kernels.sh
+
+taco/build: submodules
+	 mkdir -p compiler/taco/build 
+	 cd compiler/taco/build && cmake -DCMAKE_BUILD_TYPE=Release .. && make -j8
+
+# ---- Build and run taco-bench (cpu baseline) ----
+taco-bench: compiler/build/taco-bench guard-TACO_TENSOR_PATH
+	export GEN=$(GEN)
+	mkdir -p $(TACO_TENSOR_PATH)/other
+	mkdir -p results-cpu 
+ifeq ($(BENCHES),"")
+	$(CMD) --benchmark_out_format="csv" --benchmark_out="$(TACO_OUT)" --benchmark_repetitions=10 --benchmark_counters_tabular=true
+else
+	$(CMD) --benchmark_filter="$(BENCHES)" --benchmark_out_format="csv" --benchmark_out="$(TACO_OUT)" --benchmark_repetitions=10 --benchmark_counters_tabular=true
+endif
+
+compiler/build/taco-bench: submodules compiler/benchmark/googletest
+	mkdir -p compiler/build/ && cd compiler/build/ && cmake -DOPENMP=$(OPENMP) -DNEVA=$(NEVA) ../ && $(MAKE) taco-bench
+
+compiler/benchmark/googletest: submodules
+	if [ ! -d "compiler/benchmark/googletest" ] ; then git clone https://github.com/google/googletest compiler/benchmark/googletest; fi
 
-.PHONY: suitesparse-formats
+# ---- Validate taco-bench and SAM python simulator
+validate-bench: compiler/build/taco-bench guard-VALIDATION_OUTPUT_PATH
+ifeq ($(BENCHES),"")
+	$(CMD) --benchmark_repetitions=1
+else
+	$(CMD) --benchmark_filter="$(BENCHES)" --benchmark_repetitions=1
+endif
+
+# Separate target to run the SAM sim python benchmarks with taco cross validation logic.
+# validate-sam-bench: validation-path
+# 	pytest $(IGNORE_FLAGS) $(BENCHFLAGS) $(BENCHES)
+
+# ---- Setup proper environment stuff ----
 suitesparse-formats: guard-SUITESPARSE_FORMATTED_PATH guard-SUITESPARSE_PATH
 	rm -rf ${SUITESPARSE_FORMATTED_PATH}/*
 	set -e && ./scripts/generate_suitesparse_formats.sh
 
-.PHONY: frostt-formats
-frostt-formats: env taco/build
+frostt-formats: env taco/build guard-FROSTT_FORMATTED_PATH guard-FROSTT_PATH
 	./scripts/generate_frostt_formats.sh
 
 .PHONY: env
@@ -44,17 +120,6 @@ pydepends:
 	conda env export > environment.yml
 	pip list --format=freeze > requirements.txt
 
-sam: taco/build 
-	 cd compiler && bash -xe ./sam-kernels.sh
-
-taco/build: submodules
-	 mkdir -p compiler/taco/build 
-	 cd compiler/taco/build && cmake -DCMAKE_BUILD_TYPE=Release .. && make -j8
-
-#.PHONY: results
-#results:
-#	 mkdir -p $
-#	 mkdir -p results/numpy
 
 .PHONY: submodules
 submodules:

compiler/CMakeLists.txt

@@ -0,0 +1,25 @@
+cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
+project(taco-bench)
+set(CMAKE_BUILD_TYPE "release")
+option(NEVA "Build benchmarks for the neva or kiwi clusters." OFF)
+option(LANKA "Build benchmarks for the lanka cluster." OFF)
+
+add_subdirectory(taco)
+add_subdirectory(benchmark)
+
+include_directories(taco taco/include benchmark/include taco/src)
+
+file(GLOB TEST_SOURCES *.cpp *.h)
+
+set(CMAKE_CXX_FLAGS "${C_CXX_FLAGS} -std=c++14")
+
+add_executable(taco-bench ${TEST_SOURCES} bench.h)
+target_link_libraries(taco-bench benchmark::benchmark)
+target_link_libraries(taco-bench taco)
+# Only link stdc++fs on linux. Mac's don't need this set.
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  set(MACOSX TRUE)
+endif()
+if (NOT MACOSX)
+  target_link_libraries(taco-bench stdc++fs)
+endif()