feat(tests): adding VM benchmarks, useful for optimization exploratio…

…n and code refactoring This will be used to avoid any performance regression, running as often as I can on my personal computer and/or server (a CI can't be trusted with the kind of accuracy I'm seeking here ; even though the relative benchmarks are running on a CI it's fine here because we are only looking at the difference between ArkScript and other languages (whose versions are fixed)).
ArkScript-lang · Apr 17, 2024 · 517a4c3 · 517a4c3
1 parent ecb0547
commit 517a4c3
Show file tree

Hide file tree

Showing 7 changed files with 189 additions and 71 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -14,7 +14,7 @@ repos:
     rev: '336fdd7c3cab698ead0b1c95157b9e74d3906b62'
     hooks:
       - id: cppcheck
-        args: [ "--platform=unix64", "-j", "4", "-I include src" ]
+        args: [ "--platform=unix64", "--inline-suppr" ]
   - repo: https://github.com/compilerla/conventional-pre-commit
     rev: 'v3.2.0'
     hooks:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -16,54 +16,56 @@ set(ARK_COMMIT ${GIT_COMMIT_HASH})
 
 include(cmake/link_time_optimization.cmake)
 include(cmake/sanitizers.cmake)
+include(cmake/CPM.cmake)
 include(GNUInstallDirs)  # Uses GNU Install directory variables
 
 # configure installer.iss
 configure_file(
-    ${ark_SOURCE_DIR}/Installer.iss.in
-    ${ark_SOURCE_DIR}/Installer.iss)
+        ${ark_SOURCE_DIR}/Installer.iss.in
+        ${ark_SOURCE_DIR}/Installer.iss)
 
 # setting up compilations options
 
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
     set(CMAKE_COMPILER_IS_CLANG ON)
-endif()
+endif ()
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 # files needed for the library ArkReactor
 file(GLOB_RECURSE SOURCE_FILES
-    ${ark_SOURCE_DIR}/src/arkreactor/*.cpp
-    ${ark_SOURCE_DIR}/lib/fmt/src/format.cc)
+        ${ark_SOURCE_DIR}/src/arkreactor/*.cpp
+        ${ark_SOURCE_DIR}/lib/fmt/src/format.cc)
 
 add_library(ArkReactor SHARED ${SOURCE_FILES})
-
-target_compile_features(ArkReactor PRIVATE cxx_std_20)
-
 enable_lto(ArkReactor)
+target_include_directories(ArkReactor
+        PUBLIC
+        ${ark_SOURCE_DIR}/include)
+target_compile_features(ArkReactor PRIVATE cxx_std_20)
 
 if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANG OR APPLE)
     target_compile_options(ArkReactor
-        PUBLIC
-        -Wall -Wextra -pedantic -Wstrict-aliasing
-        -Wno-unknown-pragmas            # We use pragmas to disable warnings we understand.
-                                        # So we need to disable the warning about pragmas.
-        -Wno-unknown-warning-option     # Disable warnings about disabling warnings we have
-                                        # disabled.
-        )
+            PUBLIC
+            -Wall -Wextra -pedantic -Wstrict-aliasing
+            -Wno-unknown-pragmas            # We use pragmas to disable warnings we understand.
+            # So we need to disable the warning about pragmas.
+            -Wno-unknown-warning-option     # Disable warnings about disabling warnings we have
+            # disabled.
+    )
 
     if (CMAKE_COMPILER_IS_GNUCXX)
         # The package utf8 has an issue with constant overflow.
         # Once this is fixed remove this flag:
         target_compile_options(ArkReactor PUBLIC -Wno-overflow)
-    endif()
+    endif ()
 
     if (APPLE)
         # The standard SSH libraries are depreciate on APPLE.
         # Thus they currently generate a warning that we have to ignore for now.
         # Though we should look to fix that in the future.
         target_compile_options(ArkReactor PUBLIC -Wno-deprecated-declarations)
-    endif()
+    endif ()
 
     # The following sub-modules link with -L/usr/local/lib
     # This causes a warning that is fatal. To prevent this
@@ -74,105 +76,98 @@ if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANG OR APPLE)
     #   lib/modules/msgpack/CMakeFiles/msgpack.dir/__/submodules/msgpack-cpp/src/vrefbuffer.c.o
     #   lib/modules/msgpack/CMakeFiles/msgpack.dir/__/submodules/msgpack-cpp/src/zone.c.o
     target_compile_options(ArkReactor
-        PUBLIC
-        -Wno-unused-command-line-argument)
+            PUBLIC
+            -Wno-unused-command-line-argument)
 
     # The nlohmann/json.hpp external project has unused typedefs in the code
     # to compensate for this error we remove the following warning.
     if (CMAKE_COMPILER_IS_CLANG OR APPLE)
         target_compile_options(ArkReactor PUBLIC -Wno-unused-local-typedef)
     elseif (CMAKE_COMPILER_IS_GNUCXX)
         target_compile_options(ArkReactor PUBLIC -Wno-unused-local-typedefs)
-    endif()
+    endif ()
 
 elseif (MSVC)
     target_compile_options(ArkReactor
-        PUBLIC
-        /W4
-        /MP4      # build multiple source files concurrently
-        /EHa      # set exception model to standard C++ stack unwinding
-        /wd4267   # disable warning about data loss (size_t -> int)
-        /wd4244   # disable warning about data loss (size_t -> char)
-        /wd4505   # disable warning about unused static function was deleted
-        /wd4068)  # disable warnings about unknown pragmas.
-endif()
+            PUBLIC
+            /W4
+            /MP4      # build multiple source files concurrently
+            /EHa      # set exception model to standard C++ stack unwinding
+            /wd4267   # disable warning about data loss (size_t -> int)
+            /wd4244   # disable warning about data loss (size_t -> char)
+            /wd4505   # disable warning about unused static function was deleted
+            /wd4068)  # disable warnings about unknown pragmas.
+endif ()
 
 # Link libraries
 
 add_subdirectory("${ark_SOURCE_DIR}/lib/termcolor" EXCLUDE_FROM_ALL)
+target_link_libraries(ArkReactor PUBLIC termcolor)
 
 target_include_directories(ArkReactor
-    PUBLIC
-    "${ark_SOURCE_DIR}/lib/picosha2/"
-    "${ark_SOURCE_DIR}/lib/fmt/include")
-
-target_link_libraries(ArkReactor PUBLIC termcolor)
+        PUBLIC
+        "${ark_SOURCE_DIR}/lib/picosha2/"
+        "${ark_SOURCE_DIR}/lib/fmt/include")
 
 if (UNIX OR LINUX)
     find_package(Threads)
     target_link_libraries(ArkReactor PRIVATE ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
-endif()
+endif ()
 
 # configuring Constants.hpp
 message(STATUS "ArkScript version ${ARK_VERSION_MAJOR}.${ARK_VERSION_MINOR}.${ARK_VERSION_PATCH}")
 configure_file(
-    ${ark_SOURCE_DIR}/include/Ark/Constants.hpp.in
-    ${ark_SOURCE_DIR}/include/Ark/Constants.hpp)
-
-# including content of project
-target_include_directories(ArkReactor
-    PUBLIC
-    ${ark_SOURCE_DIR}/include)
-
-target_compile_features(ArkReactor PRIVATE cxx_std_20)
+        ${ark_SOURCE_DIR}/include/Ark/Constants.hpp.in
+        ${ark_SOURCE_DIR}/include/Ark/Constants.hpp)
 
 # Installation rules
 
 # Installs the dynamic library file.
 install(TARGETS ArkReactor
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
 # Install header files
 install(DIRECTORY ${ark_SOURCE_DIR}/include/
         DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
 # Install the standard library
-if(NOT ARK_NO_STD)
+if (NOT ARK_NO_STD)
     install(DIRECTORY ${ark_SOURCE_DIR}/lib/std/
-        DESTINATION ${CMAKE_INSTALL_LIBDIR}/Ark/std
-        FILES_MATCHING PATTERN "*.ark"
-        PATTERN "std/tests" EXCLUDE
-        PATTERN "std/.github" EXCLUDE)
-endif()
+            DESTINATION ${CMAKE_INSTALL_LIBDIR}/Ark/std
+            FILES_MATCHING PATTERN "*.ark"
+            PATTERN "std/tests" EXCLUDE
+            PATTERN "std/.github" EXCLUDE)
+endif ()
 
 # COMPILATION RELATED
 
 target_compile_definitions(ArkReactor PRIVATE ARK_EXPORT)
 
-option(ARK_BUILD_EXE        "Build a standalone arkscript executable" Off)
-option(ARK_ENABLE_SYSTEM    "Enable sys:exec" On)  # enable use of (sys:exec "command here")
-option(ARK_PROFILER_COUNT   "Enable creations/copies/moves counting on the Value" Off)
-option(ARK_PROFILER_MIPS    "Enable MIPS calculation" Off)
-option(ARK_NO_STDLIB        "Do not install the standard library with the Ark library" Off)
-option(ARK_BUILD_MODULES    "Build the std library modules or not" Off)
-option(ARK_SANITIZERS       "Enable ASAN and UBSAN" Off)
-option(ARK_TESTS            "Build ArkScript unit tests" Off)
+option(ARK_BUILD_EXE "Build a standalone arkscript executable" Off)
+option(ARK_ENABLE_SYSTEM "Enable sys:exec" On)  # enable use of (sys:exec "command here")
+option(ARK_PROFILER_COUNT "Enable creations/copies/moves counting on the Value" Off)
+option(ARK_PROFILER_MIPS "Enable MIPS calculation" Off)
+option(ARK_NO_STDLIB "Do not install the standard library with the Ark library" Off)
+option(ARK_BUILD_MODULES "Build the std library modules or not" Off)
+option(ARK_SANITIZERS "Enable ASAN and UBSAN" Off)
+option(ARK_TESTS "Build ArkScript unit tests" Off)
+option(ARK_BENCHMARKS "Build ArkScript benchmarks" Off)
 
 if (ARK_PROFILER_COUNT)
     target_compile_definitions(ArkReactor PRIVATE -DARK_PROFILER_COUNT)
-endif()
+endif ()
 
 if (ARK_ENABLE_SYSTEM)
     target_compile_definitions(ArkReactor PRIVATE -DARK_ENABLE_SYSTEM)
-endif()
+endif ()
 
 if (ARK_PROFILER_MIPS)
     target_compile_definitions(ArkReactor PRIVATE -DARK_PROFILER_MIPS)
-endif()
+endif ()
 
 if (ARK_BUILD_MODULES)
     add_subdirectory(${ark_SOURCE_DIR}/lib/modules)
-endif()
+endif ()
 
 # TODO: consider using ctest
 if (ARK_TESTS)
@@ -190,13 +185,25 @@ if (ARK_TESTS)
     add_compile_definitions(BOOST_UT_DISABLE_MODULE)
     target_compile_features(unittests PRIVATE cxx_std_20)
     target_compile_definitions(unittests PRIVATE ARK_TESTS_ROOT="${CMAKE_CURRENT_SOURCE_DIR}/")
-endif()
+endif ()
+
+if (ARK_BENCHMARKS)
+    set(BENCHMARK_ENABLE_GTEST_TESTS Off)
+    set(BENCHMARK_ENABLE_TESTING Off)
+    CPMAddPackage("gh:google/[email protected]")
+
+    add_executable(bench tests/benchmarks/main.cpp)
+    target_link_libraries(bench PUBLIC ArkReactor termcolor benchmark::benchmark)
+    target_compile_features(bench PRIVATE cxx_std_20)
+    target_compile_definitions(bench PRIVATE ARK_TESTS_ROOT="${CMAKE_CURRENT_SOURCE_DIR}/")
+    enable_lto(bench)
+endif ()
 
 if (ARK_BUILD_EXE)
     # additional files needed for the exe (repl, command line and stuff)
     file(GLOB_RECURSE EXE_SOURCES
-        ${ark_SOURCE_DIR}/src/arkscript/*.cpp
-        ${ark_SOURCE_DIR}/lib/fmt/src/format.cc)
+            ${ark_SOURCE_DIR}/src/arkscript/*.cpp
+            ${ark_SOURCE_DIR}/lib/fmt/src/format.cc)
     add_executable(arkscript ${EXE_SOURCES})
 
     if (MSVC)
@@ -206,7 +213,7 @@ if (ARK_BUILD_EXE)
         #   /wd4267:    conversion from 'size_t' to 'type', possible loss of data
         #   /wd4244:    conversion from 'type1' to 'type2', possible loss of data
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4267 /wd4244")
-    endif()
+    endif ()
 
     add_subdirectory("${ark_SOURCE_DIR}/lib/replxx" EXCLUDE_FROM_ALL)
     add_subdirectory("${ark_SOURCE_DIR}/lib/clipp" EXCLUDE_FROM_ALL)
@@ -218,11 +225,11 @@ if (ARK_BUILD_EXE)
 
     # Installs the arkscript executable.
     install(TARGETS arkscript
-        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
-endif()
+            RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+endif ()
 
 if (ARK_SANITIZERS)
     message(STATUS "Enabling address sanitizer and undefined behavior sanitizer")
     add_address_sanitizer()
     add_undefined_sanitizer()
-endif()
+endif ()
diff --git a/cmake/CPM.cmake b/cmake/CPM.cmake
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: MIT
+#
+# SPDX-FileCopyrightText: Copyright (c) 2019-2023 Lars Melchior and contributors
+
+set(CPM_DOWNLOAD_VERSION 0.39.0)
+set(CPM_HASH_SUM "66639bcac9dd2907b2918de466783554c1334446b9874e90d38e3778d404c2ef")
+
+if(CPM_SOURCE_CACHE)
+  set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+elseif(DEFINED ENV{CPM_SOURCE_CACHE})
+  set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+else()
+  set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+endif()
+
+# Expand relative path. This is important if the provided path contains a tilde (~)
+get_filename_component(CPM_DOWNLOAD_LOCATION ${CPM_DOWNLOAD_LOCATION} ABSOLUTE)
+
+file(DOWNLOAD
+     https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
+     ${CPM_DOWNLOAD_LOCATION} EXPECTED_HASH SHA256=${CPM_HASH_SUM}
+)
+
+include(${CPM_DOWNLOAD_LOCATION})
diff --git a/tests/benchmarks/ackermann.ark b/tests/benchmarks/ackermann.ark
@@ -0,0 +1,11 @@
+(let ackermann (fun (m n) {
+    (if (> m 0)
+        # then
+        (if (= 0 n)
+            # then
+            (ackermann (- m 1) 1)
+            # else
+            (ackermann (- m 1) (ackermann m (- n 1))))
+        # else
+        (+ 1 n))}))
+(ackermann 3 7)
diff --git a/tests/benchmarks/fibonacci.ark b/tests/benchmarks/fibonacci.ark
@@ -0,0 +1,5 @@
+(let fibo (fun (n)
+    (if (< n 2)
+        n
+        (+ (fibo (- n 1)) (fibo (- n 2))))))
+(fibo 22)
diff --git a/tests/benchmarks/main.cpp b/tests/benchmarks/main.cpp
@@ -0,0 +1,50 @@
+#include <benchmark/benchmark.h>
+
+#include <string>
+
+#include <Ark/VM/State.hpp>
+#include <Ark/VM/VM.hpp>
+
+// cppcheck-suppress constParameterCallback
+void ark_quicksort(benchmark::State& s)
+{
+    Ark::State state;
+    state.doFile(std::string(ARK_TESTS_ROOT) + "tests/benchmarks/quicksort.ark");
+
+    for (auto _ : s)
+    {
+        Ark::VM vm(state);
+        benchmark::DoNotOptimize(vm.run());
+    }
+}
+BENCHMARK(ark_quicksort)->Unit(benchmark::kMillisecond);
+
+// cppcheck-suppress constParameterCallback
+void ark_ackermann(benchmark::State& s)
+{
+    Ark::State state;
+    state.doFile(std::string(ARK_TESTS_ROOT) + "tests/benchmarks/ackermann.ark");
+
+    for (auto _ : s)
+    {
+        Ark::VM vm(state);
+        benchmark::DoNotOptimize(vm.run());
+    }
+}
+BENCHMARK(ark_ackermann)->Unit(benchmark::kMillisecond)->Iterations(50);
+
+// cppcheck-suppress constParameterCallback
+void ark_fibonacci(benchmark::State& s)
+{
+    Ark::State state;
+    state.doFile(std::string(ARK_TESTS_ROOT) + "tests/benchmarks/fibonacci.ark");
+
+    for (auto _ : s)
+    {
+        Ark::VM vm(state);
+        benchmark::DoNotOptimize(vm.run());
+    }
+}
+BENCHMARK(ark_fibonacci)->Unit(benchmark::kMillisecond)->Iterations(100);
+
+BENCHMARK_MAIN();
diff --git a/tests/benchmarks/quicksort.ark b/tests/benchmarks/quicksort.ark
@@ -0,0 +1,21 @@
+(let filter (fun (lst cond) {
+    (mut output [])
+    (mut i 0)
+    (while (< i (len lst)) {
+        (if (cond (@ lst i))
+            (append! output (@ lst i)))
+        (set i (+ 1 i))})
+    output }))
+
+(let quicksort (fun (array) {
+    (if (empty? array)
+        []
+        {
+            (let pivot (head array))
+            (mut less (quicksort (filter (tail array) (fun (e) (< e pivot)))))
+            (let more (quicksort (filter (tail array) (fun (e) (>= e pivot)))))
+            (concat! less [pivot] more)
+            less })}))
+
+(let a [3 6 1 5 1 65 324 765 1 6 3 0 6 9 6 5 3 2 5 6 7 64 645 7 345 432 432 4 324 23])
+(quicksort a)