WIP

Signed-off-by: Andrew Stein <[email protected]>
finos · Sep 9, 2024 · af6486a · af6486a
1 parent 4e03507
commit af6486a
Show file tree

Hide file tree

Showing 22 changed files with 59 additions and 5,586 deletions.
diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml
@@ -113,7 +113,7 @@ runs:
           id: vcpkg-step
           shell: pwsh
           run: |
-              vcpkg.exe install boost-thread boost-algorithm boost-filesystem boost-multi-index boost-multiprecision boost-program-options boost-system boost-unordered boost-uuid
+              vcpkg.exe install boost-headers boost-thread boost-algorithm boost-filesystem boost-multi-index boost-multiprecision boost-program-options boost-system boost-unordered boost-uuid
               vcpkg.exe integrate install
               echo "VCPKG_INSTALLATION_ROOT=${env:VCPKG_INSTALLATION_ROOT}"
               echo "VCPKG_INSTALLATION_ROOT=${env:VCPKG_INSTALLATION_ROOT}" >> $env:GITHUB_OUTPUT

diff --git a/cmake/arrow.txt.in b/cmake/arrow.txt.in
@@ -10,7 +10,7 @@ project(arrow-download NONE)
 include(ExternalProject)
 ExternalProject_Add(apachearrow
   GIT_REPOSITORY    https://github.com/apache/arrow.git
-  GIT_TAG           apache-arrow-12.0.0
+  GIT_TAG           apache-arrow-17.0.0
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/arrow-src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/arrow-build"
   CONFIGURE_COMMAND ""

diff --git a/cmake/arrow/CMakeLists.txt b/cmake/arrow/CMakeLists.txt
diff --git a/cmake/arrow/config.h b/cmake/arrow/config.h
diff --git a/cmake/flatbuffers.txt.in b/cmake/flatbuffers.txt.in
@@ -5,7 +5,7 @@ project(flatbuffers-download NONE)
 include(ExternalProject)
 ExternalProject_Add(flatbuffers
   GIT_REPOSITORY    https://github.com/google/flatbuffers.git
-  GIT_TAG           v2.0.5
+  GIT_TAG           v23.5.26
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/flatbuffers-src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/flatbuffers-build"
   CONFIGURE_COMMAND ""

diff --git a/cmake/modules/FindFlatbuffers.cmake b/cmake/modules/FindFlatbuffers.cmake
diff --git a/cmake/modules/FindInstallDependency.cmake b/cmake/modules/FindInstallDependency.cmake
@@ -37,13 +37,26 @@ function(psp_build_dep name cmake_file)
     endif()
 
     if(${name} STREQUAL arrow)
-        # Overwrite arrow's CMakeLists with our custom, minimal CMakeLists.
-        configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/cpp/ COPYONLY)
-        configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/config.h ${CMAKE_BINARY_DIR}/${name}-src/cpp/src/arrow/util/ COPYONLY)
+        set(ARROW_SIMD_LEVEL "NONE")
+        set(ARROW_DEFINE_OPTIONS ON)
+        set(ARROW_RUNTIME_SIMD_LEVEL "NONE")
+        set(ARROW_BUILD_SHARED OFF)
+        set(ARROW_BUILD_STATIC ON)
+        set(ARROW_JEMALLOC OFF)
+        set(ARROW_CSV ON)
+        set(ARROW_LZ4 ON)
+        set(ARROW_WITH_ZSTD ON)
+        set(ARROW_WITH_LZ4 ON)
+        set(ARROW_ENABLE_THREADING OFF)
+        set(ARROW_NO_EXPORT ON)
+        set(ARROW_BOOST_USE_SHARED OFF)
+
+        include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-build/src)
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/cpp/
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL)
 
+        set_property(DIRECTORY "${CMAKE_BINARY_DIR}/${name}-src/cpp/" APPEND PROPERTY COMPILE_DEFINITIONS "-DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF -DARROW_NO_DEPRECATED_API")
         include_directories(SYSTEM ${CMAKE_BINARY_DIR}/${name}-src/cpp/src/)
     elseif(${name} STREQUAL exprtk)
         # no cmakelists - just include the header
@@ -78,7 +91,7 @@ function(psp_build_dep name cmake_file)
     endif()
 
     if(NOT PSP_WASM_BUILD AND (MACOS OR NOT MANYLINUX))
-        if(${name} STREQUAL arrow OR ${name} STREQUAL flatbuffers OR ${name} STREQUAL double-conversion OR ${name} STREQUAL re2)
+        if(${name} STREQUAL flatbuffers OR ${name} STREQUAL double-conversion OR ${name} STREQUAL re2)
             target_compile_options(${name} PRIVATE -fvisibility=hidden)
         endif()
     endif()

diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt
@@ -303,9 +303,9 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
                     -fexceptions \
                     -g3 \
                     ")
-                 if (PSP_WASM_EXCEPTIONS)
-                     set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
-                 endif()
+                if (PSP_WASM_EXCEPTIONS)
+                    set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
+                endif()
             endif ()
         else()
             set(OPT_FLAGS " \
@@ -406,10 +406,18 @@ set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Disable rapidjson tests")
 set(CMAKE_C_FLAGS "  \
 -O3 \
 ")
+if (PSP_WASM_EXCEPTIONS)
+set(CMAKE_CXX_FLAGS " -fwasm-exceptions \
+-O3 \
+-g0 \
+")
+else()
 set(CMAKE_CXX_FLAGS " \
 -O3 \
 ")
 
+endif()
+
 if(PSP_PYODIDE)
     set(RELOCATABLE_FLAGS "-sRELOCATABLE=1 -sSIDE_MODULE=2 -sWASM_BIGINT=1")
 
@@ -640,7 +648,7 @@ if(PSP_WASM_BUILD AND NOT PSP_PYTHON_BUILD)
     add_library(psp ${WASM_SOURCE_FILES})
     target_compile_definitions(psp PRIVATE PSP_ENABLE_WASM=1)
     set_target_properties(psp PROPERTIES COMPILE_FLAGS "")
-    target_link_libraries(psp PRIVATE arrow re2 protos)
+    target_link_libraries(psp PRIVATE arrow_static re2 protos)
 
     add_executable(perspective_esm src/cpp/binding_api.cpp)
     target_link_libraries(perspective_esm psp protos)
@@ -695,34 +703,19 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
             # .dll not importable
             # set_property(TARGET psppy PROPERTY SUFFIX .pyd)
         elseif(MACOS OR NOT MANYLINUX)
-            # target_compile_options(psppy PRIVATE -Wdeprecated-declarations)
             set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
-            # set_property(TARGET psppy PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
-
             target_compile_options(psp PRIVATE -fvisibility=hidden)
-            # target_compile_options(psppy PRIVATE -fvisibility=hidden)
         elseif(MANYLINUX)
         else()
             target_compile_options(psp PRIVATE -fvisibility=hidden)
-            # target_compile_options(psppy PRIVATE -Wdeprecated-declarations)
         endif()
 
         # Link against minimal arrow static library
-        target_link_libraries(psp PRIVATE arrow re2 protos)
-        # target_link_libraries(psppy psp)
-
-        # The compiled libraries will be put in CMAKE_LIBRARY_OUTPUT_DIRECTORY by default. In the
-        # setup.py file, we designate this to be in the build/lib.<platform> directory. However,
-        # since we want to be able to test perspective in-source, we also copy the libraries into
-        # the source folder. These two commands do that.
-        # add_custom_command(TARGET psp POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:psp> ${PSP_PYTHON_SRC}/table/)
-        # add_custom_command(TARGET psppy POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:psppy> ${PSP_PYTHON_SRC}/table/)
-
-    # #######################
+        target_link_libraries(psp PRIVATE arrow_static re2 protos lz4_static libzstd_static)
     else()
         add_library(psp STATIC ${WASM_SOURCE_FILES})
         target_compile_options(psp PRIVATE -fvisibility=hidden)
-        target_link_libraries(psp PRIVATE arrow re2 protos)
+        target_link_libraries(psp PRIVATE arrow_static re2 protos)
     endif()
 
     if(PSP_CPP_BUILD_STRICT AND NOT WIN32)
@@ -745,6 +738,6 @@ if(NOT DEFINED ENV{PSP_DISABLE_CLANGD})
     include(SetupClangd)
 endif()
 
-if(NOT WIN32)
-    include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
-endif()
+# if(NOT WIN32)
+include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
+# endif()
diff --git a/cpp/perspective/src/cpp/arrow_csv.cpp b/cpp/perspective/src/cpp/arrow_csv.cpp
@@ -15,14 +15,7 @@
 #include <perspective/arrow_csv.h>
 #include <arrow/util/value_parsing.h>
 #include <arrow/io/memory.h>
-
-#ifdef PSP_ENABLE_WASM
-// This causes build warnings
-// https://github.com/emscripten-core/emscripten/issues/8574
-#include <perspective/vendor/arrow_single_threaded_reader.h>
-#else
 #include <arrow/csv/reader.h>
-#endif
 
 template <class TimePoint>
 static inline arrow::TimestampType::c_type

diff --git a/cpp/perspective/src/cpp/table.cpp b/cpp/perspective/src/cpp/table.cpp
@@ -16,7 +16,6 @@
 #include "perspective/data_table.h"
 #include "perspective/raw_types.h"
 #include "perspective/schema.h"
-// #include "arrow/vendored/datetime/date.h"
 #include "rapidjson/document.h"
 #include <chrono>
 #include <ctime>
@@ -448,18 +447,6 @@ rapidjson_type_to_dtype(const rapidjson::Value& value) {
             std::chrono::system_clock::time_point tp;
 
             if (parse_all_date_time(tm, tp, str)) {
-                LOG_DEBUG(
-                    "Parsed date: " << tm.tm_year + 1900 << "-" << tm.tm_mon + 1
-                                    << "-" << tm.tm_mday << " " << tm.tm_hour
-                                    << ":" << tm.tm_min << ":" << tm.tm_sec
-                );
-                auto tpm =
-                    std::chrono::duration_cast<std::chrono::milliseconds>(
-                        tp.time_since_epoch()
-                    )
-                        .count();
-                LOG_DEBUG("TP: " << tpm << '\n');
-
                 if (tm.tm_hour == 0 && tm.tm_min == 0 && tm.tm_sec == 0) {
                     return t_dtype::DTYPE_DATE;
                 }