Meta-programming tools and arrow types traits support.

CMakeLists.txt

@@ -54,11 +54,14 @@ set(SPARROW_HEADERS
     ${SPARROW_INCLUDE_DIR}/sparrow/buffer.hpp
     ${SPARROW_INCLUDE_DIR}/sparrow/fixed_size_layout.hpp
     ${SPARROW_INCLUDE_DIR}/sparrow/data_type.hpp
+    ${SPARROW_INCLUDE_DIR}/sparrow/data_traits.hpp
     ${SPARROW_INCLUDE_DIR}/sparrow/dynamic_bitset.hpp
     ${SPARROW_INCLUDE_DIR}/sparrow/iterator.hpp
     ${SPARROW_INCLUDE_DIR}/sparrow/mp_utils.hpp
     ${SPARROW_INCLUDE_DIR}/sparrow/sparrow_version.hpp
     ${SPARROW_INCLUDE_DIR}/sparrow/variable_size_binary_layout.hpp
+
+    ${SPARROW_INCLUDE_DIR}/sparrow/details/3rdparty/float16_t.hpp
 )
 
 add_library(sparrow INTERFACE)

include/sparrow/data_traits.hpp

@@ -0,0 +1,141 @@
+// Copyright 2024 Man Group Operations Limited
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "data_type.hpp"
+#include "fixed_size_layout.hpp"
+#include "variable_size_binary_layout.hpp"
+
+namespace sparrow
+{
+
+    template<class T>
+    struct common_native_types_traits
+    {
+        using value_type = T;
+        using default_layout = fixed_size_layout<T>;
+    };
+
+    template<>
+    struct arrow_traits<std::nullopt_t>
+    {
+        static constexpr data_type type_id = data_type::NA;
+        using value_type = std::nullopt_t;
+        using default_layout = fixed_size_layout<value_type>; // TODO: replace this by a special layout that's always empty
+
+    };
+
+    template<>
+    struct arrow_traits<bool> : common_native_types_traits<bool>
+    {
+        static constexpr data_type type_id = data_type::BOOL;
+    };
+
+    template<>
+    struct arrow_traits<std::uint8_t> : common_native_types_traits<std::uint8_t>
+    {
+        static constexpr data_type type_id = data_type::UINT8;
+    };
+
+    template<>
+    struct arrow_traits<std::int8_t> : common_native_types_traits<std::int8_t>
+    {
+        static constexpr data_type type_id = data_type::INT8;
+    };
+
+    template<>
+    struct arrow_traits<std::uint16_t> : common_native_types_traits<std::uint16_t>
+    {
+        static constexpr data_type type_id = data_type::UINT16;
+    };
+
+    template<>
+    struct arrow_traits<std::int16_t> : common_native_types_traits<std::int16_t>
+    {
+        static constexpr data_type type_id = data_type::INT16;
+    };
+
+    template<>
+    struct arrow_traits<std::uint32_t> : common_native_types_traits<std::uint32_t>
+    {
+        static constexpr data_type type_id = data_type::UINT32;
+    };
+
+    template<>
+    struct arrow_traits<std::int32_t> : common_native_types_traits<std::int32_t>
+    {
+        static constexpr data_type type_id = data_type::INT32;
+    };
+
+    template<>
+    struct arrow_traits<std::uint64_t> : common_native_types_traits<std::uint64_t>
+    {
+        static constexpr data_type type_id = data_type::UINT64;
+    };
+
+    template<>
+    struct arrow_traits<std::int64_t> : common_native_types_traits<std::int64_t>
+    {
+        static constexpr data_type type_id = data_type::INT64;
+    };
+
+    template<>
+    struct arrow_traits<float16_t> : common_native_types_traits<float16_t>
+    {
+        static constexpr data_type type_id = data_type::HALF_FLOAT;
+    };
+
+    template<>
+    struct arrow_traits<float32_t> : common_native_types_traits<float32_t>
+    {
+        static constexpr data_type type_id = data_type::FLOAT;
+    };
+
+    template<>
+    struct arrow_traits<float64_t> : common_native_types_traits<float64_t>
+    {
+        static constexpr data_type type_id = data_type::DOUBLE;
+    };
+
+    template<>
+    struct arrow_traits<std::string>
+    {
+        static constexpr data_type type_id = data_type::STRING;
+        using value_type = std::string;
+        using default_layout = variable_size_binary_layout<value_type, std::string_view, std::string_view>; // FIXME: this is incorrect, change when we have the right types
+
+    };
+
+    namespace predicate
+    {
+
+        struct {
+            template<class T>
+            consteval bool operator()(mpl::typelist<T>)
+            {
+                return sparrow::is_arrow_base_type<T>;
+            }
+        } constexpr is_arrow_base_type;
+
+        struct {
+            template<class T>
+            consteval bool operator()(mpl::typelist<T>)
+            {
+                return sparrow::is_arrow_traits< arrow_traits<T> >;
+            }
+        } constexpr has_arrow_traits;
+    }
+
+}

include/sparrow/data_type.hpp

@@ -14,8 +14,46 @@
 
 #pragma once
 
+#include <cstdint>
+#include <climits>
+#include <string>
+#include <vector>
+#include <optional>
+
+#include "sparrow/mp_utils.hpp"
+
+// TODO: use exclusively `std::float16_t etc. once we switch to c++23, see https://en.cppreference.com/w/cpp/types/floating-point
+#if __cplusplus <= 202002L
+#   include "details/3rdparty/float16_t.hpp"
+#else
+#   include <stdfloat>
+#endif
+
+
 namespace sparrow
 {
+// TODO: use exclusively `std::float16_t etc. once we switch to c++23, see https://en.cppreference.com/w/cpp/types/floating-point
+#if __cplusplus <= 202002L
+    using float16_t = numeric::float16_t;
+    using float32_t = float;
+    using float64_t = double;
+#else
+    using float16_t = std::float16_t;
+    using float32_t = std::float32_t;
+    using float64_t = std::float64_t;
+#endif
+
+    // We need to be sure the current target platform is setup to support correctly these types.
+    static_assert(sizeof(float16_t) == 2);
+    static_assert(sizeof(float32_t) == 4);
+    static_assert(sizeof(float64_t) == 8);
+    static_assert(std::is_floating_point_v<float16_t>);
+    static_assert(std::is_floating_point_v<float32_t>);
+    static_assert(std::is_floating_point_v<float64_t>);
+    static_assert(CHAR_BIT == 8);
+
+
+    /// Runtime identifier of arrow data types, usually associated with raw bytes with the associated value.
     // TODO: does not support all types specified by the Arrow specification
     // yet
     enum class data_type
@@ -38,9 +76,114 @@ namespace sparrow
         // Variable-length bytes (no guarantee of UTF8-ness)
         BINARY,
         // Fixed-size binary. Each value occupies the same number of bytes
-        FIXED_SIZE_BINARY
+        FIXED_SIZE_BINARY,
     };
 
+    /// C++ types value representation types matching Arrow types.
+    // NOTE: this needs to be in sync-order with `data_type`
+    using all_base_types_t = mpl::typelist
+    <
+        std::nullopt_t  // REVIEW: not sure about if we need to have this one? for representing NA? is this the right type?
+    ,   bool
+    ,   std::uint8_t
+    ,   std::int8_t
+    ,   std::uint16_t
+    ,   std::int16_t
+    ,   std::uint32_t
+    ,   std::int32_t
+    ,   std::uint64_t
+    ,   std::int64_t
+    ,   float16_t
+    ,   float32_t
+    ,   float64_t
+    ,   std::string
+    //,   std::vector<std::byte> // REVIEW should this be uint8_t? char? buffer<unit8_t>?
+    // TODO: add missing fundamental types here
+    >;
+
+    /// Type list of every C++ representation types supported by default, in order matching `data_type` related values.
+    static constexpr all_base_types_t all_base_types;
+
+    /// Matches C++ representation types which are supported by default.
+    template< class T >
+    concept is_arrow_base_type = mpl::contains<T>(all_base_types);
+
+    /// Provides compile-time information about Arrow data types.
+    /// Custom types can be made compatible by implementing this traits type.
+    ///
+    /// The following information must be provided if the type is an arrow type:
+    /// - type_id : the runtime identifier value for that type, see `data_type`
+    /// - value_type : the value representation type to use in C++ (usually T)
+    /// - default_layout: the layout to use for that type FIXME: be more precise than that
+    /// - MORE TO COME SOON, SEE TODOs BELOW
+    ///
+    /// @tparam Type for C++ value-representation that this type describes the traits of.
+    ///
+    /// @note: See ./arrow_traits.hpp for implementations for default base types.
+    /// @see `is_arrow_traits`, `has_arrow_type_traits`
+    template<class T>
+    struct arrow_traits;
+
+    /// Matches valid and complete `arrow_traits` specializations for type T.
+    /// Every type that needs to be compatible with this library's interface must
+    /// provide a specialization of `arrow_traits`
+    /// @see `arrow_traits`, `has_arrow_type_traits`
+    template<class T>
+    concept is_arrow_traits = mpl::is_type_instance_of_v< T, arrow_traits >
+        and requires
+        {
+            /// Must provide a compile-time value of type `data_type`.
+            /// This is used to identify which arrow data type is represented in `value_type`
+            requires std::same_as< std::remove_cvref_t<decltype(T::type_id)>, ::sparrow::data_type >;
+
+            /// The C++ representation of the arrow value. For `arrow_traits<X>`, this is usually `X`.
+            typename T::value_type;
+
+            /// The arrow (binary) layout to use by default for representing a set of data for that type.
+            typename T::default_layout;
+
+            // TODO: add more interface requirements on the traits here
+            // TODO: add conversion operations between bytes and the value type
+        }
+        ;
+
+
+    /// Matches types providing valid and complete `arrow_traits` specialization.
+    /// @see `is_arrow_traits`, `arrow_traits`
+    template< class T >
+    concept has_arrow_type_traits =
+        requires { typename ::sparrow::arrow_traits<T>; }
+        and is_arrow_traits< ::sparrow::arrow_traits<T> >
+        ;
+
+    /// Matches any type which is one of the base C++ types supported or at least that provides an `arrow_traits` specialization.
+    template< class T >
+    concept any_arrow_type = is_arrow_base_type<T> or has_arrow_type_traits<T>;
+
+    /// @returns Arrow type id to use for a given C++ representation of that type.
+    /// @see `arrow_traits`
+    template< has_arrow_type_traits T >
+    constexpr
+    auto arrow_type_id() -> data_type
+    {
+        return arrow_traits<T>::type_id;
+    }
+
+    /// @returns Arrow type id to use for the type of a given object.
+    /// @see `arrow_traits`
+    template< has_arrow_type_traits T >
+    constexpr
+    auto arrow_type_id(const T&) -> data_type
+    {
+        return arrow_type_id<T>();
+    }
+
+
+    /// Binary layout type to use by default for the given C++ representation T of an arrow value.
+    template< has_arrow_type_traits T >
+    using default_layout_t = typename arrow_traits<T>::default_layout;
+
+
     // For now, a tiny wrapper around data_type
     // More data and functions to come
     class data_descriptor
@@ -63,5 +206,24 @@ namespace sparrow
 
         data_type m_id;
     };
+
+
+    namespace impl
+    {
+        template <class C, bool is_const>
+        struct get_inner_reference
+            : std::conditional<is_const, typename C::inner_const_reference, typename C::inner_reference>
+        {
+        };
+
+        template <class C, bool is_const>
+        using get_inner_reference_t = typename get_inner_reference<C, is_const>::type;
+    }  // namespace impl
+
+    template <class T>
+    concept layout_offset = std::same_as<T, std::int32_t> || std::same_as<T, std::int64_t>;
+
+
+
 }
 

include/sparrow/details/3rdparty/README.md

@@ -0,0 +1,7 @@
+# 3rd Party sources
+
+This directory contains 3rd-party source code that we decided not to handle through automatic dependency management for various reasons described below.
+Please prefer automatic dependency management if you can!
+
+Reasonning for:
+- `float16_t.hpp`: we need a `float16_t` type but currently cannot use C++23's standard definition as some toolchains cannot use the most recent versions of compilers supporting C++23. Hence we decided to use this header (modified with a note about where it was taken from) but only if building with C++<23