From fa0b71d1375d1e8438920379d370296389d038cc Mon Sep 17 00:00:00 2001 From: Nathaniel Rupprecht Date: Tue, 2 Apr 2024 21:36:36 -0400 Subject: [PATCH] Overhaul of storage engine - Use entry creators to control how entries are serialized into one or more pages. This will be how overflow pages work, though this feature is not yet implemented. - Entry payload serializers return the data to serialize as a sequence of bytes. - Introduced DatabaseEntry object as a way to retrieve data, even if it is split over multiple pages, i.e., uses overflow pages. - Lots of refactors to use the new storage system. --- CMakeLists.txt | 122 +++++----- README.md | 2 + applications/data-manager-example.cpp | 71 ++++-- applications/database-string-pk.example.cpp | 15 +- include/NeverSQL/data/Page.h | 2 +- include/NeverSQL/data/btree/BTree.h | 56 +++-- include/NeverSQL/data/btree/BTreeNodeMap.h | 41 +++- include/NeverSQL/data/btree/EntryCopier.h | 23 ++ include/NeverSQL/data/btree/EntryCreator.h | 151 +++++++++++++ .../NeverSQL/data/internals/DatabaseEntry.h | 33 +++ .../internals/DocumentPayloadSerializer.h | 46 ++++ .../data/internals/EntryPayloadSerializer.h | 22 ++ .../NeverSQL/data/internals/OverflowEntry.h | 38 ++++ .../NeverSQL/data/internals/SinglePageEntry.h | 33 +++ .../data/internals/SpanPayloadSerializer.h | 34 +++ include/NeverSQL/data/internals/Utility.h | 8 +- include/NeverSQL/database/DataManager.h | 18 +- include/NeverSQL/database/Query.h | 39 ++++ include/NeverSQL/utility/Defines.h | 1 - source/NeverSQL/data/btree/BTree.cpp | 212 ++++++++++++------ source/NeverSQL/data/btree/BTreeNodeMap.cpp | 127 ++++++++--- source/NeverSQL/data/btree/EntryCopier.cpp | 15 ++ source/NeverSQL/data/btree/EntryCreator.cpp | 84 +++++++ .../NeverSQL/data/internals/DatabaseEntry.cpp | 64 ++++++ .../internals/DocumentPayloadSerializer.cpp | 38 ++++ source/NeverSQL/database/DataManager.cpp | 82 ++----- source/NeverSQL/utility/PageDump.cpp | 35 ++- 27 files changed, 1114 insertions(+), 298 deletions(-) create mode 100644 include/NeverSQL/data/btree/EntryCopier.h create mode 100644 include/NeverSQL/data/btree/EntryCreator.h create mode 100644 include/NeverSQL/data/internals/DatabaseEntry.h create mode 100644 include/NeverSQL/data/internals/DocumentPayloadSerializer.h create mode 100644 include/NeverSQL/data/internals/EntryPayloadSerializer.h create mode 100644 include/NeverSQL/data/internals/OverflowEntry.h create mode 100644 include/NeverSQL/data/internals/SinglePageEntry.h create mode 100644 include/NeverSQL/data/internals/SpanPayloadSerializer.h create mode 100644 source/NeverSQL/data/btree/EntryCopier.cpp create mode 100644 source/NeverSQL/data/btree/EntryCreator.cpp create mode 100644 source/NeverSQL/data/internals/DatabaseEntry.cpp create mode 100644 source/NeverSQL/data/internals/DocumentPayloadSerializer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cfe71a..aeb1850 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,11 +3,11 @@ cmake_minimum_required(VERSION 3.14) include(cmake/prelude.cmake) project( - NeverSQL - VERSION 0.1.0 - DESCRIPTION "A small implementation of a no-sql database." - HOMEPAGE_URL "https://example.com/" - LANGUAGES CXX + NeverSQL + VERSION 0.1.0 + DESCRIPTION "A small implementation of a no-sql database." + HOMEPAGE_URL "https://example.com/" + LANGUAGES CXX ) set(CMAKE_CXX_STANDARD 20) @@ -16,10 +16,10 @@ set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20") if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - # Needed to get rid of a warning from chrono. - # https://stackoverflow.com/questions/76859275/error-compiling-a-cpp-containing-stdchrono-errorstatic-constexpr-unsigned-fra - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") -endif() + # Needed to get rid of a warning from chrono. + # https://stackoverflow.com/questions/76859275/error-compiling-a-cpp-containing-stdchrono-errorstatic-constexpr-unsigned-fra + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") +endif () include(cmake/project-is-top-level.cmake) include(cmake/variables.cmake) @@ -30,66 +30,72 @@ option(BUILD_NEVERSQL_APPLICATIONS ON) # ---- Declare library ---- -include (FetchContent) +include(FetchContent) +# Lightning logging library. FetchContent_Declare( Lightning GIT_REPOSITORY https://github.com/nrupprecht/Lightning.git - GIT_TAG c254ecc + GIT_TAG f81f107 ) FetchContent_MakeAvailable(Lightning) include_directories(${Lightning_SOURCE_DIR}/include) add_library( - NeverSQL_NeverSQL - # Source files. - source/NeverSQL/data/DataAccessLayer.cpp - source/NeverSQL/data/Document.cpp - source/NeverSQL/data/FreeList.cpp - source/NeverSQL/data/Page.cpp - source/NeverSQL/data/PageCache.cpp - source/NeverSQL/data/btree/BTree.cpp - source/NeverSQL/data/btree/BTreeNodeMap.cpp - source/NeverSQL/database/DataManager.cpp - source/NeverSQL/utility/HexDump.cpp - source/NeverSQL/utility/PageDump.cpp - source/NeverSQL/utility/DisplayTable.cpp - source/NeverSQL/recovery/WriteAheadLog.cpp + NeverSQL_NeverSQL + # Source files. + source/NeverSQL/data/DataAccessLayer.cpp + source/NeverSQL/data/Document.cpp + source/NeverSQL/data/FreeList.cpp + source/NeverSQL/data/Page.cpp + source/NeverSQL/data/PageCache.cpp + source/NeverSQL/data/btree/BTree.cpp + source/NeverSQL/data/btree/BTreeNodeMap.cpp + source/NeverSQL/data/btree/EntryCreator.cpp + source/NeverSQL/data/btree/EntryCopier.cpp + source/NeverSQL/data/internals/DatabaseEntry.cpp + source/NeverSQL/data/internals/DocumentPayloadSerializer.cpp + source/NeverSQL/database/DataManager.cpp + source/NeverSQL/recovery/WriteAheadLog.cpp + source/NeverSQL/utility/HexDump.cpp + source/NeverSQL/utility/PageDump.cpp + source/NeverSQL/utility/DisplayTable.cpp + ) add_library(NeverSQL::NeverSQL ALIAS NeverSQL_NeverSQL) include(GenerateExportHeader) generate_export_header( - NeverSQL_NeverSQL - BASE_NAME NeverSQL - EXPORT_FILE_NAME export/NeverSQL/NeverSQL_export.hpp - CUSTOM_CONTENT_FROM_VARIABLE pragma_suppress_c4251 + NeverSQL_NeverSQL + BASE_NAME NeverSQL + EXPORT_FILE_NAME export/NeverSQL/NeverSQL_export.hpp + CUSTOM_CONTENT_FROM_VARIABLE pragma_suppress_c4251 ) -if(NOT BUILD_SHARED_LIBS) - target_compile_definitions(NeverSQL_NeverSQL PUBLIC NEVERSQL_STATIC_DEFINE) -endif() +if (NOT BUILD_SHARED_LIBS) + target_compile_definitions(NeverSQL_NeverSQL PUBLIC NEVERSQL_STATIC_DEFINE) +endif () set_target_properties( - NeverSQL_NeverSQL PROPERTIES - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN YES - VERSION "${PROJECT_VERSION}" - SOVERSION "${PROJECT_VERSION_MAJOR}" - EXPORT_NAME NeverSQL - OUTPUT_NAME NeverSQL + NeverSQL_NeverSQL PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN YES + VERSION "${PROJECT_VERSION}" + SOVERSION "${PROJECT_VERSION_MAJOR}" + EXPORT_NAME NeverSQL + OUTPUT_NAME NeverSQL ) target_include_directories( - NeverSQL_NeverSQL ${warning_guard} - PUBLIC - "$" + NeverSQL_NeverSQL ${warning_guard} + PUBLIC + "$" ) target_include_directories( - NeverSQL_NeverSQL SYSTEM - PUBLIC - "$" + NeverSQL_NeverSQL SYSTEM + PUBLIC + "$" ) target_compile_features(NeverSQL_NeverSQL PUBLIC cxx_std_20) @@ -98,27 +104,27 @@ target_compile_features(NeverSQL_NeverSQL PUBLIC cxx_std_20) #if (BUILD_NEVERSQL_APPLICATIONS) - message("Building applications.") - add_subdirectory("${PROJECT_SOURCE_DIR}/applications") +message("Building applications.") +add_subdirectory("${PROJECT_SOURCE_DIR}/applications") #else() # message("Not building applications.") #endif() # ---- Install rules ---- -if(NOT CMAKE_SKIP_INSTALL_RULES) - include(cmake/install-rules.cmake) -endif() +if (NOT CMAKE_SKIP_INSTALL_RULES) + include(cmake/install-rules.cmake) +endif () # ---- Developer mode ---- -if(NOT NeverSQL_DEVELOPER_MODE) - return() -elseif(NOT PROJECT_IS_TOP_LEVEL) - message( - AUTHOR_WARNING - "Developer mode is intended for developers of NeverSQL" - ) -endif() +if (NOT NeverSQL_DEVELOPER_MODE) + return() +elseif (NOT PROJECT_IS_TOP_LEVEL) + message( + AUTHOR_WARNING + "Developer mode is intended for developers of NeverSQL" + ) +endif () include(cmake/dev-mode.cmake) diff --git a/README.md b/README.md index b06752b..b377045 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ Some useful resources on databases and database implementations: * Mongodb * https://github.com/mongodb/mongo * BSON spec: https://bsonspec.org/, https://bsonspec.org/spec.html +* WiredTiger + * https://source.wiredtiger.com/11.2.0/arch-index.html * Other tutorials / similar projects * https://cstack.github.io/db_tutorial/ * https://adambcomer.com/blog/simple-database/motivation-design/ diff --git a/applications/data-manager-example.cpp b/applications/data-manager-example.cpp index c357dad..d54782c 100644 --- a/applications/data-manager-example.cpp +++ b/applications/data-manager-example.cpp @@ -16,22 +16,22 @@ using namespace neversql; void SetupLogger(Severity min_severity = Severity::Info); int main() { + // 0.004311 ms per addition, 233,612 pages, 10,000,000 entries. SetupLogger(Severity::Info); // ---> Your database path here. - std::filesystem::path database_path = "path-to-your-directory"; + std::filesystem::path database_path = + "/Users/nathaniel/Documents/Nathaniel/Programs/C++/NeverSQL/database-dmgr-test"; + remove_all(database_path); - // std::filesystem::remove_all(database_path); + primary_key_t num_to_insert = 10'000'000; // 48 - primary_key_t num_to_insert = 100'000'000; + DataManager manager(database_path); - neversql::DataManager manager(database_path); + LOG_SEV(Info) << formatting::Format("Database has {:L} pages.", manager.GetDataAccessLayer().GetNumPages()); - LOG_SEV(Info) << lightning::formatting::Format("Database has {:L} pages.", - manager.GetDataAccessLayer().GetNumPages()); - - manager.AddCollection("elements", neversql::DataTypeEnum::UInt64); + manager.AddCollection("elements", DataTypeEnum::UInt64); primary_key_t pk = 0; auto starting_time_point = std::chrono::high_resolution_clock::now(); @@ -41,10 +41,10 @@ int main() { try { for (; pk < num_to_insert; ++pk) { // Create a document. - neversql::Document builder; - builder.AddElement("data", StringValue{formatting::Format("Brave new world.\nEntry number {}.", pk)}); - builder.AddElement("pk", IntegralValue{static_cast(pk)}); - builder.AddElement("is_even", BooleanValue{pk % 2 == 0}); + Document builder; + builder.AddElement("data", StringValue {formatting::Format("Brave new world.\nEntry number {}.", pk)}); + builder.AddElement("pk", IntegralValue {static_cast(pk)}); + builder.AddElement("is_even", BooleanValue {pk % 2 == 0}); // Add the document. manager.AddValue("elements", builder); @@ -79,8 +79,13 @@ int main() { LOG_SEV(Major) << lightning::formatting::Format("Database has {:L} pages.", total_pages); // Node dump the main index page. - manager.NodeDumpPage(2, std::cout); + manager.NodeDumpPage(3, std::cout); std::cout << std::endl; + try { + manager.NodeDumpPage(4, std::cout); + std::cout << std::endl; + } catch (...) { + } // Search for some elements. auto first_to_probe = num_to_insert / 2; @@ -88,29 +93,49 @@ int main() { for (primary_key_t pk_probe = first_to_probe; pk_probe < last_to_probe; ++pk_probe) { auto result = manager.Retrieve("elements", pk_probe); if (result.IsFound()) { - auto& view = result.value_view; + memory::MemoryBuffer buffer; + + auto& entry = *result.entry; + do { + auto data = entry.GetData(); + buffer.Append(data); + } while (entry.Advance()); + + auto view = std::span {buffer.Data(), buffer.Size()}; // Interpret the data as a document. - auto document = neversql::ReadDocumentFromBuffer(view); - - LOG_SEV(Info) << formatting::Format( - "Found key {:L} on page {:L}, search depth {}, value: \n{@BYELLOW}{}{@RESET}", - pk_probe, - result.search_result.node->GetPageNumber(), - result.search_result.GetSearchDepth(), - neversql::PrettyPrint(*document)); + if (auto document = neversql::ReadDocumentFromBuffer(view)) { + LOG_SEV(Info) << formatting::Format( + "Found key {:L} on page {:L}, search depth {}, value: \n{@BYELLOW}{}{@RESET}", + pk_probe, + result.search_result.node->GetPageNumber(), + result.search_result.GetSearchDepth(), + neversql::PrettyPrint(*document)); + } + else { + LOG_SEV(Error) << "Could not read document."; + } } else { LOG_SEV(Info) << formatting::Format("{@BRED}Key {} was not found.{@RESET}", pk_probe); } } + // auto it_begin = manager.Begin("elements"); + // auto end_it = manager.End("elements"); + // for (auto it = it_begin; it != end_it; ++it) { + // auto view = *it; + // // Interpret the data as a document. + // neversql::DocumentReader reader(view); + // LOG_SEV(Info) << formatting::Format("Value: \n{@BYELLOW}{}{@RESET}", neversql::PrettyPrint(reader)); + // } + return 0; } void SetupLogger(Severity min_severity) { auto console = lightning::NewSink(); - lightning::Global::GetCore()->AddSink(console); + Global::GetCore()->AddSink(console); console->SetFilter(min_severity <= LoggingSeverity); // Formatter for "low levels" of severity displace file and line number. diff --git a/applications/database-string-pk.example.cpp b/applications/database-string-pk.example.cpp index a4117e5..8370324 100644 --- a/applications/database-string-pk.example.cpp +++ b/applications/database-string-pk.example.cpp @@ -8,6 +8,7 @@ #include "NeverSQL/data/btree/BTree.h" #include "NeverSQL/data/internals/Utility.h" #include "NeverSQL/database/DataManager.h" +#include "NeverSQL/database/Query.h" #include "NeverSQL/utility/HexDump.h" #include "NeverSQL/utility/PageDump.h" @@ -20,7 +21,8 @@ int main() { SetupLogger(Severity::Info); // ---> Your database path here. - std::filesystem::path database_path = "your-path-here"; + std::filesystem::path database_path = + "/Users/nathaniel/Documents/Nathaniel/Programs/C++/NeverSQL/database-string"; std::filesystem::remove_all(database_path); @@ -106,7 +108,16 @@ int main() { auto result = manager.Retrieve("elements", neversql::internal::SpanValue(name)); if (result.IsFound()) { // Interpret the data as a document. - auto document = neversql::ReadDocumentFromBuffer(result.value_view); + + memory::MemoryBuffer buffer; + auto& entry = *result.entry; + do { + auto data = entry.GetData(); + buffer.Append(data); + } while (entry.Advance()); + auto view = std::span {buffer.Data(), buffer.Size()}; + + auto document = neversql::ReadDocumentFromBuffer(view); LOG_SEV(Info) << formatting::Format( "Found key {:?} on page {:L}, search depth {}, value: \n{@BYELLOW}{}{@RESET}", diff --git a/include/NeverSQL/data/Page.h b/include/NeverSQL/data/Page.h index 6e91c05..9725fbe 100644 --- a/include/NeverSQL/data/Page.h +++ b/include/NeverSQL/data/Page.h @@ -47,7 +47,7 @@ class Page { template requires std::is_trivially_copyable_v page_size_t WriteToPage(page_size_t offset, const T& data) { - std::span view(reinterpret_cast(&data), sizeof(T)); + std::span view(reinterpret_cast(&data), sizeof(T)); return WriteToPage(offset, view); } diff --git a/include/NeverSQL/data/btree/BTree.h b/include/NeverSQL/data/btree/BTree.h index e0d3a7a..2398395 100644 --- a/include/NeverSQL/data/btree/BTree.h +++ b/include/NeverSQL/data/btree/BTree.h @@ -13,8 +13,9 @@ #include "NeverSQL/containers/FixedStack.h" #include "NeverSQL/data/PageCache.h" #include "NeverSQL/data/btree/BTreeNodeMap.h" -#include "NeverSQL/data/internals/KeyComparison.h" +#include "NeverSQL/data/btree/EntryCreator.h" #include "NeverSQL/utility/DataTypes.h" +#include "NeverSQL/data/internals/DatabaseEntry.h" namespace neversql { @@ -28,22 +29,33 @@ using TreePosition = FixedStack>; //! Includes the search path (in pages) that was taken, along with the node that was found. struct SearchResult { TreePosition path; - std::optional node {}; + std::optional node; //! \brief Get how many layers had to be searched to find the node. std::size_t GetSearchDepth() const noexcept { return path.Size(); } + + bool IsFound() const noexcept { return node.has_value(); } +}; + +//! \brief Structure that represents data on retrieving data from the data manager. +struct RetrievalResult { + SearchResult search_result; + + std::unique_ptr entry; + + bool IsFound() const noexcept { return search_result.IsFound(); } }; //! \brief Convenient structure for packing up data to store in a B-tree. struct StoreData { //! \brief The key is some collection of bytes. It is context dependent how to compare different keys. - GeneralKey key {}; + GeneralKey key; - //! \brief The payload of the store operation. - std::span serialized_value {}; + //! \brief Entry creator, which knows how to store the data inside the tree. + std::unique_ptr entry_creator; - //! \brief Whether to serialize the size of the key. If false, it is assumed that all keys have a fixed sizes - //! that is known by the B-tree manager. + //! \brief Whether to serialize the size of the key. If false, it is assumed that all keys have a fixed + //! sizes that is known by the B-tree manager. //! bool serialize_key_size = false; @@ -74,14 +86,14 @@ class BTreeManager { static std::unique_ptr CreateNewBTree(PageCache& page_cache, DataTypeEnum key_type); //! \brief Add a value with a specified key to the BTree. - void AddValue(GeneralKey key, std::span value); + void AddValue(GeneralKey key, std::unique_ptr&& entry_creator); //! \brief Add a value with an auto-incrementing key to the B-tree. //! //! Only works if the B-tree is configured to generate auto-incrementing keys. //! //! \param value The value payload to add to the B-tree. - void AddValue(std::span value); + void AddValue(std::unique_ptr&& entry_creator); //! \brief Get the root page number of the B-tree. page_number_t GetRootPageNumber() const noexcept { return root_page_; } @@ -131,8 +143,15 @@ class BTreeManager { //! \brief Initialize the B-tree manager object from the data in its root page. void initialize(); + //! \brief Get the next primary key. primary_key_t getNextPrimaryKey() const; + //! \brief Get a new page on which overflow entries can be written. + page_number_t getNextOverflowPage(); + + //! \brief Get the current overflow page. + page_number_t getCurrentOverflowPage() const; + BTreeNodeMap newNodePage(BTreePageType type, page_size_t reserved_space) const; std::optional loadNodePage(page_number_t page_number) const; @@ -145,14 +164,14 @@ class BTreeManager { bool addElementToNode(BTreeNodeMap& node_map, const StoreData& data, bool unique_keys = true) const; //! \brief Split a node. This may, recursively, lead to more splits if the split causes the parent node to - //! be full. - void splitNode(BTreeNodeMap& node, SearchResult& result, std::optional data); + //! be full. + void splitNode(BTreeNodeMap& node, SearchResult& result, std::optional> data); //! \brief Split a single node, returning the key that was split on and the nodes. - SplitPage splitSingleNode(BTreeNodeMap& node, std::optional data); + SplitPage splitSingleNode(BTreeNodeMap& node, std::optional> data); //! \brief Special case for splitting the root node, which causes the height of the tree to increase by one. - void splitRoot(std::optional data); + void splitRoot(std::optional> data); //! \brief Vacuums the node, removing any fragmented space. void vacuum(BTreeNodeMap& node) const; @@ -160,6 +179,9 @@ class BTreeManager { //! \brief Look for the leaf node where a key should be inserted or can be found. SearchResult search(GeneralKey key) const; + //! \brief Try to retrieve data from a B-tree. + RetrievalResult retrieve(GeneralKey key) const; + //! \brief Checks if the key is less than or equal to the other key. //! //! Uses the lt comparison provided, uses std::ranges::equal to check if the keys are equal. @@ -170,7 +192,7 @@ class BTreeManager { //! Uses the debug_key_func_ if it is available, otherwise, string-ize the bytes. //! //! \param key The key to convert to a string. - //! \return string representation of the key, implementation defined. + //! \return Returns a string representation of the key, implementation defined. std::string debugKey(GeneralKey key) const; // ================================================================================================= @@ -184,6 +206,9 @@ class BTreeManager { //! page_number_t root_page_ {}; + //! \brief The current page available for overflow entries. Zero if no page is being used. + page_number_t current_overflow_page_number_ {}; + //! \brief Whether the key's size needs to be serialized. TODO: Get this from the key type. bool serialize_key_size_ = false; @@ -199,6 +224,9 @@ class BTreeManager { //! \brief The maximum entry size, in bytes, before an overflow page is needed page_size_t max_entry_size_ = 256; + //! \brief The minimum amount of space, in bytes, to have to allow an entry to be added to a page. + page_size_t min_space_for_entry_ = 128; + //! \brief The maximum number of entries per page. //! NOTE(Nate): I am adding this for now to make testing easier. page_size_t max_entries_per_page_ = 10000; diff --git a/include/NeverSQL/data/btree/BTreeNodeMap.h b/include/NeverSQL/data/btree/BTreeNodeMap.h index da36a36..9607408 100644 --- a/include/NeverSQL/data/btree/BTreeNodeMap.h +++ b/include/NeverSQL/data/btree/BTreeNodeMap.h @@ -8,6 +8,7 @@ #include "NeverSQL/data/Page.h" #include "NeverSQL/data/btree/BTreePageHeader.h" +#include "NeverSQL/data/internals/DatabaseEntry.h" #include "NeverSQL/data/internals/KeyPrinting.h" namespace neversql { @@ -24,9 +25,9 @@ using GeneralKey = std::span; //! \brief Helper structure that represents a cell in a leaf node. struct DataNodeCell { + std::byte flags; const std::span key; - const entry_size_t size_of_entry; - const std::byte* start_of_value; + std::span data; bool key_size_is_serialized = false; @@ -35,26 +36,41 @@ struct DataNodeCell { // ================================================================================================= //! \brief Get a span of the value in the cell. - NO_DISCARD std::span SpanValue() const noexcept { - return std::span(start_of_value, size_of_entry); - } + NO_DISCARD std::span SpanValue() const noexcept { return data; } - NO_DISCARD page_size_t GetSize() const noexcept { - return static_cast(key.size() + sizeof(entry_size_t) + size_of_entry + NO_DISCARD page_size_t GetCellSize() const noexcept { + return static_cast(key.size() + sizeof(entry_size_t) + data.size() + (key_size_is_serialized ? 2 : 0)); } + + NO_DISCARD page_size_t GetDataSize() const noexcept { return static_cast(data.size()); } }; //! \brief Helper structure that represents a cell in an internal node. struct PointersNodeCell { + std::byte flags; const std::span key; + + //! \brief The pointer value. + //! + //! \note The data of a pointers node cell (the entry) is just the page number. const page_number_t page_number; bool key_size_is_serialized = false; - NO_DISCARD page_size_t GetSize() const noexcept { + NO_DISCARD page_size_t GetCellSize() const noexcept { return static_cast(key.size() + sizeof(page_number_t) + (key_size_is_serialized ? 2 : 0)); } + + NO_DISCARD static page_size_t GetDataSize() noexcept { return sizeof(page_number_t); } +}; + +struct SpaceRequirement { + page_size_t pointer_space; + + page_size_t cell_header_space; + + page_size_t max_entry_space; }; namespace utility { @@ -96,6 +112,12 @@ class BTreeNodeMap { //! \return The amount of free space in the node, in the free space section. NO_DISCARD page_size_t GetDefragmentedFreeSpace() const; + //! \brief Calculate the space requirements for adding a new entry to a node, given the key. + //! + //! This calculates the amount of space needed for the pointer, the cell, and the maximum amount of space + //! available for the entry. + NO_DISCARD SpaceRequirement CalculateSpaceRequirements(GeneralKey key) const; + //! \brief Get the largest key of any element in the node. If there are no keys, returns nullopt. NO_DISCARD std::optional GetLargestKey() const; @@ -143,6 +165,9 @@ class BTreeNodeMap { //! \brief Get a span of the offsets in the node. std::span getPointers() const; + //! \brief Get the offset to the start of the free space in the node. + page_size_t getCellOffsetByIndex(page_size_t cell_index) const; + //! \brief Get the primary key from a cell, given the cell offset. GeneralKey getKeyForCell(page_size_t cell_offset) const; diff --git a/include/NeverSQL/data/btree/EntryCopier.h b/include/NeverSQL/data/btree/EntryCopier.h new file mode 100644 index 0000000..6694e06 --- /dev/null +++ b/include/NeverSQL/data/btree/EntryCopier.h @@ -0,0 +1,23 @@ +// +// Created by Nathaniel Rupprecht on 3/30/24. +// + +#pragma once + +#include "NeverSQL/data/btree/EntryCreator.h" + +namespace neversql::internal { + +//! \brief Entry copier that copies the payload, no matter what type of payload it is (single page entry or +//! overflow page header / entry). +class EntryCopier : public EntryCreator { +public: + EntryCopier(std::byte flags, std::span payload); + + //! \brief Return the stored flags + std::byte GenerateFlags() const override { return flags_; } +private: + std::byte flags_; +}; + +} // namespace neversql::internal \ No newline at end of file diff --git a/include/NeverSQL/data/btree/EntryCreator.h b/include/NeverSQL/data/btree/EntryCreator.h new file mode 100644 index 0000000..0329f05 --- /dev/null +++ b/include/NeverSQL/data/btree/EntryCreator.h @@ -0,0 +1,151 @@ +// +// Created by Nathaniel Rupprecht on 3/26/24. +// + +#pragma once + +#include "NeverSQL/data/internals/EntryPayloadSerializer.h" +#include "NeverSQL/utility/Defines.h" + +namespace neversql { +class Page; +class BTreeManager; +} // namespace neversql + +namespace neversql::internal { + +enum EntryFlags : uint8_t { + IsActive = 0b1000'0000, + KeySizeIsSerialized = 0b0100'0000, + // ... + NoteFlag = 0b0010, + IsSinglePageEntry = 0b0001 +}; + +inline bool GetIsActive(std::byte flags) { + return (static_cast(flags) & static_cast(EntryFlags::IsActive)) != 0; +} + +inline bool GetKeySizeIsSerialized(std::byte flags) { + return (static_cast(flags) & static_cast(EntryFlags::KeySizeIsSerialized)) != 0; +} + +inline bool IsNoteFlagTrue(std::byte flags) { + return (static_cast(flags) & static_cast(EntryFlags::NoteFlag)) != 0; +} + +inline bool GetIsSinglePageEntry(std::byte flags) { + return (static_cast(flags) & static_cast(EntryFlags::IsSinglePageEntry)) != 0; +} + +inline bool GetNextOverflowPageIsPresent(std::byte flags) { + return IsNoteFlagTrue(flags) && !GetIsSinglePageEntry(flags); +} + +inline bool GetIsEntrySizeSerialized(std::byte flags) { + return IsNoteFlagTrue(flags) && GetIsSinglePageEntry(flags); +} + +//! \brief Object that knows how to create entries inside a B-tree, or read B-tree entries to create a +//! DatabaseEntry. +//! +//! An EntryCreator will be created to create a data payload in a B-tree for some object, like a document, +//! that needs to be stored in the database. The EntryCreator may need to request that the B-tree create +//! overflow pages for it, or at least notify it of the current overflow page number. +//! +//! Single page entry layout (data cell): +// clang-format off +//! [flags: 1 byte] [key_size: 2 bytes]? [key: 8 bytes | variable] + [entry_size: 2 bytes] [entry_data: entry_size bytes] +// clang-format on +//! \note Whether the entry is a single page or overflow page entry is determined by the flags byte. +//! +//! Start of overflow entry layout, "overflow header" (data cell): +// clang-format off +//! [flags: 1 byte] [key_size: 2 bytes]? [key: 8 bytes | variable] + [overflow_key: 8 bytes] [overflow page number: 8 bytes] +// clang-format on +//! \note An overflow entry, when the key size is not serialized, takes at least 19 bytes. +//! When the key is serialized, it takes at least 21 bytes. The entry size is 16 bytes. +//! +//! Overflow entry continuation layout (data cell): +// clang-format off +//! [flags: 1 byte] [overflow page number: 8 bytes] + [next overflow page number: 8 bytes]? [entry_size: 2 bytes]? [entry_data: entry_size bytes] +// clang-format on +//! \note Whether the next overflow page is present is determined by the flags byte. +//! \note Whether the entry size is serialized is determined by the flags byte. +//! +//! Tombstone entry layout (data cell, not created by an EntryCreator, but listed here for now): +// clang-format off +//! [flags: 1 byte] [cell_size: 2 bytes] [cell contents (freed space): cell_size bytes] +// clang-format on +//! +//! \note: The B-tree is responsible for creating the cell parts before the "+" (i.e., the key related parts), +//! except for (part of) the flags, while the entry creator creates (part of the) flags and the control fields +//! and data payload for the rest of the cell. For tombstone cells, the flags are used to indicate that the +//! cell has been freed +//! +//! Flags: +//! 0b DK00 00NT +//! * D: Deleted flag: 1 if the entry is active, 0 if it is a tombstone (deleted space). +//! * K: Key flag: 1 if the key size is serialized, 0 if the key size is not serialized. +//! ... unused flags... +//! * N: Note flag: Depends on whether the entry is a single page entry or an overflow entry (currently not +//! used for single page entries): +//! * If T == 0: 1 if the next overflow page is present, 0 if the next overflow page is not present. +//! * If T == 1: 1 if the entry size is serialized, 0 if the entry size is not serialized. +//! * T: Type flag: 1 if the entry is a single page entry, 0 if the entry is an overflow entry. +//! \note The B-tree is responsible for setting the D and K flags, while the EntryCreator is responsible for +//! setting the N and T flags. +//! +class EntryCreator { +public: + virtual ~EntryCreator() = default; + + explicit EntryCreator(std::unique_ptr&& payload, bool serialize_size = true); + + //! \brief The minimum amount of space that the part of an entry that the EntryCreator creates can take up + //! in a page. + page_size_t GetMinimumEntrySize() const; + + //! \brief Get how much space the EntryCreator wants in the initial page. + //! + //! The EntryCreator may change its internal stage, e.g., store the amount of space it decided on, or store + //! whether an overflow page is necessary, when this function is called. + page_size_t GetRequestedSize(page_size_t maximum_entry_size); + + //! \brief Generate the EntryCreator's part of the flags. + //! + //! Should be called after GetRequiredSize. + virtual std::byte GenerateFlags() const; + + //! \brief Create an entry, starting with the given offset in the page. + //! + //! \return Returns the offset to the place after the entry, in the original page (even if an overflow page + //! was created and data was added to other pages. + page_size_t Create(page_size_t starting_offset, Page* page, const BTreeManager* btree_manager); + + //! \brief Return whether the EntryCreator is going to create overflow pages. + bool GetNeedsOverflow() const noexcept { return overflow_page_needed_; } + +protected: + void createOverflowEntry(page_size_t starting_offset, Page* page, const BTreeManager* btree_manager); + page_size_t createSinglePageEntry(page_size_t starting_offset, Page* page); + + bool overflow_page_needed_ = false; + bool serialize_size_ = true; + + std::unique_ptr payload_; +}; + +//! \brief Create an entry creator with a payload of type Payload_t. +template +std::unique_ptr MakeCreator(Args_t&&... args) { + return std::make_unique(std::make_unique(std::forward(args)...)); +} + +//! \brief Create an entry creator with a payload of type Payload_t that does not serialize the entry size. +template +std::unique_ptr MakeSizelessCreator(Args_t&&... args) { + return std::make_unique(std::make_unique(std::forward(args)...), false); +} + +} // namespace neversql::internal \ No newline at end of file diff --git a/include/NeverSQL/data/internals/DatabaseEntry.h b/include/NeverSQL/data/internals/DatabaseEntry.h new file mode 100644 index 0000000..a7ee417 --- /dev/null +++ b/include/NeverSQL/data/internals/DatabaseEntry.h @@ -0,0 +1,33 @@ +// +// Created by Nathaniel Rupprecht on 3/26/24. +// + +#pragma once + +#include "NeverSQL/data/Page.h" + +namespace neversql { +class BTreeManager; +} + +namespace neversql::internal { + +//! \brief An object that allow for access of the data payload of an entry in a B-tree, abstracting away the +//! exact layout of the data (e.g. whether it is stored in a leaf node or in an overflow page). +class DatabaseEntry { +public: + //! \brief Get the entry data in the current focus of the entry. + virtual std::span GetData() const noexcept = 0; + + //! \brief Go to the next part of the database entry. Returns true if there was another entry to go to. + virtual bool Advance() = 0; + + virtual ~DatabaseEntry() = default; +}; + +//! \brief Read an entry, starting with the given offset in the page. +std::unique_ptr ReadEntry(page_size_t starting_offset, + const Page* page, + const BTreeManager* btree_manager); + +} // namespace neversql::internal \ No newline at end of file diff --git a/include/NeverSQL/data/internals/DocumentPayloadSerializer.h b/include/NeverSQL/data/internals/DocumentPayloadSerializer.h new file mode 100644 index 0000000..1e2885d --- /dev/null +++ b/include/NeverSQL/data/internals/DocumentPayloadSerializer.h @@ -0,0 +1,46 @@ +// +// Created by Nathaniel Rupprecht on 3/27/24. +// + +#pragma once + +#include "NeverSQL/data/Document.h" +#include "NeverSQL/data/internals/EntryPayloadSerializer.h" + +namespace neversql::internal { + +//! \brief An entry creator that will serialize a document into its entry. +//! +//! \note This is currently a naieve implementation, which just serializes the document up front to a buffer. +//! This is not ideal, as it means that the entire document must be in memory at once. This is not +//! scalable for large documents. A better implementation would be to serialize the document in chunks +//! as it is being written to disk. +class DocumentPayloadSerializer final : public EntryPayloadSerializer { +public: + explicit DocumentPayloadSerializer(std::unique_ptr document) + : document_(std::move(document)) { + initialize(); + } + + explicit DocumentPayloadSerializer(const Document& document) + : document_(&document) { + initialize(); + } + + bool HasData() override; + std::byte GetNextByte() override; + std::size_t GetRequiredSize() const override; + +private: + void initialize(); + const Document& getDocument() const; + + //! \brief The document to be stored, can be owned or not. + std::variant, const Document*> document_; + + std::size_t current_index_ = 0; + + lightning::memory::MemoryBuffer buffer_; +}; + +} // namespace neversql::internal diff --git a/include/NeverSQL/data/internals/EntryPayloadSerializer.h b/include/NeverSQL/data/internals/EntryPayloadSerializer.h new file mode 100644 index 0000000..fa3bc44 --- /dev/null +++ b/include/NeverSQL/data/internals/EntryPayloadSerializer.h @@ -0,0 +1,22 @@ +// +// Created by Nathaniel Rupprecht on 3/28/24. +// + +#pragma once + +#include + +namespace neversql::internal { + +//! \brief Base class for objects that act as byte generators for entry payloads. They serialize whatever the +//! entry payload is into bytes. +class EntryPayloadSerializer { +public: + virtual ~EntryPayloadSerializer() = default; + + virtual bool HasData() = 0; + virtual std::byte GetNextByte() = 0; + virtual std::size_t GetRequiredSize() const = 0; +}; + +} // namespace neversql::internal \ No newline at end of file diff --git a/include/NeverSQL/data/internals/OverflowEntry.h b/include/NeverSQL/data/internals/OverflowEntry.h new file mode 100644 index 0000000..7dfc3d6 --- /dev/null +++ b/include/NeverSQL/data/internals/OverflowEntry.h @@ -0,0 +1,38 @@ +// +// Created by Nathaniel Rupprecht on 3/26/24. +// + +#pragma once + +#include "NeverSQL/data/internals/DatabaseEntry.h" + +namespace neversql::internal { + + +//! \brief Represents an entry that is stored across one or more overflow pages. +//! +class OverflowEntry : public DatabaseEntry { +public: + OverflowEntry(std::span entry_header, + const BTreeManager* btree_manager) : btree_manager_(btree_manager) { + // Get information from the header, get the first overflow page. + // TODO: Implement. + } + + std::span GetData() const noexcept override { + // TODO: Implement. + return {}; + } + + bool Advance() override { + // TODO: Implement. + return false; + } + +private: + std::span data_; + + const BTreeManager* btree_manager_; +}; + +} // namespace neversql::internal \ No newline at end of file diff --git a/include/NeverSQL/data/internals/SinglePageEntry.h b/include/NeverSQL/data/internals/SinglePageEntry.h new file mode 100644 index 0000000..c2e75a4 --- /dev/null +++ b/include/NeverSQL/data/internals/SinglePageEntry.h @@ -0,0 +1,33 @@ +// +// Created by Nathaniel Rupprecht on 3/28/24. +// + +#pragma once + +#include "NeverSQL/data/internals/DatabaseEntry.h" + +namespace neversql::internal { + +class SinglePageEntry : public DatabaseEntry { +public: + SinglePageEntry(page_size_t starting_offset, const Page* page) + : starting_offset_(starting_offset) + , page_(page) { + entry_size_ = page->Read(starting_offset); + } + + //! \brief Get the data. All the data is on the same page. + std::span GetData() const noexcept override { + return page_->ReadFromPage(starting_offset_ + 2, entry_size_); + } + + //! \brief There is no further page to advance to. + bool Advance() override { return false; } + +private: + page_size_t starting_offset_; + page_size_t entry_size_; + const Page* page_; +}; + +} // namespace neversql::internal \ No newline at end of file diff --git a/include/NeverSQL/data/internals/SpanPayloadSerializer.h b/include/NeverSQL/data/internals/SpanPayloadSerializer.h new file mode 100644 index 0000000..7b74d2b --- /dev/null +++ b/include/NeverSQL/data/internals/SpanPayloadSerializer.h @@ -0,0 +1,34 @@ +// +// Created by Nathaniel Rupprecht on 3/28/24. +// + +#pragma once + +#include "NeverSQL/data/internals/DatabaseEntry.h" + +namespace neversql::internal { + +//! \brief A payload serializer that serializes a span of data, without any additional information or +//! treatments based on the type of the data. +class SpanPayloadSerializer final : public EntryPayloadSerializer { +public: + explicit SpanPayloadSerializer(std::span data) + : data_(data) {} + + bool HasData() override { return current_index_ < data_.size(); } + + std::byte GetNextByte() override { + if (HasData()) { + return data_[current_index_++]; + } + return {}; + } + + std::size_t GetRequiredSize() const override { return data_.size(); } + +private: + std::span data_; + std::size_t current_index_ = 0; +}; + +} // namespace neversql::internal diff --git a/include/NeverSQL/data/internals/Utility.h b/include/NeverSQL/data/internals/Utility.h index fc1a558..f82e863 100644 --- a/include/NeverSQL/data/internals/Utility.h +++ b/include/NeverSQL/data/internals/Utility.h @@ -8,13 +8,13 @@ namespace neversql::internal { -template requires std::is_trivially_copyable_v -std::span SpanValue(const T& value) noexcept { - return std::span(reinterpret_cast(&value), sizeof(T)); +template requires std::is_trivially_copyable_v +std::span SpanValue(const Value_t& value) noexcept { + return std::span(reinterpret_cast(&value), sizeof(Value_t)); } inline std::span SpanValue(const std::string& value) noexcept { - return std::span(reinterpret_cast(value.data()), value.size()); + return std::span(reinterpret_cast(value.data()), value.size()); } } // namespace neversql::internal diff --git a/include/NeverSQL/database/DataManager.h b/include/NeverSQL/database/DataManager.h index 25d32b8..4c35754 100644 --- a/include/NeverSQL/database/DataManager.h +++ b/include/NeverSQL/database/DataManager.h @@ -7,20 +7,10 @@ #include "NeverSQL/data/Document.h" #include "NeverSQL/data/PageCache.h" #include "NeverSQL/data/btree/BTree.h" -#include "NeverSQL/database/Query.h" #include "NeverSQL/utility/HexDump.h" namespace neversql { -//! \brief Structure that represents data on retrieving data from the data manager. -struct RetrievalResult { - SearchResult search_result; - page_size_t cell_offset {}; - std::span value_view; - - bool IsFound() const noexcept { return search_result.node.has_value(); } -}; - //! \brief Object that manages the data in the database, e.g. setting up B-trees and indices within the //! database. class DataManager { @@ -34,7 +24,7 @@ class DataManager { // General key methods // ======================================== - void AddValue(const std::string& collection_name, GeneralKey key, std::span value); + // void AddValue(const std::string& collection_name, GeneralKey key, std::span value); void AddValue(const std::string& collection_name, GeneralKey key, const Document& document); @@ -49,10 +39,10 @@ class DataManager { // ======================================== //! \brief Add a value to the database. - void AddValue(const std::string& collection_name, primary_key_t key, std::span value); + // void AddValue(const std::string& collection_name, primary_key_t key, std::span value); //! \brief Add a value to the database using an auto incrementing key. - void AddValue(const std::string& collection_name, std::span value); + // void AddValue(const std::string& collection_name, std::span value); //! \brief Add a document to the database. void AddValue(const std::string& collection_name, const Document& document); @@ -90,8 +80,10 @@ class DataManager { //! \brief The data access layer for the database. DataAccessLayer data_access_layer_; + //! \brief The page cache for the database. mutable PageCache page_cache_; + //! \brief Cache the collection index. std::unique_ptr collection_index_; //! \brief Cache the collections that are in the database. diff --git a/include/NeverSQL/database/Query.h b/include/NeverSQL/database/Query.h index a8c442e..b8c7ebf 100644 --- a/include/NeverSQL/database/Query.h +++ b/include/NeverSQL/database/Query.h @@ -76,6 +76,45 @@ using GreaterThan = Comparison>; template using GreaterEqual = Comparison>; +//! \brief A condition that a document has a field of a certain name. Optionally, the type of the field can be +//! checked as well. +class HasField : public Condition { + friend class lightning::ImplBase; + +protected: + class Impl : public Condition::Impl { + public: + explicit Impl(std::string field_name, std::optional type = {}) + : field_name_(std::move(field_name)) + , type_(type) {} + + bool Test(const Document& document) const override { + if (auto field = document.GetElement(field_name_)) { + if (type_) { + return field->get().GetDataType() == *type_; + } + return true; + } + return false; + } + + std::shared_ptr Copy() const override { + return std::make_shared(field_name_, type_); + } + + private: + std::string field_name_; + std::optional type_; + }; + + explicit HasField(const std::shared_ptr& impl) + : Condition(impl) {} + +public: + explicit HasField(const std::string& field_name, std::optional type = {}) + : Condition(std::make_shared(field_name, type)) {} +}; + //! \brief A query iterator. This wraps an ordinary BTreeManager::Iterator and filters the results based on a //! condition. This allows us to iterate though a collection, only counting documents that meet a //! certain condition. diff --git a/include/NeverSQL/utility/Defines.h b/include/NeverSQL/utility/Defines.h index 8ffd973..a024911 100644 --- a/include/NeverSQL/utility/Defines.h +++ b/include/NeverSQL/utility/Defines.h @@ -5,7 +5,6 @@ #pragma once #include -#include // std::byte #include #include diff --git a/source/NeverSQL/data/btree/BTree.cpp b/source/NeverSQL/data/btree/BTree.cpp index 5332392..d588e22 100644 --- a/source/NeverSQL/data/btree/BTree.cpp +++ b/source/NeverSQL/data/btree/BTree.cpp @@ -4,8 +4,11 @@ #include "NeverSQL/data/btree/BTree.h" // Other files. +#include "NeverSQL/data/btree/EntryCopier.h" +#include "NeverSQL/data/internals/DatabaseEntry.h" #include "NeverSQL/data/internals/KeyComparison.h" #include "NeverSQL/data/internals/KeyPrinting.h" +#include "NeverSQL/data/internals/SpanPayloadSerializer.h" #include "NeverSQL/data/internals/Utility.h" namespace neversql { @@ -148,7 +151,7 @@ std::unique_ptr BTreeManager::CreateNewBTree(PageCache& page_cache // TODO: Right now, we only support string and uint64_t keys, and so assume that key size is specified if // the key type is a string. if (key_type == DataTypeEnum::String) { - auto flags = header.GetFlags(); + const auto flags = header.GetFlags(); header.SetFlags(flags | 0b100); } @@ -159,13 +162,13 @@ std::unique_ptr BTreeManager::CreateNewBTree(PageCache& page_cache offset = root_node.GetPage().WriteToPage(offset, static_cast(key_type)); offset = root_node.GetPage().WriteToPage(offset, 0); if (key_type == DataTypeEnum::UInt64) { - offset = root_node.GetPage().WriteToPage(offset, 0); + root_node.GetPage().WriteToPage(offset, 0); } return std::make_unique(root_node.GetPageNumber(), page_cache); } -void BTreeManager::AddValue(GeneralKey key, std::span value) { +void BTreeManager::AddValue(GeneralKey key, std::unique_ptr&& entry_creator) { LOG_SEV(Debug) << "Adding value with key " << debugKey(key) << " to the B-tree."; // Search for the leaf node where the key should be inserted. @@ -190,7 +193,7 @@ void BTreeManager::AddValue(GeneralKey key, std::span value) { // TODO: More complex strategies could include vacuuming, looking for fragmented free space, etc. auto space_available = result.node->GetDefragmentedFreeSpace(); // Cell offset, entry size, and the entry itself. - auto necessary_space = sizeof(page_size_t) + sizeof(entry_size_t) + value.size(); + auto necessary_space = sizeof(page_size_t) + sizeof(entry_size_t) + entry_creator->GetMinimumEntrySize(); // Space required for the key. if (serialize_key_size_) { necessary_space += sizeof(uint16_t); @@ -202,11 +205,14 @@ void BTreeManager::AddValue(GeneralKey key, std::span value) { << " bytes. Number of elements is " << num_elements << ". Total size of this entry is " << necessary_space << " bytes."; - if (necessary_space <= space_available && num_elements + 1 <= max_entries_per_page_) { + // We must have at least `space_available` space to add an entry to this page. + if (min_space_for_entry_ <= space_available && necessary_space <= space_available + && num_elements + 1 <= max_entries_per_page_) + { // TODO: Return expected type, or some more detailed info, generally, this will fail b/c of key // uniqueness violations. StoreData store_data {.key = key, - .serialized_value = value, + .entry_creator = std::move(entry_creator), .serialize_key_size = serialize_key_size_, .serialize_data_size = true}; NOSQL_ASSERT(addElementToNode(*result.node, store_data), @@ -217,7 +223,8 @@ void BTreeManager::AddValue(GeneralKey key, std::span value) { // Else, we have to split the node and re-balance the tree. LOG_SEV(Trace) << "Not enough free space, node " << result.node->GetPageNumber() << " must be split."; - splitNode(*result.node, result, StoreData {.key = key, .serialized_value = value}); + StoreData store_data {.key = key, .entry_creator = std::move(entry_creator)}; + splitNode(*result.node, result, store_data); // Sanity check. auto&& header = result.node->GetHeader(); @@ -227,7 +234,7 @@ void BTreeManager::AddValue(GeneralKey key, std::span value) { } } -void BTreeManager::AddValue(std::span value) { +void BTreeManager::AddValue(std::unique_ptr&& entry_creator) { NOSQL_REQUIRE(key_type_ == DataTypeEnum::UInt64, "cannot add value with auto-incrementing key to B-tree with non-uint64_t key type"); @@ -238,7 +245,7 @@ void BTreeManager::AddValue(std::span value) { GeneralKey key_span(reinterpret_cast(&next_key), sizeof(next_key)); // Add the value with the next primary key. - AddValue(key_span, value); + AddValue(key_span, std::move(entry_creator)); } void BTreeManager::initialize() { @@ -281,6 +288,25 @@ primary_key_t BTreeManager::getNextPrimaryKey() const { return pk; } +page_number_t BTreeManager::getNextOverflowPage() { + auto root = loadNodePage(root_page_); + + // Get a new page. + const auto new_page = newNodePage(BTreePageType::Leaf, 0); + // TODO: Any setup needed for the page to be an overflow page? Set some flags? + + current_overflow_page_number_ = new_page.GetPageNumber(); + const auto counter_offset = + static_cast(root->GetHeader().GetReservedStart() + 2 + sizeof(primary_key_t)); + root->GetPage().WriteToPage(counter_offset, current_overflow_page_number_); + + return current_overflow_page_number_; +} + +page_number_t BTreeManager::getCurrentOverflowPage() const { + return current_overflow_page_number_; +} + BTreeNodeMap BTreeManager::newNodePage(BTreePageType type, page_size_t reserved_space) const { BTreeNodeMap node(page_cache_.GetNewPage()); // Set the comparison and string debug functions. These are a B-tree property, not stored per-page. @@ -318,8 +344,9 @@ std::optional BTreeManager::loadNodePage(page_number_t page_number bool BTreeManager::addElementToNode(BTreeNodeMap& node_map, const StoreData& data, bool unique_keys) const { auto header = node_map.GetHeader(); LOG_SEV(Debug) << "Adding element with pk = " << debugKey(data.key) << " to page " - << node_map.GetPageNumber() << ", data size is " << data.serialized_value.size() - << " bytes, unique-keys = " << unique_keys << "."; + << node_map.GetPageNumber() << ", unique-keys = " << unique_keys << "."; + + auto& entry_creator = *data.entry_creator; // ======================================= // Check if there is enough free space to add the data. @@ -327,10 +354,11 @@ bool BTreeManager::addElementToNode(BTreeNodeMap& node_map, const StoreData& dat // ============ Pointer space ============ // Offset to value: sizeof(page_size_t) // ============ Cell space ============ - // [Potentially size of key: sizeof(uint16_t)] - // Primary key, either fixed size (sizeof(primary_key_t)) or variable size. - // [Potentially size of value: sizeof(entry_size_t), if store_size is true] - // Value: serialized_value.size() + // [flags: 1 byte] + // [Key size: 2 bytes]? [Key: 8 bytes | variable] + // ---------------- Entry ---------------- + // [Entry size: 2 bytes] + // [Entry data: Entry size bytes] // ======================================= if (unique_keys) { @@ -338,7 +366,7 @@ bool BTreeManager::addElementToNode(BTreeNodeMap& node_map, const StoreData& dat // If the key is already in the node, we cannot add it again. auto cell = node_map.getCell(lower_bound->first); bool found_key = - std::visit([data](auto&& cell) { return std::ranges::equal(cell.key, data.key); }, cell); + std::visit([&data](auto&& cell) { return std::ranges::equal(cell.key, data.key); }, cell); if (found_key) { LOG_SEV(Trace) << "Key " << debugKey(data.key) << " already in node on page " << header.GetPageNumber() << "."; @@ -352,20 +380,26 @@ bool BTreeManager::addElementToNode(BTreeNodeMap& node_map, const StoreData& dat auto& page = node_map.GetPage(); - // TODO: If we allow for overflow pages, this needs to change. - auto pointer_space = sizeof(page_size_t); - auto cell_space = - data.key.size() + (data.serialize_data_size ? sizeof(entry_size_t) : 0) + data.serialized_value.size(); - if (header.AreKeySizesSpecified()) { - cell_space += sizeof(uint16_t); - } + // Given the current free space and the space needed for the pointer and the other parts of the cell, what + // is the maximum amount of space available for the entry (not counting any page entry space restrictions). + auto space_requirements = node_map.CalculateSpaceRequirements(data.key); + auto maximum_available_space_for_entry = space_requirements.max_entry_space; + + auto maximum_entry_size = std::min(max_entry_size_, maximum_available_space_for_entry); + + auto entry_size = entry_creator.GetRequestedSize(maximum_entry_size); + + LOG_SEV(Trace) << "Entry creator requested " << entry_size + << " bytes of space, maximum available space was " << maximum_available_space_for_entry + << ". Will use overflow page: " << entry_creator.GetNeedsOverflow() << "."; + auto pointer_space = space_requirements.pointer_space; + auto cell_space = space_requirements.cell_header_space + entry_size; + auto required_space = pointer_space + cell_space; LOG_SEV(Trace) << "Entry will take up " << pointer_space << " bytes of pointer space and " << cell_space << " bytes of cell space, for a total of " << required_space << " bytes."; - // Check whether we would need an overflow page. - // TODO: Implement overflow pages. - + // Sanity check. if (auto defragmented_space = header.GetDefragmentedFreeSpace(); defragmented_space < required_space) { LOG_SEV(Trace) << "Not enough space to add element to node " << node_map.GetPageNumber() << ", required space was " << required_space << ", defragmented space was " @@ -379,12 +413,21 @@ bool BTreeManager::addElementToNode(BTreeNodeMap& node_map, const StoreData& dat auto entry_start_offset = entry_end_offset - cell_space; auto offset = static_cast(entry_start_offset); + LOG_SEV(Trace) << "Starting to write cell at offset " << offset << "."; // ======================================= - // Cell layout (leaf cell): - // [primary-key][size-of-value][value] + // B-tree responsible part. // ======================================= + // Write the flags. + auto flags = entry_creator.GenerateFlags(); + // Flags the the B-tree is responsible for. + flags |= static_cast(internal::EntryFlags::IsActive); + if (header.AreKeySizesSpecified()) { + flags |= static_cast(internal::EntryFlags::KeySizeIsSerialized); + } + offset = page.WriteToPage(offset, flags); + // If the page is set up to use fixed primary_key_t length keys, just write the key. if (header.AreKeySizesSpecified()) { // Write the size of the key, key size is stored as a uint16_t. @@ -393,18 +436,11 @@ bool BTreeManager::addElementToNode(BTreeNodeMap& node_map, const StoreData& dat } offset = page.WriteToPage(offset, data.key); - if (data.serialize_data_size) { - // Write the size of the value. - // TODO: What to do if we need an overflow page. This obviously only works as is if we are storing the - // whole entry here. - auto data_size = static_cast(data.serialized_value.size()); - offset = page.WriteToPage(offset, data_size); - } - - offset = page.WriteToPage(offset, data.serialized_value); + // Ask the EntryCreator to create the entry itself. + LOG_SEV(Trace) << "Creating entry at offset " << offset << " on page " << page.GetPageNumber() << "."; + offset = entry_creator.Create(offset, &page, this); // Make sure we wrote the correct amount of data. - NOSQL_ASSERT(offset == entry_end_offset, "incorrect amount of data written to cell in node " << header.GetPageNumber() << ", expected " << cell_space << " bytes, wrote " @@ -426,7 +462,9 @@ bool BTreeManager::addElementToNode(BTreeNodeMap& node_map, const StoreData& dat return true; } -void BTreeManager::splitNode(BTreeNodeMap& node, SearchResult& result, std::optional data) { +void BTreeManager::splitNode(BTreeNodeMap& node, + SearchResult& result, + std::optional> data) { LOG_SEV(Debug) << "Splitting node on page " << node.GetPageNumber() << "."; if (node.GetHeader().IsRootPage()) { LOG_SEV(Trace) << " * Splitting root node."; @@ -449,12 +487,22 @@ void BTreeManager::splitNode(BTreeNodeMap& node, SearchResult& result, std::opti NOSQL_ASSERT(parent, "could not find parent node"); // Create the data store specification. + StoreData store_data {.key = split_data.split_key, - .serialized_value = internal::SpanValue(split_data.left_page), + .entry_creator = internal::MakeSizelessCreator( + internal::SpanValue(split_data.left_page)), .serialize_key_size = serialize_key_size_, .serialize_data_size = false}; - if (!addElementToNode(*parent, store_data)) { + // Need to make sure there is enough space in the parent node. + auto space_requirements = parent->CalculateSpaceRequirements(store_data.key); + auto maximum_entry_size = std::min(max_entry_size_, space_requirements.max_entry_space); + if (maximum_entry_size < store_data.entry_creator->GetMinimumEntrySize()) { + // If the entry is too large to fit in the parent, we have to split the parent. + LOG_SEV(Trace) << " * Parent node " << parent_page_number << " is too small to add the new right page."; + splitNode(*parent, result, store_data); + } + else if (!addElementToNode(*parent, store_data)) { // If there is not enough space to add the new right page to the parent, we have to split the parent. LOG_SEV(Trace) << " * Parent node " << parent_page_number << " is full, splitting it."; splitNode(*parent, result, store_data); @@ -465,7 +513,8 @@ void BTreeManager::splitNode(BTreeNodeMap& node, SearchResult& result, std::opti << " is a pointers page with no additional data, there must be a right pointer"); } -SplitPage BTreeManager::splitSingleNode(BTreeNodeMap& node, std::optional data) { +SplitPage BTreeManager::splitSingleNode(BTreeNodeMap& node, + std::optional> data) { // Balanced split: create a new page, move half of the elements to the new page. // Unbalanced split: move all, or almost all, elements to the new page. Most efficient for adding // consecutive keys. @@ -495,15 +544,15 @@ SplitPage BTreeManager::splitSingleNode(BTreeNodeMap& node, std::optional( - node.getCell(pointers[static_cast(num_elements_to_move - 1)])); + auto pointers_cell = + std::get(node.getCell(pointers[static_cast(num_elements_to_move - 1)])); new_node.GetHeader().SetAdditionalData(pointers_cell.page_number); // TODO: WriteToPage. return_data.SetKey(pointers_cell.key); } else { auto data_cell = - std::get(node.getCell(pointers[static_cast(num_elements_to_move - 1)])); + std::get(node.getCell(pointers[static_cast(num_elements_to_move - 1)])); return_data.SetKey(data_cell.key); } LOG_SEV(Trace) << "Split key will be " << debugKey(return_data.split_key) << "."; @@ -513,19 +562,23 @@ SplitPage BTreeManager::splitSingleNode(BTreeNodeMap& node, std::optional(i)]); + auto cell = node.getCell(pointers[static_cast(i)]); std::visit( - [&new_node, this](auto&& cell) { - using T = std::decay_t; + [&new_node, this](const Node_t& cell) { + using T = std::decay_t; StoreData store_data {.key = cell.key, .serialize_key_size = serialize_key_size_}; if constexpr (std::is_same_v) { - store_data.serialized_value = internal::SpanValue(cell.page_number); + store_data.entry_creator = internal::MakeSizelessCreator( + internal::SpanValue(cell.page_number)), store_data.serialize_data_size = false; addElementToNode(new_node, store_data); } else if constexpr (std::is_same_v) { - store_data.serialized_value = cell.SpanValue(); + // NOTE: We only need to copy literally the entry in the page. In particular, it does not matter + // if the entry is the header for an overflow page. + store_data.entry_creator = + internal::MakeCreator(cell.SpanValue()); store_data.serialize_data_size = true; addElementToNode(new_node, store_data); } @@ -541,8 +594,8 @@ SplitPage BTreeManager::splitSingleNode(BTreeNodeMap& node, std::optional remaining_pointers(pointers.data() + num_elements_to_move, - pointers.size() - num_elements_to_move); + std::span remaining_pointers(pointers.data() + num_elements_to_move, + pointers.size() - num_elements_to_move); std::vector pointers_copy(remaining_pointers.size()); std::ranges::copy(remaining_pointers, pointers_copy.begin()); // Span for the vector @@ -551,16 +604,16 @@ SplitPage BTreeManager::splitSingleNode(BTreeNodeMap& node, std::optional(num_elements_to_move) * sizeof(page_size_t))); + header.SetFreeBegin(header.GetFreeStart() - (num_elements_to_move * sizeof(page_size_t))); // ======================================= // Potentially add data. // ======================================= if (data) { - LOG_SEV(Trace) << "Data requested to be added to a node, pk = " << debugKey(data->key) << "."; - auto& node_to_add_to = lte(data->key, return_data.split_key) ? new_node : node; + auto& data_ref = data->get(); + LOG_SEV(Trace) << "Data requested to be added to a node, pk = " << debugKey(data_ref.key) << "."; + auto& node_to_add_to = lte(data_ref.key, return_data.split_key) ? new_node : node; addElementToNode(node_to_add_to, *data); } @@ -570,10 +623,10 @@ SplitPage BTreeManager::splitSingleNode(BTreeNodeMap& node, std::optional data) { +void BTreeManager::splitRoot(std::optional> data) { LOG_SEV(Debug) << "Splitting root node."; // If the key type is UInt64, we are using auto-incrementing primary keys (this is the assumption for now, @@ -621,13 +674,13 @@ void BTreeManager::splitRoot(std::optional data) { LOG_SEV(Trace) << "Split key will be " << debugKey(split_key) << "."; for (page_size_t i = 0; i < root->GetNumPointers(); ++i) { - auto cell = root->getNthCell(i); + auto nth_cell = root->getNthCell(i); auto& node_to_add_to = i <= num_for_left ? left_child : right_child; std::visit( - [&](auto&& cell) { + [&](Cell_t&& cell) { StoreData store_data {.key = cell.key, .serialize_key_size = serialize_key_size_}; - using T = std::decay_t; + using T = std::decay_t; if constexpr (std::is_same_v) { if (i == num_for_left) { // Add this page as the rightmost pointer. @@ -636,14 +689,16 @@ void BTreeManager::splitRoot(std::optional data) { << node_to_add_to.GetPageNumber() << ") to " << cell.page_number << "."; } else { - store_data.serialized_value = internal::SpanValue(cell.page_number); + store_data.entry_creator = internal::MakeSizelessCreator( + internal::SpanValue(cell.page_number)); store_data.serialize_data_size = false; NOSQL_ASSERT(addElementToNode(node_to_add_to, store_data), "we should be able to add to this cell"); } } else if constexpr (std::is_same_v) { - store_data.serialized_value = cell.SpanValue(); + // Entry copier copies the entire entry. + store_data.entry_creator = std::make_unique(cell.flags, cell.SpanValue()); store_data.serialize_data_size = true; NOSQL_ASSERT(addElementToNode(node_to_add_to, store_data), "we should be able to add to this cell"); @@ -652,7 +707,7 @@ void BTreeManager::splitRoot(std::optional data) { NOSQL_FAIL("unhandled case"); } }, - cell); + nth_cell); } // If the root was a pointers page, we need to set the rightmost pointer in the root to the right child. @@ -664,8 +719,9 @@ void BTreeManager::splitRoot(std::optional data) { // Add data, if any. if (data) { - LOG_SEV(Trace) << "Data requested to be added to a node, pk = " << debugKey(data->key) << "."; - auto& node_to_add_to = lte(data->key, split_key) ? left_child : right_child; + auto& data_ref = data->get(); + LOG_SEV(Trace) << "Data requested to be added to a node, pk = " << debugKey(data_ref.key) << "."; + auto& node_to_add_to = lte(data_ref.key, split_key) ? left_child : right_child; // Only store the size of the root was NOT a pointers page (meaning we expect data to be stored, not // pointers). addElementToNode(node_to_add_to, *data, !root->IsPointersPage()); @@ -681,7 +737,8 @@ void BTreeManager::splitRoot(std::optional data) { // Add the two child pages to the root page, which is a pointers page (so we don't serialize the data size). StoreData store_data {.key = split_key, - .serialized_value = internal::SpanValue(left_page_number), + .entry_creator = internal::MakeSizelessCreator( + internal::SpanValue(left_page_number)), .serialize_key_size = serialize_key_size_, .serialize_data_size = false}; addElementToNode(*root, store_data); @@ -719,7 +776,7 @@ void BTreeManager::vacuum(BTreeNodeMap& node) const { // Move the cell to the rightmost position possible. auto cell = node.getCell(offset); // Get the size of the cell. - auto cell_size = std::visit([](auto&& cell) { return cell.GetSize(); }, cell); + auto cell_size = std::visit([](auto&& cell) { return cell.GetCellSize(); }, cell); // Adjust the next point to be at the start of where the cell must be copied. next_point -= cell_size; @@ -780,6 +837,19 @@ SearchResult BTreeManager::search(GeneralKey key) const { return result; } +RetrievalResult BTreeManager::retrieve(GeneralKey key) const { + RetrievalResult result; + result.search_result = search(key); + if (result.search_result.IsFound()) { + // Get cell index. + const auto cell_index = result.search_result.path.Top()->get().second; + const auto cell_offset = result.search_result.node->getCellOffsetByIndex(cell_index); + + result.entry = internal::ReadEntry(cell_offset, &result.search_result.node->GetPage(), this); + } + return result; +} + bool BTreeManager::lte(GeneralKey key1, GeneralKey key2) const { if (cmp_(key1, key2)) { return true; diff --git a/source/NeverSQL/data/btree/BTreeNodeMap.cpp b/source/NeverSQL/data/btree/BTreeNodeMap.cpp index 9ccdcea..568d631 100644 --- a/source/NeverSQL/data/btree/BTreeNodeMap.cpp +++ b/source/NeverSQL/data/btree/BTreeNodeMap.cpp @@ -3,6 +3,8 @@ // #include "NeverSQL/data/btree/BTreeNodeMap.h" + +#include // Other files. namespace neversql { @@ -40,6 +42,30 @@ page_size_t BTreeNodeMap::GetDefragmentedFreeSpace() const { return getHeader().GetDefragmentedFreeSpace(); } +SpaceRequirement BTreeNodeMap::CalculateSpaceRequirements(GeneralKey key) const { + SpaceRequirement requirement; + + auto&& header = getHeader(); + + // Amount of space needed for the pointer. + auto pointer_space = sizeof(page_size_t); + // Calculate amount of space for the cell. + // [Flags: 1 byte] [Key size: 2 bytes]? [Key: 8 bytes | variable] + auto cell_header_space = sizeof(uint8_t) + key.size(); + if (header.AreKeySizesSpecified()) { + cell_header_space += sizeof(uint16_t); + } + + // Given the current free space and the space needed for the pointer and the other parts of the cell, what + // is the maximum amount of space available for the entry (not counting any page entry space restrictions). + requirement.max_entry_space = static_cast(header.GetDefragmentedFreeSpace() - pointer_space - cell_header_space); + requirement.pointer_space = pointer_space; + requirement.cell_header_space = cell_header_space; + + return requirement; +} + + std::optional BTreeNodeMap::GetLargestKey() const { if (auto&& pointers = getPointers(); !pointers.empty()) { return getKeyForCell(pointers.back()); @@ -65,11 +91,12 @@ BTreePageHeader BTreeNodeMap::getHeader() const { std::optional BTreeNodeMap::getCellByKey(GeneralKey key) const { std::span pointers = getPointers(); - // Note: there was an issue trying to point in the span directly as the second argument, so I am using a placeholder - // value (0) and just using the compare function directly, ignoring its second argument (0). - auto it = std::ranges::lower_bound(pointers, 0 /* unused */, [this, key](page_size_t ptr, [[maybe_unused]] int) { - return cmp_(getKeyForCell(ptr), key); - }); + // Note: there was an issue trying to point in the span directly as the second argument, so I am using a + // placeholder value (0) and just using the compare function directly, ignoring its second argument (0). + auto it = + std::ranges::lower_bound(pointers, 0 /* unused */, [this, key](page_size_t ptr, [[maybe_unused]] int) { + return cmp_(getKeyForCell(ptr), key); + }); if (it == pointers.end()) { return std::nullopt; } @@ -81,14 +108,14 @@ std::optional BTreeNodeMap::getCellByKey(GeneralKey key) const { return {}; } -std::optional> BTreeNodeMap::getCellLowerBoundByPK(GeneralKey key) const { +std::optional> BTreeNodeMap::getCellLowerBoundByPK( + GeneralKey key) const { std::span pointers = getPointers(); - auto it = std::ranges::lower_bound( - pointers, key, cmp_, [this](auto&& ptr) { return getKeyForCell(ptr); }); + auto it = std::ranges::lower_bound(pointers, key, cmp_, [this](auto&& ptr) { return getKeyForCell(ptr); }); if (it == pointers.end()) { return {}; } - return std::make_optional(std::pair{*it, static_cast(std::distance(pointers.begin(), it))}); + return std::make_optional(std::pair {*it, static_cast(std::distance(pointers.begin(), it))}); } std::pair BTreeNodeMap::searchForNextPageInPointersPage(GeneralKey key) const { @@ -110,10 +137,11 @@ std::pair BTreeNodeMap::searchForNextPageInPointers } // Get the offset to the first key that is greater auto offset = getCellLowerBoundByPK(key); - NOSQL_ASSERT(offset.has_value(), "could not find a cell with a key greater than or equal to " << debugKey(key)); - // Offset gets us to the primary key, so we need to add the size of the primary key to get to the page - // number. - return {page_->Read(offset->first + sizeof(primary_key_t)), offset->second}; + NOSQL_ASSERT(offset.has_value(), + "could not find a cell with a key greater than or equal to " << debugKey(key)); + + auto pointer_cell = std::get(getCell(offset->first)); + return {pointer_cell.page_number, offset->second}; } std::span BTreeNodeMap::getPointers() const { @@ -124,13 +152,22 @@ std::span BTreeNodeMap::getPointers() const { return page_->GetSpan(start_ptrs, num_pointers); } +page_size_t BTreeNodeMap::getCellOffsetByIndex(page_size_t cell_index) const { + auto&& pointers = getPointers(); + NOSQL_ASSERT(cell_index < pointers.size(), "cell number " << cell_index << " is out of range"); + return pointers[cell_index]; +} + GeneralKey BTreeNodeMap::getKeyForCell(page_size_t cell_offset) const { + // Bypass flags. + cell_offset += 1; + if (getHeader().AreKeySizesSpecified()) { auto key_size = page_->Read(cell_offset); return page_->GetSpan(cell_offset + sizeof(uint16_t), key_size); } // TODO: For now at least, assume that keys whose size are not specified are uint64_t. This can be relaxed - // later. + // later. return page_->GetSpan(cell_offset, sizeof(primary_key_t)); } @@ -141,32 +178,66 @@ GeneralKey BTreeNodeMap::getKeyForNthCell(page_size_t cell_index) const { } std::variant BTreeNodeMap::getCell(page_size_t cell_offset) const { - const auto& header = getHeader(); - auto key_size_is_serialized = header.AreKeySizesSpecified(); + // Single page entry. + // [flags: 1 byte] + // [key_size: 2 bytes]? + // [key: 8 bytes | variable] + // ---- Entry ------------------- + // [entry_size: 2 bytes] + // [entry_data: entry_size bytes] + + // Overflow entry header. + // [flags: 1 byte] + // [key_size: 2 bytes]? + // [key: 8 bytes | variable] + // ---- Entry ------------------- + // [overflow_key: 8 bytes] + // [overflow page number: 8 bytes] + std::span key; - auto post_key_offset = cell_offset; + auto entry_offset = cell_offset; + + // Flags. + const auto flags = page_->Read(entry_offset); + entry_offset += 1; + // ==== Read the flags ==== + const bool is_active = internal::GetIsActive(flags); + NOSQL_ASSERT(is_active, "cannot load entry, entry is inactive"); + const bool is_single_page = internal::GetIsSinglePageEntry(flags); + const bool key_size_is_serialized = internal::GetKeySizeIsSerialized(flags); + const bool is_note_flag_true = internal::IsNoteFlagTrue(flags); + // ======================== + if (key_size_is_serialized) { - auto key_size = page_->Read(cell_offset); - post_key_offset += sizeof(uint16_t) + key_size; - key = page_->GetSpan(cell_offset + sizeof(uint16_t), key_size); + const auto key_size = page_->Read(entry_offset); + entry_offset += sizeof(uint16_t); + key = page_->GetSpan(entry_offset, key_size); + entry_offset += key_size; } else { - // TODO: For now at least, assume that keys whose size are not specified are uint64_t. This can be relaxed later. - post_key_offset += sizeof(primary_key_t); - key = page_->GetSpan(cell_offset, sizeof(primary_key_t)); + // TODO: For now at least, assume that keys whose size are not specified are uint64_t. + // This can be relaxed later. + key = page_->GetSpan(entry_offset, sizeof(primary_key_t)); + entry_offset += sizeof(primary_key_t); } if (getHeader().IsPointersPage()) { return PointersNodeCell {.key = key, - .page_number = page_->Read(post_key_offset), + .page_number = page_->Read(entry_offset), .key_size_is_serialized = key_size_is_serialized}; } + + // If this is an overflow header, it is 16 bytes. Otherwise, the size of the entry is stored in the next 2 + // bytes. + const auto potential_entry_size = page_->Read(entry_offset); + const auto entry_data = is_single_page + ? page_->ReadFromPage(entry_offset + (is_note_flag_true ? sizeof(page_size_t) : 0), + potential_entry_size) + : page_->ReadFromPage(entry_offset, 16); + return DataNodeCell { - .key = key, - .size_of_entry = page_->Read(post_key_offset), - .start_of_value = page_->GetData() + post_key_offset + sizeof(entry_size_t), - .key_size_is_serialized = key_size_is_serialized}; + .flags = flags, .key = key, .data = entry_data, .key_size_is_serialized = key_size_is_serialized}; } std::variant BTreeNodeMap::getNthCell(page_size_t cell_number) const { diff --git a/source/NeverSQL/data/btree/EntryCopier.cpp b/source/NeverSQL/data/btree/EntryCopier.cpp new file mode 100644 index 0000000..3aa6993 --- /dev/null +++ b/source/NeverSQL/data/btree/EntryCopier.cpp @@ -0,0 +1,15 @@ +// +// Created by Nathaniel Rupprecht on 3/30/24. +// + +#include "NeverSQL/data/btree/EntryCopier.h" +// Other files. +#include "NeverSQL/data/internals/SpanPayloadSerializer.h" + +namespace neversql::internal { + +EntryCopier::EntryCopier(std::byte flags, std::span payload) + : EntryCreator(std::make_unique(payload), false) + , flags_(flags & std::byte {0xFF}) {} + +} // namespace neversql::internal \ No newline at end of file diff --git a/source/NeverSQL/data/btree/EntryCreator.cpp b/source/NeverSQL/data/btree/EntryCreator.cpp new file mode 100644 index 0000000..3091812 --- /dev/null +++ b/source/NeverSQL/data/btree/EntryCreator.cpp @@ -0,0 +1,84 @@ +// +// Created by Nathaniel Rupprecht on 3/27/24. +// + +#include "NeverSQL/data/btree/EntryCreator.h" +// Other files. +#include "NeverSQL/data/Page.h" + +namespace neversql::internal { + +EntryCreator::EntryCreator(std::unique_ptr&& payload, bool serialize_size) + : serialize_size_(serialize_size) + , payload_(std::move(payload)) {} + +page_size_t EntryCreator::GetMinimumEntrySize() const { + // An overflow page header needs 16 bytes (plus the flags and whatever the B-tree needs). + return 16; +} + +page_size_t EntryCreator::GetRequestedSize(page_size_t maximum_entry_size) { +if (maximum_entry_size == 1) { + std::cout << ""; +} + + NOSQL_REQUIRE(GetMinimumEntrySize() <= maximum_entry_size, + "maximum entry size too small (" + << maximum_entry_size << ", minimum is " << GetMinimumEntrySize() + << "), this should have been checked before calling this function"); + + auto size = (serialize_size_ ? sizeof(page_size_t) : 0) + payload_->GetRequiredSize(); + if (maximum_entry_size < size) { + LOG_SEV(Trace) << "Size of entry is " << size << ", which is larger than the maximum entry size of " + << maximum_entry_size << ". Overflow page needed."; + overflow_page_needed_ = true; + return 16; + } + return size; +} + +std::byte EntryCreator::GenerateFlags() const { + using enum EntryFlags; + // The note flag is set if the entry is an overflow page entry or the entry size is serialized. + uint8_t flags = IsActive | (serialize_size_ || overflow_page_needed_ ? NoteFlag : 0) + | (overflow_page_needed_ ? 0 : IsSinglePageEntry); + return std::byte {flags}; +} + +//! \brief Create an entry, starting with the given offset in the page. +page_size_t EntryCreator::Create(page_size_t starting_offset, Page* page, const BTreeManager* btree_manager) { + // Header or single page entry. + if (overflow_page_needed_) { + createOverflowEntry(starting_offset, page, btree_manager); + return starting_offset + 2 * sizeof(page_number_t); + } + + // Single page entry. + return createSinglePageEntry(starting_offset, page); +} + +void EntryCreator::createOverflowEntry(page_size_t starting_offset, + Page* page, + const BTreeManager* btree_manager) { + // [overflow_key: 8 bytes] [overflow page number: 8 bytes] + + // For each subsequent overflow page: + // [next overflow page number: 8 bytes]? [entry_size: 2 bytes] [entry_data: entry_size bytes] + + NOSQL_FAIL("unimplemented"); +} + +page_size_t EntryCreator::createSinglePageEntry(page_size_t starting_offset, Page* page) { + // Entry size. + auto offset = starting_offset; + if (serialize_size_) { + const auto entry_size = static_cast(payload_->GetRequiredSize()); + offset = page->WriteToPage(starting_offset, entry_size); + } + while (payload_->HasData()) { + offset = page->WriteToPage(offset, payload_->GetNextByte()); + } + return offset; +} + +} // namespace neversql::internal \ No newline at end of file diff --git a/source/NeverSQL/data/internals/DatabaseEntry.cpp b/source/NeverSQL/data/internals/DatabaseEntry.cpp new file mode 100644 index 0000000..9467395 --- /dev/null +++ b/source/NeverSQL/data/internals/DatabaseEntry.cpp @@ -0,0 +1,64 @@ +// +// Created by Nathaniel Rupprecht on 3/28/24. +// + +#include "NeverSQL/data/internals/DatabaseEntry.h" +// Other files. +#include "NeverSQL/data/btree/BTree.h" +#include "NeverSQL/data/btree/EntryCreator.h" +#include "NeverSQL/data/internals/OverflowEntry.h" +#include "NeverSQL/data/internals/SinglePageEntry.h" + +namespace neversql::internal { + +std::unique_ptr ReadEntry(page_size_t starting_offset, + const Page* page, + const BTreeManager* btree_manager) { + // TODO: This function takes care of some things that are the B-tree's responsibility, i.e., key related + // things. These things should probably be moved to the B-tree, and the offset should be at the start of + // the entry, and the flags are passed in. + + // Single page entry. + // [flags: 1 byte] + // [key_size: 2 bytes]? + // [key: 8 bytes | variable] + // ----------------------------------- + // [entry_size: 4 bytes] + // [entry_data: entry_size bytes] + + // Overflow entry header. + // [flags: 1 byte] + // [key_size: 2 bytes]? + // [key: 8 bytes | variable] + // ----------------------------------- + // [overflow_key: 8 bytes] + // [overflow page number: 8 bytes] + + LOG_SEV(Trace) << "Reading entry, starting offset is " << starting_offset << "."; + + // Read flags to determine whether the entry is a single database entry or an overflow entry. + const auto flags = page->Read(starting_offset); + // Read individual flags. + const bool is_active = GetIsActive(flags); + NOSQL_ASSERT(is_active, "cannot load entry, entry is inactive"); + const bool is_single_page = GetIsSinglePageEntry(flags); + const bool key_size_serialized = GetKeySizeIsSerialized(flags); + + auto entry_offset = starting_offset + 1; // Skip the flags. + if (key_size_serialized) { + const auto key_size = page->Read(entry_offset); + entry_offset += sizeof(page_size_t) + key_size; + } + else { + entry_offset += sizeof(primary_key_t); + } + + if (is_single_page) { + return std::make_unique(entry_offset, page); + } + + auto header = page->ReadFromPage(entry_offset, 16); + return std::make_unique(header, btree_manager); +} + +} // namespace neversql::internal \ No newline at end of file diff --git a/source/NeverSQL/data/internals/DocumentPayloadSerializer.cpp b/source/NeverSQL/data/internals/DocumentPayloadSerializer.cpp new file mode 100644 index 0000000..224c064 --- /dev/null +++ b/source/NeverSQL/data/internals/DocumentPayloadSerializer.cpp @@ -0,0 +1,38 @@ +// +// Created by Nathaniel Rupprecht on 3/27/24. +// + +#include "NeverSQL/data/internals/DocumentPayloadSerializer.h" +// Other files. +#include +#include + +namespace neversql::internal { + +bool DocumentPayloadSerializer::HasData() { + return current_index_ < buffer_.Size(); +} + +std::byte DocumentPayloadSerializer::GetNextByte() { + if (HasData()) { + return buffer_.Data()[current_index_++]; + } + return {}; +} + +std::size_t DocumentPayloadSerializer::GetRequiredSize() const { + return buffer_.Size(); +} + +void DocumentPayloadSerializer::initialize() { + std::visit([this](auto& document) { document->WriteToBuffer(buffer_); }, document_); +} + +const Document& DocumentPayloadSerializer::getDocument() const { + if (std::holds_alternative>(document_)) { + return *std::get>(document_); + } + return *std::get(document_); +} + +} // namespace neversql::internal \ No newline at end of file diff --git a/source/NeverSQL/database/DataManager.cpp b/source/NeverSQL/database/DataManager.cpp index 8bd9a39..5d98b15 100644 --- a/source/NeverSQL/database/DataManager.cpp +++ b/source/NeverSQL/database/DataManager.cpp @@ -4,6 +4,7 @@ #include "NeverSQL/database/DataManager.h" // Other files. +#include "NeverSQL/data/internals/DocumentPayloadSerializer.h" #include "NeverSQL/data/internals/Utility.h" #include "NeverSQL/utility/PageDump.h" @@ -11,7 +12,7 @@ namespace neversql { DataManager::DataManager(const std::filesystem::path& database_path) : data_access_layer_(database_path) - , page_cache_(database_path / "walfiles", 16 /* Just a random number for now */, &data_access_layer_) + , page_cache_(database_path / "walfiles", 256 /* Just a random number for now */, &data_access_layer_) , collection_index_(nullptr) { // TODO: Make the meta page more independent from the database. auto&& meta = data_access_layer_.GetMeta(); @@ -53,41 +54,25 @@ void DataManager::AddCollection(const std::string& collection_name, DataTypeEnum auto page_number = btree->GetRootPageNumber(); - neversql::Document document; - document.AddElement("collection_name", StringValue {collection_name}); - document.AddElement("index_page_number", IntegralValue {page_number}); + auto document = std::make_unique(); + document->AddElement("collection_name", StringValue {collection_name}); + document->AddElement("index_page_number", IntegralValue {page_number}); - // NOTE: This is not the best way to do this, I just want to get something that works. - [[maybe_unused]] auto size = document.CalculateRequiredSize(); - lightning::memory::MemoryBuffer buffer; - - WriteToBuffer(buffer, document); - std::span value(buffer.Data(), buffer.Size()); - collection_index_->AddValue(internal::SpanValue(collection_name), value); + auto creator = internal::MakeCreator(std::move(document)); + collection_index_->AddValue(internal::SpanValue(collection_name), std::move(creator)); // Cache the collection in the data manager. collections_.emplace(collection_name, std::move(btree)); } -void DataManager::AddValue(const std::string& collection_name, - GeneralKey key, - std::span value) { +void DataManager::AddValue(const std::string& collection_name, GeneralKey key, const Document& document) { // Find the collection. auto it = collections_.find(collection_name); // TODO: Error handling without throwing. NOSQL_ASSERT(it != collections_.end(), "Collection '" << collection_name << "' does not exist."); - it->second->AddValue(key, value); -} -void DataManager::AddValue(const std::string& collection_name, GeneralKey key, const Document& document) { - // Serialize the document and add it to the database. - [[maybe_unused]] auto size = document.CalculateRequiredSize(); - lightning::memory::MemoryBuffer buffer; - - document.WriteToBuffer(buffer); - // WriteToBuffer(buffer, document); - std::span value(buffer.Data(), buffer.Size()); - AddValue(collection_name, key, value); + auto creator = internal::MakeCreator(std::move(document)); + it->second->AddValue(key, std::move(creator)); } SearchResult DataManager::Search(const std::string& collection_name, GeneralKey key) const { @@ -99,61 +84,30 @@ SearchResult DataManager::Search(const std::string& collection_name, GeneralKey } RetrievalResult DataManager::Retrieve(const std::string& collection_name, GeneralKey key) const { - RetrievalResult result {.search_result = Search(collection_name, key)}; - if (result.search_result.node) { - if (auto offset = *result.search_result.node->getCellByKey(key)) { - result.cell_offset = offset; - result.value_view = std::get(result.search_result.node->getCell(offset)).SpanValue(); - } - else { - // Element *DID NOT EXIST IN THE NODE* that it was expected to exist in. - result.search_result.node = {}; - } - } - return result; -} - -void DataManager::AddValue(const std::string& collection_name, - primary_key_t key, - std::span value) { // Find the collection. auto it = collections_.find(collection_name); // TODO: Error handling without throwing. NOSQL_ASSERT(it != collections_.end(), "Collection '" << collection_name << "' does not exist."); - - GeneralKey key_span = internal::SpanValue(key); - it->second->AddValue(key_span, value); + return it->second->retrieve(key); } -void DataManager::AddValue(const std::string& collection_name, std::span value) { +void DataManager::AddValue(const std::string& collection_name, const Document& document) { // Find the collection. auto it = collections_.find(collection_name); // TODO: Error handling without throwing. NOSQL_ASSERT(it != collections_.end(), "Collection '" << collection_name << "' does not exist."); - it->second->AddValue(value); -} - -void DataManager::AddValue(const std::string& collection_name, const Document& document) { - // Serialize the document and add it to the database. - // TODO: Deal with documents that are too long. - // NOTE: This is not the best way to do this, I just want to get something that works. - [[maybe_unused]] auto size = document.CalculateRequiredSize(); - lightning::memory::MemoryBuffer buffer; - - WriteToBuffer(buffer, document); - std::span value(buffer.Data(), buffer.Size()); - AddValue(collection_name, value); + auto creator = internal::MakeCreator(std::move(document)); + it->second->AddValue(std::move(creator)); } SearchResult DataManager::Search(const std::string& collection_name, primary_key_t key) const { - GeneralKey key_span = internal::SpanValue(key); + const GeneralKey key_span = internal::SpanValue(key); return Search(collection_name, key_span); } RetrievalResult DataManager::Retrieve(const std::string& collection_name, primary_key_t key) const { - GeneralKey key_span = internal::SpanValue(key); - + const GeneralKey key_span = internal::SpanValue(key); return Retrieve(collection_name, key_span); } @@ -184,13 +138,13 @@ bool DataManager::HexDumpPage(page_number_t page_number, auto page = page_cache_.GetPage(page_number); auto view = page->GetView(); std::istringstream stream(std::string {view}); - neversql::utility::HexDump(stream, out, options); + HexDump(stream, out, options); return true; } bool DataManager::NodeDumpPage(page_number_t page_number, std::ostream& out) const { if (auto page = collection_index_->loadNodePage(page_number)) { - neversql::utility::PageInspector::NodePageDump(*page, out); + utility::PageInspector::NodePageDump(*page, out); return true; } return false; diff --git a/source/NeverSQL/utility/PageDump.cpp b/source/NeverSQL/utility/PageDump.cpp index 46b7110..f1b021b 100644 --- a/source/NeverSQL/utility/PageDump.cpp +++ b/source/NeverSQL/utility/PageDump.cpp @@ -12,15 +12,14 @@ namespace neversql::utility { - void PageInspector::NodePageDump(const BTreeNodeMap& node, std::ostream& out) { - lightning::memory::StringMemoryBuffer buffer; std::vector numbers; std::vector offsets; std::vector cell_types; std::vector primary_keys; + std::vector flags; std::vector data_size; std::vector data; @@ -39,16 +38,18 @@ void PageInspector::NodePageDump(const BTreeNodeMap& node, std::ostream& out) { using T = std::decay_t; if constexpr (std::is_same_v) { auto view = cell.SpanValue(); - std::string_view sv{reinterpret_cast(view.data()), view.size()}; + std::string_view sv {reinterpret_cast(view.data()), view.size()}; cell_types.emplace_back("Data cell"); primary_keys.push_back(cell.key); - data_size.push_back(cell.size_of_entry); + flags.push_back(cell.flags); + data_size.push_back(cell.GetDataSize()); data.emplace_back(sv); } else if constexpr (std::is_same_v) { cell_types.emplace_back("Pointer cell"); primary_keys.push_back(cell.key); - data_size.push_back(0); + flags.push_back(cell.flags); + data_size.push_back(cell.GetDataSize()); data.push_back(std::to_string(cell.page_number)); } else { @@ -70,6 +71,13 @@ void PageInspector::NodePageDump(const BTreeNodeMap& node, std::ostream& out) { table.AddColumn( "PK", primary_keys, [](const GeneralKey& pk) { return internal::HexDumpBytes(pk); }, "BLUE", "BBLUE"); + table.AddColumn( + "Flags", + flags, + [](const std::byte& flag) { return std::format("0b{:b}", static_cast(flag)); }, + "BWHITE", + "BBLUE"); + table.AddColumn( "Data size", data_size, @@ -95,7 +103,7 @@ void PageInspector::NodePageDump(const BTreeNodeMap& node, std::ostream& out) { // Write header above the table, using the same width. auto header_width = table.GetTotalWidth(); - { // Write HEADER + { // Write HEADER std::fill_n(std::ostream_iterator(out), header_width, '='); std::string fmt_string = "\n|{@BWHITE}{:^" + std::to_string(header_width - 2) + "}{@RESET}|\n"; out << lightning::formatting::Format(fmt_string, "HEADER"); @@ -115,16 +123,21 @@ void PageInspector::NodePageDump(const BTreeNodeMap& node, std::ostream& out) { out << lightning::formatting::Format("| {:<20}{@BBLUE}{}{@RESET}\n", "Flags:", buffer.MoveString()); buffer.Clear(); - out << lightning::formatting::Format("| {:<20}{@BWHITE}{}{@RESET}\n", "Free start:", header.GetFreeStart()); + out << lightning::formatting::Format( + "| {:<20}{@BWHITE}{}{@RESET}\n", "Free start:", header.GetFreeStart()); out << lightning::formatting::Format("| {:<20}{@BWHITE}{}{@RESET}\n", "Free end:", header.GetFreeEnd()); - out << lightning::formatting::Format("| {:<20}{@BWHITE}{}{@RESET}\n", "Reserved start:", header.GetReservedStart()); - out << lightning::formatting::Format("| {:<20}{@BGREEN}{}{@RESET}\n", "Page number:", header.GetPageNumber()); - out << lightning::formatting::Format("| {:<20}{@BWHITE}{}{@RESET}\n", "Additional data:", header.GetAdditionalData()); + out << lightning::formatting::Format( + "| {:<20}{@BWHITE}{}{@RESET}\n", "Reserved start:", header.GetReservedStart()); + out << lightning::formatting::Format( + "| {:<20}{@BGREEN}{}{@RESET}\n", "Page number:", header.GetPageNumber()); + out << lightning::formatting::Format( + "| {:<20}{@BWHITE}{}{@RESET}\n", "Additional data:", header.GetAdditionalData()); out << "|\n|\n"; out << "| Hex dump of header:\n"; out << lightning::formatting::Format( - "| {@BYELLOW}{}{@RESET}\n", internal::HexDumpBytes(node.GetPage().GetSpan(0, header.GetPointersStart()), false)); + "| {@BYELLOW}{}{@RESET}\n", + internal::HexDumpBytes(node.GetPage().GetSpan(0, header.GetPointersStart()), false)); { std::fill_n(std::ostream_iterator(out), header_width, '=');