Skip to content

Commit

Permalink
Overhaul of storage engine
Browse files Browse the repository at this point in the history 
- Use entry creators to control how entries are serialized into one or more pages. This will be how overflow pages work, though this feature is not yet implemented.
- Entry payload serializers return the data to serialize as a sequence of bytes.
- Introduced DatabaseEntry object as a way to retrieve data, even if it is split over multiple pages, i.e., uses overflow pages.
- Lots of refactors to use the new storage system.
  • Loading branch information
@nrupprecht
nrupprecht committed Apr 3, 2024
1 parent fddfb46 commit fa0b71d
Show file tree
Hide file tree
Showing 27 changed files with 1,114 additions and 298 deletions.
122 changes: 64 additions & 58 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ cmake_minimum_required(VERSION 3.14)
include(cmake/prelude.cmake)

project(
NeverSQL
VERSION 0.1.0
DESCRIPTION "A small implementation of a no-sql database."
HOMEPAGE_URL "https://example.com/"
LANGUAGES CXX
NeverSQL
VERSION 0.1.0
DESCRIPTION "A small implementation of a no-sql database."
HOMEPAGE_URL "https://example.com/"
LANGUAGES CXX
)

set(CMAKE_CXX_STANDARD 20)
Expand All @@ -16,10 +16,10 @@ set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20")

if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
# Needed to get rid of a warning from chrono.
# https://stackoverflow.com/questions/76859275/error-compiling-a-cpp-containing-stdchrono-errorstatic-constexpr-unsigned-fra
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
endif()
# Needed to get rid of a warning from chrono.
# https://stackoverflow.com/questions/76859275/error-compiling-a-cpp-containing-stdchrono-errorstatic-constexpr-unsigned-fra
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
endif ()

include(cmake/project-is-top-level.cmake)
include(cmake/variables.cmake)
Expand All @@ -30,66 +30,72 @@ option(BUILD_NEVERSQL_APPLICATIONS ON)

# ---- Declare library ----

include (FetchContent)
include(FetchContent)

# Lightning logging library.
FetchContent_Declare(
Lightning
GIT_REPOSITORY https://github.com/nrupprecht/Lightning.git
GIT_TAG c254ecc
GIT_TAG f81f107
)
FetchContent_MakeAvailable(Lightning)
include_directories(${Lightning_SOURCE_DIR}/include)

add_library(
NeverSQL_NeverSQL
# Source files.
source/NeverSQL/data/DataAccessLayer.cpp
source/NeverSQL/data/Document.cpp
source/NeverSQL/data/FreeList.cpp
source/NeverSQL/data/Page.cpp
source/NeverSQL/data/PageCache.cpp
source/NeverSQL/data/btree/BTree.cpp
source/NeverSQL/data/btree/BTreeNodeMap.cpp
source/NeverSQL/database/DataManager.cpp
source/NeverSQL/utility/HexDump.cpp
source/NeverSQL/utility/PageDump.cpp
source/NeverSQL/utility/DisplayTable.cpp
source/NeverSQL/recovery/WriteAheadLog.cpp
NeverSQL_NeverSQL
# Source files.
source/NeverSQL/data/DataAccessLayer.cpp
source/NeverSQL/data/Document.cpp
source/NeverSQL/data/FreeList.cpp
source/NeverSQL/data/Page.cpp
source/NeverSQL/data/PageCache.cpp
source/NeverSQL/data/btree/BTree.cpp
source/NeverSQL/data/btree/BTreeNodeMap.cpp
source/NeverSQL/data/btree/EntryCreator.cpp
source/NeverSQL/data/btree/EntryCopier.cpp
source/NeverSQL/data/internals/DatabaseEntry.cpp
source/NeverSQL/data/internals/DocumentPayloadSerializer.cpp
source/NeverSQL/database/DataManager.cpp
source/NeverSQL/recovery/WriteAheadLog.cpp
source/NeverSQL/utility/HexDump.cpp
source/NeverSQL/utility/PageDump.cpp
source/NeverSQL/utility/DisplayTable.cpp

)
add_library(NeverSQL::NeverSQL ALIAS NeverSQL_NeverSQL)

include(GenerateExportHeader)
generate_export_header(
NeverSQL_NeverSQL
BASE_NAME NeverSQL
EXPORT_FILE_NAME export/NeverSQL/NeverSQL_export.hpp
CUSTOM_CONTENT_FROM_VARIABLE pragma_suppress_c4251
NeverSQL_NeverSQL
BASE_NAME NeverSQL
EXPORT_FILE_NAME export/NeverSQL/NeverSQL_export.hpp
CUSTOM_CONTENT_FROM_VARIABLE pragma_suppress_c4251
)

if(NOT BUILD_SHARED_LIBS)
target_compile_definitions(NeverSQL_NeverSQL PUBLIC NEVERSQL_STATIC_DEFINE)
endif()
if (NOT BUILD_SHARED_LIBS)
target_compile_definitions(NeverSQL_NeverSQL PUBLIC NEVERSQL_STATIC_DEFINE)
endif ()

set_target_properties(
NeverSQL_NeverSQL PROPERTIES
CXX_VISIBILITY_PRESET hidden
VISIBILITY_INLINES_HIDDEN YES
VERSION "${PROJECT_VERSION}"
SOVERSION "${PROJECT_VERSION_MAJOR}"
EXPORT_NAME NeverSQL
OUTPUT_NAME NeverSQL
NeverSQL_NeverSQL PROPERTIES
CXX_VISIBILITY_PRESET hidden
VISIBILITY_INLINES_HIDDEN YES
VERSION "${PROJECT_VERSION}"
SOVERSION "${PROJECT_VERSION_MAJOR}"
EXPORT_NAME NeverSQL
OUTPUT_NAME NeverSQL
)

target_include_directories(
NeverSQL_NeverSQL ${warning_guard}
PUBLIC
"$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>"
NeverSQL_NeverSQL ${warning_guard}
PUBLIC
"$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>"
)

target_include_directories(
NeverSQL_NeverSQL SYSTEM
PUBLIC
"$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/export>"
NeverSQL_NeverSQL SYSTEM
PUBLIC
"$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/export>"
)

target_compile_features(NeverSQL_NeverSQL PUBLIC cxx_std_20)
Expand All @@ -98,27 +104,27 @@ target_compile_features(NeverSQL_NeverSQL PUBLIC cxx_std_20)


#if (BUILD_NEVERSQL_APPLICATIONS)
message("Building applications.")
add_subdirectory("${PROJECT_SOURCE_DIR}/applications")
message("Building applications.")
add_subdirectory("${PROJECT_SOURCE_DIR}/applications")
#else()
# message("Not building applications.")
#endif()

# ---- Install rules ----

if(NOT CMAKE_SKIP_INSTALL_RULES)
include(cmake/install-rules.cmake)
endif()
if (NOT CMAKE_SKIP_INSTALL_RULES)
include(cmake/install-rules.cmake)
endif ()

# ---- Developer mode ----

if(NOT NeverSQL_DEVELOPER_MODE)
return()
elseif(NOT PROJECT_IS_TOP_LEVEL)
message(
AUTHOR_WARNING
"Developer mode is intended for developers of NeverSQL"
)
endif()
if (NOT NeverSQL_DEVELOPER_MODE)
return()
elseif (NOT PROJECT_IS_TOP_LEVEL)
message(
AUTHOR_WARNING
"Developer mode is intended for developers of NeverSQL"
)
endif ()

include(cmake/dev-mode.cmake)
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ Some useful resources on databases and database implementations:
* Mongodb
* https://github.com/mongodb/mongo
* BSON spec: https://bsonspec.org/, https://bsonspec.org/spec.html
* WiredTiger
* https://source.wiredtiger.com/11.2.0/arch-index.html
* Other tutorials / similar projects
* https://cstack.github.io/db_tutorial/
* https://adambcomer.com/blog/simple-database/motivation-design/
Expand Down
71 changes: 48 additions & 23 deletions applications/data-manager-example.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,22 @@ using namespace neversql;
void SetupLogger(Severity min_severity = Severity::Info);

int main() {
// 0.004311 ms per addition, 233,612 pages, 10,000,000 entries.
SetupLogger(Severity::Info);

// ---> Your database path here.
std::filesystem::path database_path = "path-to-your-directory";
std::filesystem::path database_path =
"/Users/nathaniel/Documents/Nathaniel/Programs/C++/NeverSQL/database-dmgr-test";

remove_all(database_path);

// std::filesystem::remove_all(database_path);
primary_key_t num_to_insert = 10'000'000; // 48

primary_key_t num_to_insert = 100'000'000;
DataManager manager(database_path);

neversql::DataManager manager(database_path);
LOG_SEV(Info) << formatting::Format("Database has {:L} pages.", manager.GetDataAccessLayer().GetNumPages());

LOG_SEV(Info) << lightning::formatting::Format("Database has {:L} pages.",
manager.GetDataAccessLayer().GetNumPages());

manager.AddCollection("elements", neversql::DataTypeEnum::UInt64);
manager.AddCollection("elements", DataTypeEnum::UInt64);

primary_key_t pk = 0;
auto starting_time_point = std::chrono::high_resolution_clock::now();
Expand All @@ -41,10 +41,10 @@ int main() {
try {
for (; pk < num_to_insert; ++pk) {
// Create a document.
neversql::Document builder;
builder.AddElement("data", StringValue{formatting::Format("Brave new world.\nEntry number {}.", pk)});
builder.AddElement("pk", IntegralValue{static_cast<int32_t>(pk)});
builder.AddElement("is_even", BooleanValue{pk % 2 == 0});
Document builder;
builder.AddElement("data", StringValue {formatting::Format("Brave new world.\nEntry number {}.", pk)});
builder.AddElement("pk", IntegralValue {static_cast<int32_t>(pk)});
builder.AddElement("is_even", BooleanValue {pk % 2 == 0});
// Add the document.
manager.AddValue("elements", builder);

Expand Down Expand Up @@ -79,38 +79,63 @@ int main() {
LOG_SEV(Major) << lightning::formatting::Format("Database has {:L} pages.", total_pages);

// Node dump the main index page.
manager.NodeDumpPage(2, std::cout);
manager.NodeDumpPage(3, std::cout);
std::cout << std::endl;
try {
manager.NodeDumpPage(4, std::cout);
std::cout << std::endl;
} catch (...) {
}

// Search for some elements.
auto first_to_probe = num_to_insert / 2;
auto last_to_probe = std::min(num_to_insert / 2 + 10, num_to_insert);
for (primary_key_t pk_probe = first_to_probe; pk_probe < last_to_probe; ++pk_probe) {
auto result = manager.Retrieve("elements", pk_probe);
if (result.IsFound()) {
auto& view = result.value_view;
memory::MemoryBuffer<std::byte> buffer;

auto& entry = *result.entry;
do {
auto data = entry.GetData();
buffer.Append(data);
} while (entry.Advance());

auto view = std::span {buffer.Data(), buffer.Size()};

// Interpret the data as a document.
auto document = neversql::ReadDocumentFromBuffer(view);

LOG_SEV(Info) << formatting::Format(
"Found key {:L} on page {:L}, search depth {}, value: \n{@BYELLOW}{}{@RESET}",
pk_probe,
result.search_result.node->GetPageNumber(),
result.search_result.GetSearchDepth(),
neversql::PrettyPrint(*document));
if (auto document = neversql::ReadDocumentFromBuffer(view)) {
LOG_SEV(Info) << formatting::Format(
"Found key {:L} on page {:L}, search depth {}, value: \n{@BYELLOW}{}{@RESET}",
pk_probe,
result.search_result.node->GetPageNumber(),
result.search_result.GetSearchDepth(),
neversql::PrettyPrint(*document));
}
else {
LOG_SEV(Error) << "Could not read document.";
}
}
else {
LOG_SEV(Info) << formatting::Format("{@BRED}Key {} was not found.{@RESET}", pk_probe);
}
}

// auto it_begin = manager.Begin("elements");
// auto end_it = manager.End("elements");
// for (auto it = it_begin; it != end_it; ++it) {
// auto view = *it;
// // Interpret the data as a document.
// neversql::DocumentReader reader(view);
// LOG_SEV(Info) << formatting::Format("Value: \n{@BYELLOW}{}{@RESET}", neversql::PrettyPrint(reader));
// }

return 0;
}

void SetupLogger(Severity min_severity) {
auto console = lightning::NewSink<lightning::StdoutSink>();
lightning::Global::GetCore()->AddSink(console);
Global::GetCore()->AddSink(console);
console->SetFilter(min_severity <= LoggingSeverity);

// Formatter for "low levels" of severity displace file and line number.
Expand Down
15 changes: 13 additions & 2 deletions applications/database-string-pk.example.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include "NeverSQL/data/btree/BTree.h"
#include "NeverSQL/data/internals/Utility.h"
#include "NeverSQL/database/DataManager.h"
#include "NeverSQL/database/Query.h"
#include "NeverSQL/utility/HexDump.h"
#include "NeverSQL/utility/PageDump.h"

Expand All @@ -20,7 +21,8 @@ int main() {
SetupLogger(Severity::Info);

// ---> Your database path here.
std::filesystem::path database_path = "your-path-here";
std::filesystem::path database_path =
"/Users/nathaniel/Documents/Nathaniel/Programs/C++/NeverSQL/database-string";

std::filesystem::remove_all(database_path);

Expand Down Expand Up @@ -106,7 +108,16 @@ int main() {
auto result = manager.Retrieve("elements", neversql::internal::SpanValue(name));
if (result.IsFound()) {
// Interpret the data as a document.
auto document = neversql::ReadDocumentFromBuffer(result.value_view);

memory::MemoryBuffer<std::byte> buffer;
auto& entry = *result.entry;
do {
auto data = entry.GetData();
buffer.Append(data);
} while (entry.Advance());
auto view = std::span {buffer.Data(), buffer.Size()};

auto document = neversql::ReadDocumentFromBuffer(view);

LOG_SEV(Info) << formatting::Format(
"Found key {:?} on page {:L}, search depth {}, value: \n{@BYELLOW}{}{@RESET}",
Expand Down
2 changes: 1 addition & 1 deletion include/NeverSQL/data/Page.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class Page {
template<typename T>
requires std::is_trivially_copyable_v<T>
page_size_t WriteToPage(page_size_t offset, const T& data) {
std::span<const std::byte> view(reinterpret_cast<const std::byte*>(&data), sizeof(T));
std::span view(reinterpret_cast<const std::byte*>(&data), sizeof(T));
return WriteToPage(offset, view);
}

Expand Down
Loading

0 comments on commit fa0b71d

Please sign in to comment.