From 7b0f2846df63701a9d44d72521d58244829e7afd Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Fri, 8 Nov 2024 10:48:12 -0800 Subject: [PATCH 1/4] fix compile for main --- cachelib/allocator/CMakeLists.txt | 11 ++++++++++- cachelib/allocator/memory/CompressedPtr.h | 2 +- cachelib/allocator/memory/tests/TestBase.h | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cachelib/allocator/CMakeLists.txt b/cachelib/allocator/CMakeLists.txt index 6103cdc823..483b41583e 100644 --- a/cachelib/allocator/CMakeLists.txt +++ b/cachelib/allocator/CMakeLists.txt @@ -27,7 +27,16 @@ add_library (cachelib_allocator ${SERIALIZE_THRIFT_FILES} ${DATASTRUCT_SERIALIZE_THRIFT_FILES} ${MEMORY_SERIALIZE_THRIFT_FILES} - CacheAllocator.cpp + CacheAllocatorLru2QCache.cpp + CacheAllocatorLru5B2QCache.cpp + CacheAllocatorLru5BCache.cpp + CacheAllocatorLru5BCacheWithSpinBuckets.cpp + CacheAllocatorLruCache.cpp + CacheAllocatorLruCacheWithSpinBuckets.cpp + CacheAllocatorTinyLFU5BCache.cpp + CacheAllocatorTinyLFUCache.cpp + CacheAllocatorWTinyLFU5BCache.cpp + CacheAllocatorWTinyLFUCache.cpp Cache.cpp CacheDetails.cpp CacheStats.cpp diff --git a/cachelib/allocator/memory/CompressedPtr.h b/cachelib/allocator/memory/CompressedPtr.h index b56cbf2ee8..b4be893e64 100644 --- a/cachelib/allocator/memory/CompressedPtr.h +++ b/cachelib/allocator/memory/CompressedPtr.h @@ -252,7 +252,7 @@ class CACHELIB_PACKED_ATTR CompressedPtr5B { : ptr_(compress(slabIdx, allocIdx, isMultiTiered, tid)), regionIdx_(getRegionIdx(slabIdx, isMultiTiered, tid)) {} - constexpr explicit CompressedPtr5B(PtrType ptr) noexcept + CompressedPtr5B(PtrType ptr) noexcept : ptr_(deserializePtr(ptr)), regionIdx_(deserializeRegion(ptr)) {} // number of bits for the Allocation offset in a slab. With slab size of 22 diff --git a/cachelib/allocator/memory/tests/TestBase.h b/cachelib/allocator/memory/tests/TestBase.h index 3534b14567..adb493965e 100644 --- a/cachelib/allocator/memory/tests/TestBase.h +++ b/cachelib/allocator/memory/tests/TestBase.h @@ -77,7 +77,7 @@ class AllocTestBase : public testing::Test { } template - CompressedPtrType::PtrType compress(CompressedPtrType ptr, + typename CompressedPtrType::PtrType compress(CompressedPtrType ptr, uint32_t slabIdx, uint32_t allocIdx, bool isMultiTiered) { From 58c839dbe6f718b02ec5505a7677456c7b6af848 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 20 Nov 2024 06:40:53 -0800 Subject: [PATCH 2/4] add in dependencies --- contrib/build-package.sh | 10 +++++++++- contrib/build.sh | 2 +- contrib/prerequisites-centos9.sh | 1 + contrib/prerequisites-ubuntu18.sh | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/contrib/build-package.sh b/contrib/build-package.sh index 406031bd40..088ba3c3f8 100755 --- a/contrib/build-package.sh +++ b/contrib/build-package.sh @@ -59,7 +59,7 @@ options: NAME: the dependency to build supported values are: zstd googlelog, googleflags, googletest, - fmt, sparsemap, + fmt, sparsemap, fastfloat, folly, fizz, wangle, mvfst, fbthrift, cachelib @@ -195,6 +195,14 @@ case "$1" in external_git_clone=yes ;; + fastfloat) + NAME=fastfloat + REPO=https://github.com/fastfloat/fast_float.git + REPODIR=cachelib/external/$NAME + SRCDIR=$REPODIR + external_git_clone=yes + ;; + folly) NAME=folly SRCDIR=cachelib/external/$NAME diff --git a/contrib/build.sh b/contrib/build.sh index b9de071bb7..6cb9765a2d 100755 --- a/contrib/build.sh +++ b/contrib/build.sh @@ -95,7 +95,7 @@ build_arch() build_dependencies() { - for pkg in zstd googleflags googlelog googletest sparsemap fmt folly fizz wangle mvfst fbthrift ; + for pkg in zstd googleflags googlelog googletest sparsemap fmt fastfloat folly fizz wangle mvfst fbthrift ; do # shellcheck disable=SC2086 ./contrib/build-package.sh $pass_params "$pkg" \ diff --git a/contrib/prerequisites-centos9.sh b/contrib/prerequisites-centos9.sh index b8b5478105..cea6ce6425 100755 --- a/contrib/prerequisites-centos9.sh +++ b/contrib/prerequisites-centos9.sh @@ -35,6 +35,7 @@ sudo dnf install -y \ gtest-devel \ libsodium-static \ libdwarf-static \ + xxhash \ numactl-devel diff --git a/contrib/prerequisites-ubuntu18.sh b/contrib/prerequisites-ubuntu18.sh index f8296f8935..41a05b2898 100755 --- a/contrib/prerequisites-ubuntu18.sh +++ b/contrib/prerequisites-ubuntu18.sh @@ -41,6 +41,7 @@ sudo apt-get install -y \ libdwarf-dev \ libsodium-dev \ libaio-dev \ + xxhash \ libnuma-dev # NOTE: From 47b06579d583e64504e108ba37ee7e940c1c882c Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 20 Nov 2024 06:48:36 -0800 Subject: [PATCH 3/4] add to docker containers --- .github/workflows/build-cachelib-centos-9.yml | 2 ++ .github/workflows/build-cachelib-rockylinux-9.yml | 2 ++ .github/workflows/build-cachelib-ubuntu-22.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/build-cachelib-centos-9.yml b/.github/workflows/build-cachelib-centos-9.yml index a47166f483..24973e012c 100644 --- a/.github/workflows/build-cachelib-centos-9.yml +++ b/.github/workflows/build-cachelib-centos-9.yml @@ -78,6 +78,8 @@ jobs: run: ./contrib/build-package.sh -j -v -i sparsemap - name: "Install dependency: fmt" run: ./contrib/build-package.sh -j -v -i fmt + - name: "Install dependency: fastfloat" + run: ./contrib/build-package.sh -j -v -i fastfloat - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly - name: "Install dependency: fizz" diff --git a/.github/workflows/build-cachelib-rockylinux-9.yml b/.github/workflows/build-cachelib-rockylinux-9.yml index 1741c0240c..3e6cec45c7 100644 --- a/.github/workflows/build-cachelib-rockylinux-9.yml +++ b/.github/workflows/build-cachelib-rockylinux-9.yml @@ -76,6 +76,8 @@ jobs: run: ./contrib/build-package.sh -j -v -i sparsemap - name: "Install dependency: fmt" run: ./contrib/build-package.sh -j -v -i fmt + - name: "Install dependency: fastfloat" + run: ./contrib/build-package.sh -j -v -i fastfloat - name: "Install dependency: folly" run: | # see: https://aur.archlinux.org/packages/folly#comment-862543 diff --git a/.github/workflows/build-cachelib-ubuntu-22.yml b/.github/workflows/build-cachelib-ubuntu-22.yml index e39359c69c..4de505f56e 100644 --- a/.github/workflows/build-cachelib-ubuntu-22.yml +++ b/.github/workflows/build-cachelib-ubuntu-22.yml @@ -74,6 +74,8 @@ jobs: run: ./contrib/build-package.sh -j -v -i sparsemap - name: "Install dependency: fmt" run: ./contrib/build-package.sh -j -v -i fmt + - name: "Install dependency: fastfloat" + run: ./contrib/build-package.sh -j -v -i fastfloat - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly - name: "Install dependency: fizz" From 4b6b02d87efcfb23e0a335eb554069a427750c5a Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 20 Nov 2024 07:13:10 -0800 Subject: [PATCH 4/4] actual dev libraries for xxhash --- contrib/prerequisites-centos9.sh | 2 +- contrib/prerequisites-rocky9.sh | 1 + contrib/prerequisites-ubuntu18.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/prerequisites-centos9.sh b/contrib/prerequisites-centos9.sh index cea6ce6425..8abddc3fcd 100755 --- a/contrib/prerequisites-centos9.sh +++ b/contrib/prerequisites-centos9.sh @@ -35,7 +35,7 @@ sudo dnf install -y \ gtest-devel \ libsodium-static \ libdwarf-static \ - xxhash \ + xxhash-devel \ numactl-devel diff --git a/contrib/prerequisites-rocky9.sh b/contrib/prerequisites-rocky9.sh index 06720aba2e..82c55cbad1 100755 --- a/contrib/prerequisites-rocky9.sh +++ b/contrib/prerequisites-rocky9.sh @@ -39,6 +39,7 @@ sudo dnf install -y \ libsodium-devel \ libaio-devel \ binutils-devel \ + xxhash-devel \ numactl-devel diff --git a/contrib/prerequisites-ubuntu18.sh b/contrib/prerequisites-ubuntu18.sh index 41a05b2898..5792a6b34d 100755 --- a/contrib/prerequisites-ubuntu18.sh +++ b/contrib/prerequisites-ubuntu18.sh @@ -41,7 +41,7 @@ sudo apt-get install -y \ libdwarf-dev \ libsodium-dev \ libaio-dev \ - xxhash \ + libxxhash-dev \ libnuma-dev # NOTE: