[test] Skip sharding tests

Restream · Feb 4, 2025 · a068f66 · a068f66
1 parent 1c82af8
commit a068f66
Show file tree

Hide file tree

Showing 167 changed files with 3,818 additions and 3,532 deletions.
diff --git a/.github/workflows/install_grpc.sh b/.github/workflows/install_grpc.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-git clone --branch v1.44.0 https://github.com/grpc/grpc
+git clone --branch v1.54.3 https://github.com/grpc/grpc
 cd grpc
 git submodule update --init
 mkdir -p cmake/build

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -2,64 +2,13 @@ name: build
 
 on: push
 jobs:
-  build-windows-latest:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        compiler: ["Visual Studio 17 2022"]
-      fail-fast: false
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Build Reindexer
-        run: |
-          mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" .. -Ax64
-          cmake --build . --config Release
-          cmake --build . --config Release --target face
-          cmake --build . --config Release --target swagger
-          cpack
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: windows-latest
-          path: build/*.exe
-          if-no-files-found: error
-
-  build-windows-2019:
-    runs-on: windows-2019
-    strategy:
-      matrix:
-        compiler: ["Visual Studio 16 2019", "MinGW Makefiles"]
-      fail-fast: false
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Build Reindexer
-        run: |
-          mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" ..
-          cmake --build . --config Release
-          cmake --build . --config Release --target face
-          cmake --build . --config Release --target swagger
-          cpack
-
   build:
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04]
         include:
-          - os: ubuntu-24.04
-            cc: gcc-12
-            cxx: g++-12
           - os: ubuntu-latest
             sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
-          - os: ubuntu-latest
-            sanitizer: TSAN
-            cc: gcc-12
-            cxx: g++-12
       fail-fast: false
     runs-on: ${{matrix.os}}
     env:
@@ -85,7 +34,7 @@ jobs:
           export CPLUS_INCLUDE_PATH=$GITHUB_WORKSPACE/grpc/third_party/abseil-cpp
           mkdir build && cd build
           if [[ $OS == ubuntu-latest ]]; then
-            cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
+            cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On -DGTEST_WORKERS=4 -DGTEST_SKIP_SHARDING=ON ..
           elif [[ $OS == macos* ]]; then
             cmake -DGH_CI_OSX=ON ..
           else
@@ -103,6 +52,7 @@ jobs:
         run: |
           find ./build/ -name CMakeFiles | xargs -n1 rm -rf
           rm -rf ./build/_CPack_Packages ./build/cpp_src/server/reindexer_server_resources
+          rm -rf ./build/cpp_src/cmd/reindexer_tool ./build/cpp_src/gtests/tests/fuzzing
           tar -cvf artifacts.tar build/ bindings/builtin/builtin_posix.go bindings/builtinserver/builtinserver_posix.go dependencies.sh
       - name: Archive Artifacts
         uses: actions/upload-artifact@v4
@@ -114,25 +64,14 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-22.04, ubuntu-24.04, macos-13]
-        test: ['C++', 'GO']
+        os: [ubuntu-22.04]
+        test: ['C++']
         include:
           - os: ubuntu-latest
             sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
             test: 'C++'
-          - os: ubuntu-latest
-            sanitizer: ASAN
-            test: 'GO'
-#         - os: ubuntu-latest
-#           sanitizer: TSAN
-#           cc: gcc-12
-#           cxx: g++-12
-#           test: 'C++'
-          - os: ubuntu-latest
-            sanitizer: TSAN
-            test: 'GO'
+            cc: gcc-12
+            cxx: g++-12
       fail-fast: false
     runs-on: ${{matrix.os}}
     needs: build
@@ -175,6 +114,15 @@ jobs:
           elif [[ $OS == ubuntu* ]]; then
             ./.github/workflows/install_gtest_parallel.sh
           fi
+      - name: Increase swapfile
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.test == 'C++' }}
+        run: |
+          sudo swapoff -a
+          sudo fallocate -l 8G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          sudo swapon --show
       - name: Tests
         if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
         run: |
@@ -189,13 +137,14 @@ jobs:
             fi
           else
             cd build
+            free
             ctest --verbose
           fi
 
   test-pyreindexer:
     strategy:
       matrix:
-        os: [ubuntu-22.04, ubuntu-24.04]
+        os: [ubuntu-22.04]
       fail-fast: false
     runs-on: ${{matrix.os}}
     needs: build

diff --git a/bindings/consts.go b/bindings/consts.go
@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v4.19.0"
+const ReindexerVersion = "v4.20.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

diff --git a/changelog.md b/changelog.md
@@ -1,7 +1,32 @@
+# Version 4.20.0 (04.02.2025)
+## Core
+- [fea] Optimized indexed strings memory layot (each unique indexed string now requires 20-36 bytes less memeory, depending on platform)
+- [fea] Optimized unbuilt btree-index selection
+- [fea] Optimized selections with large documents count in results
+- [fea] Reduced allocations count in inserts/upserts and select-queries
+- [fea] Changed index compatibility check in `AddIndex`: now `dense` and `regular` indexes are treated compatible
+- [fix] Fixed parsing for [EQUAL_POSITION](readme.md#search-in-array-fields-with-matching-array-indexes) with brackets in SQL
+
+## Replication
+- [fea] Improved full sync performance for namespaces with large amount of empty document
+
+## Reindexer server
+- [fix] Added more TCP-requests format checks
+- [fix] Fixed [server.yml](cpp_src/server/contrib/server.yml) format errors
+
+## Go connector
+- [fea] Added support for absolute path on Windowos platforms in `builtin`-binding
+
+## Build
+- [fix] Fixed build for ARM docker image
+
+## Ported
+- [fea/fix] Ported all the fixes and features from [v3.31.0](https://github.com/Restream/reindexer/releases/tag/v3.31.0)
+
 # Version 4.19.0 (17.12.2024)
 
 ## Core
-- [fea] Added automatic masking for reindexer user's credentials in log files and cluster/sharding JSON's (except for `async_replication` configs in `#config`-namespace)
+- [fea] Added automatic masking for reindexer user's credentials in log files and cluster/sharding JSON's (except for `async_replication`-config in `#config`-namespace)
 - [fix] Fixed assertion on attempt to use 'null'-values with `=`, `IN()`, `<`, `>`, `<=`, `>=` and `RANGE()` operators
 
 ## Replication

diff --git a/cjson/decoder.go b/cjson/decoder.go
@@ -36,8 +36,7 @@ func fieldByTag(t reflect.Type, tag string) (result reflect.StructField, ok bool
 	}
 	for i := 0; i < t.NumField(); i++ {
 		result = t.Field(i)
-		if ftag := result.Tag.Get("json"); len(ftag) > 0 {
-			ftag, _ = splitStr(ftag, ',')
+		if ftag, _ := splitStr(result.Tag.Get("json"), ','); len(ftag) > 0 {
 			if tag == ftag {
 				return result, true
 			}

diff --git a/clang-tidy/run-clang-tidy-18.py → clang-tidy/run-clang-tidy-19.py b/clang-tidy/run-clang-tidy-18.py → clang-tidy/run-clang-tidy-19.py
@@ -251,10 +251,10 @@ def main():
                       action='store_true', help='allow alpha checkers from '
                                                 'clang-analyzer.')
   parser.add_argument('-clang-tidy-binary', metavar='PATH',
-                      default='clang-tidy-18',
+                      default='clang-tidy-19',
                       help='path to clang-tidy binary')
   parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
-                      default='clang-apply-replacements-18',
+                      default='clang-apply-replacements-19',
                       help='path to clang-apply-replacements binary')
   parser.add_argument('-checks', default=None,
                       help='checks filter, when not specified, use clang-tidy '

diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt
@@ -50,7 +50,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "4.19.0")
+set (REINDEXER_VERSION_DEFAULT "4.20.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -72,6 +72,13 @@ if(MSVC)
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -Zi")
   set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG -Zi")
   set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG -Zi")
+elseif(WITH_ASAN)
+  # Using O2 instead of O3 to build a bit faster.
+  # Also this allows to avoid SEGFAULT in libasan.so during coroutines interaction on CentOS7 (gcc-12).
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g1")
+  set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g1")
+  set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
+  set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG")
 else()
   set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g1")
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g1")

diff --git a/cpp_src/client/itemimplbase.h b/cpp_src/client/itemimplbase.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <deque>
 #include <vector>
 #include "core/keyvalue/variant.h"
 #include "core/payload/payloadiface.h"
@@ -76,7 +75,7 @@ class ItemImplBase {
 
 	std::vector<std::string> precepts_;
 	bool unsafe_ = false;
-	std::deque<std::string> holder_;
+	h_vector<key_string, 16> holder_;
 	std::vector<std::unique_ptr<char[]>> largeJSONStrings_;
 };
 

diff --git a/cpp_src/client/queryresults.h b/cpp_src/client/queryresults.h
@@ -37,9 +37,8 @@ class QueryResults {
 
 	class Iterator : public CoroQueryResults::Iterator {
 	public:
-		Iterator(const QueryResults* r, const CoroQueryResults* qr, int idx, int pos, int nextPos,
-				 ResultSerializer::ItemParams itemParams) noexcept
-			: CoroQueryResults::Iterator{qr, idx, pos, nextPos, itemParams, {}}, r_(r) {}
+		Iterator(const QueryResults& r, int idx, int pos, int nextPos, ResultSerializer::ItemParams itemParams) noexcept
+			: CoroQueryResults::Iterator{&r.results_, idx, pos, nextPos, itemParams, {}}, r_(&r) {}
 		Iterator& operator*() { return *this; }
 		Iterator& operator++() noexcept {
 			try {
@@ -57,11 +56,12 @@ class QueryResults {
 			}
 			return *this;
 		}
+
 		const QueryResults* r_;
 	};
 
-	Iterator begin() const noexcept { return Iterator{this, &results_, 0, 0, 0, {}}; }
-	Iterator end() const noexcept { return Iterator{this, &results_, int(results_.Count()), 0, 0, {}}; }
+	Iterator begin() const noexcept { return Iterator{*this, 0, 0, 0, {}}; }
+	Iterator end() const noexcept { return Iterator{*this, int(results_.Count()), 0, 0, {}}; }
 
 	size_t Count() const noexcept { return results_.Count(); }
 	int TotalCount() const noexcept { return results_.TotalCount(); }