fix: Use specific package versions for better reproducibility

.github/workflows/docker.yml

@@ -4,36 +4,57 @@ on: [push]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
     container:
         image: ohhmm/openmind:latest
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
 
     steps:
       - uses: actions/checkout@v3
-      - name: Check for dockerenv file
-        run: (ls /.dockerenv && echo Found dockerenv && which clang++) || (echo No dockerenv)
 
-      - name: Install Dependencies
+      - name: Install system dependencies
         run: |
-          emerge-webrsync || exit 1
-          emerge --getbinpkg=y --usepkg=y -q dev-libs/hiredis dev-util/pkgconfig dev-util/ninja dev-util/cmake dev-cpp/boost dev-util/ccache || exit 1
+          apt-get update
+          apt-get install -y redis-server libhiredis-dev
 
-      - name: Setup ccache
-        run: |
-          ccache --max-size=2G
-          ccache --zero-stats
+      - name: Cache dependencies and build
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.ccache
+            ./build
+          key: ${{ runner.os }}-${{ hashFiles('**/CMakeLists.txt', 'vcpkg.json') }}
+          restore-keys: ${{ runner.os }}-
 
-      - name: Create Build Dir
-        run: cmake -E make_directory ./build
+      - name: Check for dockerenv file
+        run: (ls /.dockerenv && echo Found dockerenv && which clang++) || (echo No dockerenv)
+
+      - name: Initialize vcpkg
+        run: |
+          git clone https://github.com/Microsoft/vcpkg.git
+          ./vcpkg/bootstrap-vcpkg.sh
+          ./vcpkg/vcpkg integrate install
 
       - name: Configure
         working-directory: ./build
         env:
-          CC: ccache /usr/lib/llvm/17/bin/clang-17
-          CXX: ccache /usr/lib/llvm/17/bin/clang++-17
-        run: cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug \
-          -DOPENMIND_BUILD_SAMPLES=OFF -DOPENMIND_STORAGE_REDIS=ON -DBOOST_TEST_NO_AUTO_LINK=ON -DBoost_USE_STATIC_LIBS=ON
+          CCACHE_DIR: ~/.ccache
+          CCACHE_COMPRESS: "true"
+          CCACHE_MAXSIZE: "2G"
+          REDIS_HOST: redis
+          REDIS_PORT: 6379
+        run: |
+          cmake -E make_directory .
+          cmake .. -G Ninja \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DCMAKE_TOOLCHAIN_FILE=${{github.workspace}}/vcpkg/scripts/buildsystems/vcpkg.cmake \
+            -DOPENMIND_BUILD_SAMPLES=OFF \
+            -DOPENMIND_STORAGE_REDIS=ON \
+            -DOPENMIND_USE_VCPKG=ON
 
       - name: Install prerequisites
         working-directory: ./build
@@ -48,4 +69,4 @@ jobs:
 
       - name: Check
         working-directory: ./build
-        run: ctest . -j`nproc` -E ts --rerun-failed --output-on-failure
+        run: ctest . -j`nproc` -E "(ts|image_codec_test)" --rerun-failed --output-on-failure

.github/workflows/msvc.yml

@@ -6,6 +6,12 @@ jobs:
   build:
     runs-on: windows-latest
 
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+
     steps:
     - uses: actions/checkout@v3
 
@@ -14,12 +20,16 @@ jobs:
         git clone https://github.com/Microsoft/vcpkg.git
         .\vcpkg\bootstrap-vcpkg.bat
         .\vcpkg\vcpkg integrate install
+        .\vcpkg\vcpkg install hiredis:x64-windows
 
     - name: Create Build Directory
       run: mkdir build
 
     - name: Configure CMake
       working-directory: ./build
+      env:
+        REDIS_HOST: localhost
+        REDIS_PORT: 6379
       run: |
         cmake .. -G "Visual Studio 17 2022" -A x64 `
           -DCMAKE_TOOLCHAIN_FILE="${{github.workspace}}/vcpkg/scripts/buildsystems/vcpkg.cmake" `
@@ -33,4 +43,4 @@ jobs:
 
     - name: Test
       working-directory: ./build
-      run: ctest -C Release -j 4 -E "image_codec_test|ts" --rerun-failed --output-on-failure
+      run: ctest -C Release -j 4 -E "(ts|image_codec_test)" --rerun-failed --output-on-failure

CMakeLists.txt

@@ -403,9 +403,18 @@ if(NOT MSVC AND NOT OPENMIND_USE_VCPKG AND NOT OPENMIND_USE_CONAN)
 endif()
 
 # Find hiredis package
+option(OPENMIND_STORAGE_REDIS "Enable Redis storage backend" ON)
 if(OPENMIND_STORAGE_REDIS)
     find_package(PkgConfig REQUIRED)
     pkg_check_modules(HIREDIS REQUIRED hiredis)
+
+    # Set Redis connection defaults from environment
+    if(NOT DEFINED ENV{REDIS_HOST})
+        set(ENV{REDIS_HOST} "localhost")
+    endif()
+    if(NOT DEFINED ENV{REDIS_PORT})
+        set(ENV{REDIS_PORT} "6379")
+    endif()
 endif()
 
 IF (Boost_FOUND)

omnn/rt/storage/CMakeLists.txt

@@ -21,5 +21,12 @@ target_include_directories(redis_cache
     ${CMAKE_CURRENT_SOURCE_DIR}
 )
 
+# Configure Redis connection settings from environment
+target_compile_definitions(redis_cache
+    PRIVATE
+    REDIS_HOST="$ENV{REDIS_HOST}"
+    REDIS_PORT=$ENV{REDIS_PORT}
+)
+
 # Add tests subdirectory
 add_subdirectory(tests)

omnn/rt/storage/RedisCache.cpp

@@ -9,8 +9,11 @@ namespace storage {
 
 RedisCache::RedisCache(const std::string& host, int port, int timeout_ms,
                        int retry_count, int retry_delay_ms)
-    : host_(host), port_(port), timeout_ms_(timeout_ms),
-      retry_count_(retry_count), retry_delay_ms_(retry_delay_ms),
+    : host_(host.empty() ? REDIS_HOST : host),
+      port_(port == 0 ? REDIS_PORT : port),
+      timeout_ms_(timeout_ms),
+      retry_count_(retry_count),
+      retry_delay_ms_(retry_delay_ms),
       context_(nullptr) {
     if (!Connect()) {
         throw std::runtime_error("Failed to connect to Redis server");
@@ -31,13 +34,24 @@ bool RedisCache::Connect() {
 
     context_ = redisConnectWithTimeout(host_.c_str(), port_, timeout);
 
-    if (context_ == nullptr || context_->err) {
-        if (context_) {
-            redisFree(context_);
-            context_ = nullptr;
-        }
+    if (context_ == nullptr) {
+        return false;
+    }
+
+    if (context_->err) {
+        std::string error = context_->errstr ? context_->errstr : "Unknown error";
+        redisFree(context_);
+        context_ = nullptr;
+        return false;
+    }
+
+    // Set socket timeout for operations
+    if (redisSetTimeout(context_, timeout) != REDIS_OK) {
+        redisFree(context_);
+        context_ = nullptr;
         return false;
     }
+
     return true;
 }
 
@@ -54,9 +68,12 @@ bool RedisCache::RetryOperation(const std::function<bool()>& operation) {
             return true;
         }
 
-        if (!IsConnected() && !Connect()) {
-            std::this_thread::sleep_for(std::chrono::milliseconds(retry_delay_ms_));
-            continue;
+        // If connection is lost, attempt to reconnect
+        if (!IsConnected()) {
+            Disconnect();  // Ensure cleanup before reconnect
+            if (Connect()) {
+                continue;  // Retry immediately after successful reconnect
+            }
         }
 
         std::this_thread::sleep_for(std::chrono::milliseconds(retry_delay_ms_));
@@ -79,6 +96,32 @@ bool RedisCache::Set(const std::string& key, const std::string& value) {
     });
 }
 
+bool RedisCache::Set(const std::string& key, const std::string& value, int expire_seconds) {
+    return RetryOperation([this, &key, &value, expire_seconds]() {
+        if (!IsConnected()) return false;
+
+        redisReply* reply = (redisReply*)redisCommand(context_, "SET %s %s",
+                                                    key.c_str(), value.c_str());
+        if (!reply) return false;
+
+        bool success = (reply->type == REDIS_REPLY_STATUS &&
+                       strcasecmp(reply->str, "OK") == 0);
+        freeReplyObject(reply);
+
+        if (success) {
+            reply = (redisReply*)redisCommand(context_, "EXPIRE %s %d",
+                                           key.c_str(), expire_seconds);
+            if (reply) {
+                success = (reply->type == REDIS_REPLY_INTEGER && reply->integer == 1);
+                freeReplyObject(reply);
+            } else {
+                success = false;
+            }
+        }
+        return success;
+    });
+}
+
 bool RedisCache::Get(const std::string& key, std::string& value) {
     return RetryOperation([this, &key, &value]() {
         if (!IsConnected()) return false;
@@ -116,6 +159,94 @@ bool RedisCache::IsConnected() const {
     return context_ && !context_->err;
 }
 
+bool RedisCache::SetBinary(const std::string& key, const void* data, size_t size) {
+    if (!data && size > 0) return false;
+    return RetryOperation([this, &key, data, size]() {
+        if (!IsConnected()) return false;
+        redisReply* reply = (redisReply*)redisCommand(context_, "SET %s %b", key.c_str(), data ? data : "", size);
+        if (!reply) return false;
+        bool success = (reply->type == REDIS_REPLY_STATUS && strcasecmp(reply->str, "OK") == 0);
+        freeReplyObject(reply);
+        return success;
+    });
+}
+
+bool RedisCache::PipelineWithExpire(const std::vector<std::pair<std::string, std::string>>& operations, int expire_seconds) {
+    return RetryOperation([this, &operations, expire_seconds]() {
+        if (!IsConnected()) return false;
+
+        // Pipeline SET and EXPIRE commands
+        for (const auto& op : operations) {
+            redisAppendCommand(context_, "SET %s %s", op.first.c_str(), op.second.c_str());
+            redisAppendCommand(context_, "EXPIRE %s %d", op.first.c_str(), expire_seconds);
+        }
+
+        // Execute pipeline
+        bool success = true;
+        for (size_t i = 0; i < operations.size() * 2; ++i) {
+            redisReply* reply;
+            if (redisGetReply(context_, (void**)&reply) != REDIS_OK) {
+                success = false;
+                break;
+            }
+            bool cmd_success = (i % 2 == 0) ?
+                (reply->type == REDIS_REPLY_STATUS && strcasecmp(reply->str, "OK") == 0) :
+                (reply->type == REDIS_REPLY_INTEGER && reply->integer == 1);
+            freeReplyObject(reply);
+            if (!cmd_success) {
+                success = false;
+                break;
+            }
+        }
+        return success;
+    });
+}
+
+bool RedisCache::Pipeline(const std::vector<std::pair<std::string, std::string>>& operations) {
+    return RetryOperation([this, &operations]() {
+        if (!IsConnected()) return false;
+
+        // Start pipeline
+        for (const auto& op : operations) {
+            redisAppendCommand(context_, "SET %s %s",
+                             op.first.c_str(), op.second.c_str());
+        }
+
+        // Execute pipeline
+        bool success = true;
+        for (size_t i = 0; i < operations.size(); ++i) {
+            redisReply* reply;
+            if (redisGetReply(context_, (void**)&reply) != REDIS_OK) {
+                success = false;
+                break;
+            }
+            bool cmd_success = (reply->type == REDIS_REPLY_STATUS &&
+                              strcasecmp(reply->str, "OK") == 0);
+            freeReplyObject(reply);
+            if (!cmd_success) {
+                success = false;
+                break;
+            }
+        }
+        return success;
+    });
+}
+
+bool RedisCache::GetBinary(const std::string& key, std::vector<char>& data) {
+    return RetryOperation([this, &key, &data]() {
+        if (!IsConnected()) return false;
+        redisReply* reply = (redisReply*)redisCommand(context_, "GET %s", key.c_str());
+        if (!reply) return false;
+        bool success = false;
+        if (reply->type == REDIS_REPLY_STRING) {
+            data.assign(reply->str, reply->str + reply->len);
+            success = true;
+        }
+        freeReplyObject(reply);
+        return success;
+    });
+}
+
 } // namespace storage
 } // namespace rt
 } // namespace omnn

omnn/rt/storage/RedisCache.h

@@ -2,23 +2,45 @@
 
 #include <string>
 #include <memory>
+#include <vector>
+#include <chrono>
+#include <thread>
+#include <functional>
 #include <hiredis/hiredis.h>
 #include "CacheBase.h"
 
+#ifndef REDIS_HOST
+#define REDIS_HOST "localhost"
+#endif
+
+#ifndef REDIS_PORT
+#define REDIS_PORT 6379
+#endif
+
 namespace omnn {
 namespace rt {
 namespace storage {
 
 class RedisCache : public CacheBase {
 public:
-    RedisCache(const std::string& host = "localhost", int port = 6379,
+    RedisCache(const std::string& host = "", int port = 0,
                int timeout_ms = 1000, int retry_count = 5, int retry_delay_ms = 1000);
     ~RedisCache();
 
     bool Set(const std::string& key, const std::string& value) override;
+    bool Set(const std::string& key, const std::string& value, int expire_seconds);
     bool Get(const std::string& key, std::string& value) override;
     bool Clear() override;
     bool IsConnected() const;
+    bool Pipeline(const std::vector<std::pair<std::string, std::string>>& operations);
+    bool PipelineWithExpire(const std::vector<std::pair<std::string, std::string>>& operations, int expire_seconds);
+
+    // Binary data handling
+    bool SetBinary(const std::string& key, const void* data, size_t size);
+    bool GetBinary(const std::string& key, std::vector<char>& data);
+
+    // For testing purposes only
+    redisContext* GetContext() const { return context_; }
 
 private:
     bool Connect();

omnn/rt/storage/tests/CMakeLists.txt

@@ -13,6 +13,7 @@ add_test(NAME redis_cache_test COMMAND redis_cache_test)
 
 # Set test properties
 set_tests_properties(redis_cache_test PROPERTIES
-    ENVIRONMENT "OPENMIND_TEST_REDIS_RETRY_COUNT=5;OPENMIND_TEST_REDIS_RETRY_DELAY=1000"
+    ENVIRONMENT "REDIS_HOST=$ENV{REDIS_HOST};REDIS_PORT=$ENV{REDIS_PORT};OPENMIND_TEST_REDIS_RETRY_COUNT=5;OPENMIND_TEST_REDIS_RETRY_DELAY=1000"
     TIMEOUT 60
+    FIXTURES_REQUIRED "REDIS_SERVICE"
 )