ci: Improve Memurai service installation and path handling

.github/workflows/msvc.yml

@@ -0,0 +1,136 @@
+name: C/C++ CI MSVC
+
+on: [push]
+
+jobs:
+  build-on-windows:
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ilammy/msvc-dev-cmd@v1
+    - name: Cache Boost dependencies
+      uses: actions/cache@v4
+      with:
+        path: |
+          ${{ github.workspace }}/build/CMakeFiles
+          C:/Boost
+          ${{ github.workspace }}/build/bin/Db*.solutions
+        key: ${{ runner.os }}-boost-${{ hashFiles('**/vcpkg.json') }}
+        restore-keys: |
+          ${{ runner.os }}-boost-
+
+    - name: Create Build Dir
+      run: cmake -E make_directory ${{github.workspace}}/build
+
+    - name: Install Redis Dependencies
+      run: |
+        vcpkg install hiredis:x64-windows
+        choco install memurai-developer.install --no-progress --params="'/Port:6379'"
+
+        # Refresh environment variables and verify installation paths
+        $env:PATH = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+        refreshenv
+
+        # Use installation directory directly
+        $memuraiPath = "C:\Program Files\Memurai\memurai-developer.exe"
+        $memuraiCli = "C:\Program Files\Memurai\memurai-cli.exe"
+
+        # Verify executable exists
+        if (-not (Test-Path $memuraiPath)) {
+            Write-Error "Memurai executable not found at: $memuraiPath"
+            Get-ChildItem "C:\Program Files\Memurai\" -Recurse
+            exit 1
+        }
+
+        Write-Host "Configuring Memurai service..."
+        if (Get-Service -Name "memurai" -ErrorAction SilentlyContinue) {
+            Write-Host "Stopping existing Memurai service..."
+            Stop-Service -Name "memurai" -Force
+            Start-Sleep -s 5
+        }
+
+        Write-Host "Installing Memurai service..."
+        Start-Process -FilePath $memuraiPath -ArgumentList "--service-install", "--port", "6379" -Wait -NoNewWindow
+        if ($LASTEXITCODE -ne 0) {
+            Write-Error "Failed to install Memurai service"
+            exit 1
+        }
+
+        Write-Host "Starting Memurai service..."
+        Start-Service -Name "memurai"
+        Start-Sleep -s 10
+
+        Write-Host "Verifying Memurai service..."
+        $maxRetries = 10
+        $retryDelay = 3
+        $success = $false
+
+        for ($i = 1; $i -le $maxRetries; $i++) {
+            Write-Host "Verification attempt $i of $maxRetries..."
+            try {
+                $status = Get-Service -Name "memurai" -ErrorAction Stop
+                if ($status.Status -eq "Running") {
+                    $ping = & $memuraiCli ping
+                    if ($ping -eq "PONG") {
+                        Write-Host "Memurai service verified and responding!"
+                        $success = $true
+                        break
+                    }
+                }
+            } catch {
+                Write-Warning "Service check failed: $_"
+            }
+            Start-Sleep -s $retryDelay
+        }
+
+        if (-not $success) {
+            Write-Error "Failed to verify Memurai service after $maxRetries attempts"
+            Get-Service -Name "memurai" | Format-List *
+            Get-EventLog -LogName Application -Source "Memurai" -Newest 10
+            exit 1
+        }
+
+        # Set environment variables for tests
+        echo "OPENMIND_TEST_REDIS_RETRY_COUNT=10" >> $env:GITHUB_ENV
+        echo "OPENMIND_TEST_REDIS_RETRY_DELAY=3000" >> $env:GITHUB_ENV
+
+    - name: Configure
+      working-directory: ${{github.workspace}}/build
+      env:
+        CC: cl
+      run: cmake ${{github.workspace}} -DOPENMIND_USE_VCPKG=NO -DOPENMIND_BUILD_SAMPLES=OFF -DOPENMIND_BUILD_TESTS=ON -G "Ninja Multi-Config" -D CMAKE_C_COMPILER="cl.exe" -D CMAKE_CXX_COMPILER="cl.exe" -Dleveldb_TAG:STRING="1.23" -DOPENMIND_MATH_USE_LEVELDB_CACHE=NO -DOPENMIND_STORAGE_LEVELDB=NO -DOPENMIND_STORAGE_REDIS=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+    - name: Install prerequisites
+      run: cmake --build ${{github.workspace}}/build --target prerequisites --config Release
+
+    - name: Reconfigure to detect newly installed prerequisites
+      working-directory: ${{github.workspace}}/build
+      run: cmake .
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
+
+    - name: Build Debug
+      run: cmake --build ${{github.workspace}}/build --config Debug
+
+    - name: Build Release
+      run: cmake --build ${{github.workspace}}/build --config Release
+
+    - name: make check Release
+      working-directory: ${{github.workspace}}/build
+      env:
+        OPENMIND_TEST_REDIS_RETRY_COUNT: 10
+        OPENMIND_TEST_REDIS_RETRY_DELAY: 3000
+      run: ctest --timeout 1024 -C Release -j ${{steps.cpu-cores.outputs.count}} -E "ts" --output-on-failure
+
+    - name: make check Debug
+      working-directory: ${{github.workspace}}/build
+      env:
+        OPENMIND_TEST_REDIS_RETRY_COUNT: 10
+        OPENMIND_TEST_REDIS_RETRY_DELAY: 3000
+      run: ctest --timeout 2048 -C Debug -j ${{steps.cpu-cores.outputs.count}} -E "ts" --output-on-failure
+
+    - name: Install
+      working-directory: ${{github.workspace}}/build
+      run: cmake --build ${{github.workspace}}/build --target install --config Release

.github/workflows/vcpkg.yml

@@ -30,7 +30,8 @@ jobs:
       if: runner.os == 'Linux'
       run: |
         sudo apt-get update
-        sudo apt-get install -y build-essential cmake ninja-build autoconf automake autoconf-archive
+        sudo apt-get install -y build-essential cmake ninja-build autoconf automake autoconf-archive redis-server
+        sudo systemctl start redis-server
         cmake --version
         ninja --version
         gcc --version
@@ -43,7 +44,8 @@ jobs:
       run: |
         echo "VCPKG_PYTHON3=$VCPKG_PYTHON3" >> $GITHUB_ENV
         brew update
-        brew install cmake ninja autoconf automake autoconf-archive
+        brew install cmake ninja autoconf automake autoconf-archive redis
+        brew services start redis
         cmake --version
         ninja --version
         clang --version

.github/workflows/windows.yml

@@ -23,11 +23,45 @@ jobs:
     - name: Create Build Dir
       run: cmake -E make_directory ${{github.workspace}}/build
 
+    - name: Install Redis Dependencies
+      run: |
+        vcpkg install hiredis:x64-windows
+        choco install memurai-developer.install --no-progress
+        $env:PATH = [System.Environment]::GetEnvironmentVariable("Path","Machine")
+        $memuraiPath = "C:\Program Files\Memurai\memurai-developer.exe"
+        Write-Host "Stopping any existing Memurai service..."
+        & $memuraiPath --service-stop 2>&1 | Out-Null
+        Write-Host "Installing Memurai service..."
+        & $memuraiPath --service-install
+        Write-Host "Starting Memurai service..."
+        & $memuraiPath --service-start
+        Write-Host "Waiting for service to be ready..."
+        Start-Sleep -s 15  # Increased wait time for service startup
+        $retryCount = 0
+        do {
+            try {
+                $ping = & "C:\Program Files\Memurai\memurai-cli.exe" ping
+                if ($ping -eq "PONG") {
+                    Write-Host "Memurai service is responding!"
+                    break
+                }
+            } catch {
+                Write-Warning "Ping attempt failed: $_"
+            }
+            Start-Sleep -s 3
+            $retryCount++
+            Write-Host "Retry attempt $retryCount..."
+        } while ($retryCount -lt 5)
+        if ($ping -ne "PONG") {
+            Write-Error "Memurai server failed to respond after multiple attempts"
+            exit 1
+        }
+
     - name: Configure
       working-directory: ${{github.workspace}}/build
       env:
         CC: cl
-      run: cmake ${{github.workspace}} -DOPENMIND_USE_VCPKG=NO -DOPENMIND_BUILD_SAMPLES=OFF -DOPENMIND_BUILD_TESTS=ON -G "Ninja Multi-Config" -D CMAKE_C_COMPILER="cl.exe" -D CMAKE_CXX_COMPILER="cl.exe" -Dleveldb_TAG:STRING="1.23" -DOPENMIND_MATH_USE_LEVELDB_CACHE=NO -DOPENMIND_STORAGE_LEVELDB=NO
+      run: cmake ${{github.workspace}} -DOPENMIND_USE_VCPKG=NO -DOPENMIND_BUILD_SAMPLES=OFF -DOPENMIND_BUILD_TESTS=ON -G "Ninja Multi-Config" -D CMAKE_C_COMPILER="cl.exe" -D CMAKE_CXX_COMPILER="cl.exe" -Dleveldb_TAG:STRING="1.23" -DOPENMIND_MATH_USE_LEVELDB_CACHE=NO -DOPENMIND_STORAGE_LEVELDB=NO -DOPENMIND_STORAGE_REDIS=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake
 
     - name: Install prerequisites
       run: cmake --build ${{github.workspace}}/build --target prerequisites --config Release

omnn/rt/storage/CMakeLists.txt

@@ -0,0 +1,27 @@
+# Storage module CMake configuration
+set(DEPENDENCIES
+    rt
+)
+
+# Add Redis dependency
+if(OPENMIND_STORAGE_REDIS)
+    find_package(hiredis REQUIRED)
+    list(APPEND DEPENDENCIES hiredis::hiredis)
+    add_definitions(-DOPENMIND_STORAGE_REDIS)
+    if(WIN32)
+        add_definitions(-DOPENMIND_STORAGE_REDIS_MEMURAI)
+    endif()
+endif()
+
+# Add LevelDB dependency
+if(OPENMIND_STORAGE_LEVELDB)
+    find_package(leveldb REQUIRED)
+    list(APPEND DEPENDENCIES leveldb)
+    add_definitions(-DOPENMIND_STORAGE_LEVELDB)
+endif()
+
+lib(${DEPENDENCIES})
+
+if(OPENMIND_BUILD_TESTS)
+    add_subdirectory(tests)
+endif()

omnn/rt/storage/RedisCache.cpp

@@ -0,0 +1,148 @@
+#include "RedisCache.h"
+
+#ifdef OPENMIND_STORAGE_REDIS
+
+#include <hiredis/hiredis.h>
+#include <stdexcept>
+#include <cstdarg>
+#include <chrono>
+#include <thread>
+
+#ifdef _WIN32
+#include <winsock2.h>
+#pragma comment(lib, "ws2_32.lib")
+#endif
+
+namespace omnn::rt::storage {
+
+RedisCache::RedisCache(const std::string_view& host, int port, int timeout_ms)
+    : _context(nullptr, redisFree)
+    , _host(host)
+    , _port(port)
+    , _timeout_ms(timeout_ms)
+{
+#ifdef _WIN32
+    static bool wsaInitialized = false;
+    if (!wsaInitialized) {
+        WSADATA wsaData;
+        if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) {
+            throw std::runtime_error("Failed to initialize Winsock");
+        }
+        wsaInitialized = true;
+    }
+#endif
+    ensureConnection();
+}
+
+RedisCache::~RedisCache() {
+#ifdef _WIN32
+    WSACleanup();
+#endif
+}
+
+void RedisCache::ensureConnection() {
+    if (_context) return;
+
+    int retries = 5;  // Increased from 3 to 5
+    std::string last_error;
+    const char* server_type =
+#ifdef OPENMIND_STORAGE_REDIS_MEMURAI
+        "Memurai";
+#else
+        "Redis";
+#endif
+
+    while (retries--) {
+        struct timeval timeout = { 0, _timeout_ms * 1000 };
+        redisContext* c = redisConnectWithTimeout(_host.c_str(), _port, timeout);
+
+        if (c && !c->err) {
+            // Test connection with PING
+            redisReply* reply = (redisReply*)redisCommand(c, "PING");
+            if (reply && reply->type == REDIS_REPLY_STATUS &&
+                std::string_view(reply->str, reply->len) == "PONG") {
+                freeReplyObject(reply);
+                _context.reset(c);
+                return;
+            }
+            if (reply) {
+                last_error = "PING failed: " + std::string(reply->str, reply->len);
+                freeReplyObject(reply);
+            } else {
+                last_error = "PING failed: no reply";
+            }
+            redisFree(c);
+        } else {
+            last_error = c ? c->errstr : "can't allocate redis context";
+            if (c) redisFree(c);
+        }
+
+        if (retries > 0) {
+            std::this_thread::sleep_for(std::chrono::seconds(3));  // Increased from 500ms to 2s
+        }
+    }
+
+    throw std::runtime_error(std::string(server_type) + " connection error after retries: " + last_error);
+}
+
+std::unique_ptr<redisReply, void(*)(redisReply*)> RedisCache::executeCommand(const char* format, ...) {
+    ensureConnection();
+
+    va_list ap;
+    va_start(ap, format);
+    redisReply* reply = (redisReply*)redisvCommand(_context.get(), format, ap);
+    va_end(ap);
+
+    if (!reply) {
+        std::string err = _context ? _context->errstr : "unknown error";
+        _context.reset();  // Reset connection on error to force reconnect
+        throw std::runtime_error("Redis command failed: " + err);
+    }
+
+    if (reply->type == REDIS_REPLY_ERROR) {
+        std::string err(reply->str, reply->len);
+        freeReplyObject(reply);
+        throw std::runtime_error("Redis command error: " + err);
+    }
+
+    auto deleter = reinterpret_cast<void(*)(redisReply*)>(freeReplyObject);
+    return std::unique_ptr<redisReply, void(*)(redisReply*)>(reply, deleter);
+}
+
+std::string RedisCache::GetOne(const std::string_view& key) {
+    auto reply = executeCommand("GET %b", key.data(), key.size());
+
+    if (reply->type == REDIS_REPLY_NIL) {
+        return "";
+    }
+
+    if (reply->type != REDIS_REPLY_STRING) {
+        throw std::runtime_error("Unexpected Redis reply type for GET");
+    }
+
+    return std::string(reply->str, reply->len);
+}
+
+bool RedisCache::Set(const std::string_view& key, const std::string_view& v) {
+    auto reply = executeCommand("SET %b %b", key.data(), key.size(), v.data(), v.size());
+
+    return reply->type == REDIS_REPLY_STATUS &&
+           std::string_view(reply->str, reply->len) == "OK";
+}
+
+bool RedisCache::Clear(const std::string_view& key) {
+    auto reply = executeCommand("DEL %b", key.data(), key.size());
+
+    return reply->type == REDIS_REPLY_INTEGER && reply->integer > 0;
+}
+
+bool RedisCache::ResetAllDB(const CacheBase::path_str_t& path) {
+    auto reply = executeCommand("FLUSHDB");
+
+    return reply->type == REDIS_REPLY_STATUS &&
+           std::string_view(reply->str, reply->len) == "OK";
+}
+
+} // namespace omnn::rt::storage
+
+#endif // OPENMIND_STORAGE_REDIS