From cf98a37109902bfc2f7b59f20e65a1ea4ceb4d6f Mon Sep 17 00:00:00 2001 From: "daniel.barak" Date: Sun, 23 Feb 2025 11:34:41 +0200 Subject: [PATCH 1/8] caching of small files in redis --- .../apps/kaltura/lib/cache/kCacheManager.php | 1 + .../lib/storage/kFileSyncUtils.class.php | 72 ++++++++++++++++++- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/alpha/apps/kaltura/lib/cache/kCacheManager.php b/alpha/apps/kaltura/lib/cache/kCacheManager.php index 269d9be7750..15ca61f0588 100644 --- a/alpha/apps/kaltura/lib/cache/kCacheManager.php +++ b/alpha/apps/kaltura/lib/cache/kCacheManager.php @@ -9,6 +9,7 @@ class kCacheManager // cache types const CACHE_TYPE_PLAY_MANIFEST = 'playManifest'; const CACHE_TYPE_FILE_SYNC = 'fileSync'; + const CACHE_TYPE_SMALL_FILE_SYNC = 'smallfileSync'; const CACHE_TYPE_PERMISSION_MANAGER = 'permissionManager'; const CACHE_TYPE_QUERY_CACHE_KEYS = 'queryCacheKeys'; const CACHE_TYPE_QUERY_CACHE_QUERIES = 'queryCacheQueries'; diff --git a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php index 113fdb3257a..e90fa07db8d 100644 --- a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php +++ b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php @@ -19,6 +19,8 @@ class kFileSyncUtils implements kObjectChangedEventConsumer, kObjectAddedEventCo const SOURCE_TYPE_FILE = 'file'; const SOURCE_TYPE_HTTP = 'http'; + const FILE_SYNC_TYPES_TO_CACHE = 'file_sync_types_to_cache'; + const MAX_FILE_SIZE_TO_CACHE = 'max_file_size_to_cache'; /** * Contain all object types and sub types that should not be synced @@ -170,16 +172,41 @@ public static function dir_get_files(FileSyncKey $key, $strict = true) return $files; } + protected static function getCacheKey(FileSyncKey $key) + { + return self::CACHE_KEY_PREFIX . "{$key->object_id}_{$key->object_type}_{$key->object_sub_type}_{$key->version}"; + } + public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote_if_no_local = true , $strict = true , $max_file_size = 0 ) { + $cacheKey = self::getCacheKey($key); + $redisCacheStore = self::initSmallFileRedisInstance(); + if ($redisCacheStore) + { + try + { + $result = $redisCacheStore->doGet($cacheKey); + if ($result) + { + $uncompressedResult = gzuncompress($result); + $result = $uncompressedResult ? $uncompressedResult : $result; + KalturaLog::info("returning from redis cache, key [$cacheKey] size [" . strlen($result) . "]"); + return $result; + } + } + catch(Exception $e) + { + KalturaLog::info('File not found on Redis cache. [' . $e->getCode() . ' : ' . $e->getMessage() . ']'); + } + } + $cacheStore = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_FILE_SYNC); if ($cacheStore) { - $cacheKey = self::CACHE_KEY_PREFIX . "{$key->object_id}_{$key->object_type}_{$key->object_sub_type}_{$key->version}"; $result = $cacheStore->get($cacheKey); if ($result) { - KalturaLog::info("returning from cache, key [$cacheKey] size [".strlen($result)."]"); + KalturaLog::info("returning from cache, key [$cacheKey] size [" . strlen($result) . "]"); return $result; } } @@ -252,7 +279,46 @@ public static function file_put_contents ( FileSyncKey $key , $content , $strict kFile::filePutContents($fullPath , $content); self::setPermissions($fullPath); self::createSyncFileForKey($rootPath, $filePath, $key , $strict , !is_null($res), false, md5($content), kPathManager::getStorageProfileIdForKey($key)); - self::encryptByFileSyncKey($key, kPathManager::getStorageProfileIdForKey($key)); + $encryptedContent = self::encryptByFileSyncKey($key, kPathManager::getStorageProfileIdForKey($key)); + $contentToPut = $encryptedContent ? $encryptedContent : $content; + $compressedContentToPut = gzcompress($contentToPut); + $contentToPut = $compressedContentToPut ? $compressedContentToPut : $contentToPut; + + //If the file is under a specific threshold in size, save it to redis cache + $contentSize = strlen($contentToPut); + $fileSyncFilesToCache = kConf::get(self::FILE_SYNC_TYPES_TO_CACHE, kConfMapNames::RUNTIME_CONFIG, array()); + $maxFileSizeToCache = kConf::get(self::MAX_FILE_SIZE_TO_CACHE, kConfMapNames::RUNTIME_CONFIG); + if ($contentSize <= $maxFileSizeToCache && in_array("{$key->object_type}_{$key->object_sub_type}", $fileSyncFilesToCache)) + { + $redisWrapper = self::initSmallFileRedisInstance(); + if ($redisWrapper) + { + $cacheKey = self::getCacheKey($key); + try + { + $fileAdd = $redisWrapper->doSet($cacheKey, $contentToPut); + if ($fileAdd === false) + { + KalturaLog::err("Failed to add file content with key [$key]"); + } + } + catch (Exception $e) + { + KalturaLog::err('Failed to add file content with key [$key] - [' . $e->getCode() . ' : ' . $e->getMessage() . ']'); + } + } + } + } + + protected static function initSmallFileRedisInstance() + { + $redisWrapper = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC); + if (!$redisWrapper) + { + return null; + } + + return $redisWrapper; } protected static function setPermissions($filePath) From f4b31cb8e8e446f05f9609c545131eb084e49e48 Mon Sep 17 00:00:00 2001 From: "daniel.barak" Date: Mon, 24 Feb 2025 11:44:32 +0200 Subject: [PATCH 2/8] redis cache --- .../apps/kaltura/lib/cache/kCacheManager.php | 2 +- .../lib/storage/kFileSyncUtils.class.php | 83 ++++++++++++------- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/alpha/apps/kaltura/lib/cache/kCacheManager.php b/alpha/apps/kaltura/lib/cache/kCacheManager.php index 15ca61f0588..6d8c950d2a5 100644 --- a/alpha/apps/kaltura/lib/cache/kCacheManager.php +++ b/alpha/apps/kaltura/lib/cache/kCacheManager.php @@ -9,7 +9,7 @@ class kCacheManager // cache types const CACHE_TYPE_PLAY_MANIFEST = 'playManifest'; const CACHE_TYPE_FILE_SYNC = 'fileSync'; - const CACHE_TYPE_SMALL_FILE_SYNC = 'smallfileSync'; + const CACHE_TYPE_SMALL_FILE_SYNC = 'smallFileSync'; const CACHE_TYPE_PERMISSION_MANAGER = 'permissionManager'; const CACHE_TYPE_QUERY_CACHE_KEYS = 'queryCacheKeys'; const CACHE_TYPE_QUERY_CACHE_QUERIES = 'queryCacheQueries'; diff --git a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php index e90fa07db8d..017a8826c52 100644 --- a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php +++ b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php @@ -177,28 +177,40 @@ protected static function getCacheKey(FileSyncKey $key) return self::CACHE_KEY_PREFIX . "{$key->object_id}_{$key->object_type}_{$key->object_sub_type}_{$key->version}"; } - public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote_if_no_local = true , $strict = true , $max_file_size = 0 ) + private static function getFromSmallFileCache($cacheKey) { - $cacheKey = self::getCacheKey($key); $redisCacheStore = self::initSmallFileRedisInstance(); - if ($redisCacheStore) + if (!$redisCacheStore) { - try - { - $result = $redisCacheStore->doGet($cacheKey); - if ($result) - { - $uncompressedResult = gzuncompress($result); - $result = $uncompressedResult ? $uncompressedResult : $result; - KalturaLog::info("returning from redis cache, key [$cacheKey] size [" . strlen($result) . "]"); - return $result; - } - } - catch(Exception $e) + return; + } + + try + { + $result = $redisCacheStore->doGet($cacheKey); + if ($result) { - KalturaLog::info('File not found on Redis cache. [' . $e->getCode() . ' : ' . $e->getMessage() . ']'); + $uncompressedResult = gzuncompress($result); + $result = $uncompressedResult ?? $result; + KalturaLog::info("returning from redis cache, key [$cacheKey] size [" . strlen($result) . "]"); + return $result; } } + catch(Exception $e) + { + KalturaLog::info('File not found on Redis cache. [' . $e->getCode() . ' : ' . $e->getMessage() . ']'); + } + } + + public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote_if_no_local = true , $strict = true , $max_file_size = 0 ) + { + $cacheKey = self::getCacheKey($key); + + $resultFromSmallCache = self::getFromSmallFileCache($cacheKey); + if ($resultFromSmallCache) + { + return $resultFromSmallCache; + } $cacheStore = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_FILE_SYNC); if ($cacheStore) @@ -279,8 +291,13 @@ public static function file_put_contents ( FileSyncKey $key , $content , $strict kFile::filePutContents($fullPath , $content); self::setPermissions($fullPath); self::createSyncFileForKey($rootPath, $filePath, $key , $strict , !is_null($res), false, md5($content), kPathManager::getStorageProfileIdForKey($key)); - $encryptedContent = self::encryptByFileSyncKey($key, kPathManager::getStorageProfileIdForKey($key)); - $contentToPut = $encryptedContent ? $encryptedContent : $content; + self::setInSmallFileCache($key, $content); + } + + protected static function setInSmallFileCache($key, $content) + { + self::encryptByFileSyncKey($key, kPathManager::getStorageProfileIdForKey($key)); + $contentToPut = gzcompress($content) ?? $content; $compressedContentToPut = gzcompress($contentToPut); $contentToPut = $compressedContentToPut ? $compressedContentToPut : $contentToPut; @@ -288,25 +305,31 @@ public static function file_put_contents ( FileSyncKey $key , $content , $strict $contentSize = strlen($contentToPut); $fileSyncFilesToCache = kConf::get(self::FILE_SYNC_TYPES_TO_CACHE, kConfMapNames::RUNTIME_CONFIG, array()); $maxFileSizeToCache = kConf::get(self::MAX_FILE_SIZE_TO_CACHE, kConfMapNames::RUNTIME_CONFIG); + if (!$fileSyncFilesToCache || !$maxFileSizeToCache) + { + return; + } if ($contentSize <= $maxFileSizeToCache && in_array("{$key->object_type}_{$key->object_sub_type}", $fileSyncFilesToCache)) { $redisWrapper = self::initSmallFileRedisInstance(); - if ($redisWrapper) + if (!$redisWrapper) { - $cacheKey = self::getCacheKey($key); - try - { - $fileAdd = $redisWrapper->doSet($cacheKey, $contentToPut); - if ($fileAdd === false) - { - KalturaLog::err("Failed to add file content with key [$key]"); - } - } - catch (Exception $e) + return; + } + + $cacheKey = self::getCacheKey($key); + try + { + $fileAdd = $redisWrapper->doSet($cacheKey, $contentToPut); + if ($fileAdd === false) { - KalturaLog::err('Failed to add file content with key [$key] - [' . $e->getCode() . ' : ' . $e->getMessage() . ']'); + KalturaLog::err("Failed to add file content with key [$key]"); } } + catch (Exception $e) + { + KalturaLog::err("Failed to add file content with key [$key] - [" . $e->getCode() . " : " . $e->getMessage() . "]"); + } } } From dfe9e5595b853bcc3a1cff31f361df4e3fb4e652 Mon Sep 17 00:00:00 2001 From: "daniel.barak" Date: Mon, 24 Feb 2025 11:57:24 +0200 Subject: [PATCH 3/8] redis cache --- alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php index 017a8826c52..54f9dd6bfc5 100644 --- a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php +++ b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php @@ -291,12 +291,12 @@ public static function file_put_contents ( FileSyncKey $key , $content , $strict kFile::filePutContents($fullPath , $content); self::setPermissions($fullPath); self::createSyncFileForKey($rootPath, $filePath, $key , $strict , !is_null($res), false, md5($content), kPathManager::getStorageProfileIdForKey($key)); + self::encryptByFileSyncKey($key, kPathManager::getStorageProfileIdForKey($key)); self::setInSmallFileCache($key, $content); } protected static function setInSmallFileCache($key, $content) { - self::encryptByFileSyncKey($key, kPathManager::getStorageProfileIdForKey($key)); $contentToPut = gzcompress($content) ?? $content; $compressedContentToPut = gzcompress($contentToPut); $contentToPut = $compressedContentToPut ? $compressedContentToPut : $contentToPut; From d4572d7368159efd91c3f3609df090e2f33c5217 Mon Sep 17 00:00:00 2001 From: "daniel.barak" Date: Tue, 25 Feb 2025 11:37:31 +0200 Subject: [PATCH 4/8] fix --- alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php index 54f9dd6bfc5..4ffd7a518fc 100644 --- a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php +++ b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php @@ -298,8 +298,6 @@ public static function file_put_contents ( FileSyncKey $key , $content , $strict protected static function setInSmallFileCache($key, $content) { $contentToPut = gzcompress($content) ?? $content; - $compressedContentToPut = gzcompress($contentToPut); - $contentToPut = $compressedContentToPut ? $compressedContentToPut : $contentToPut; //If the file is under a specific threshold in size, save it to redis cache $contentSize = strlen($contentToPut); From 9ba427977fb09369c5c568ceaf0bc7867ea672bb Mon Sep 17 00:00:00 2001 From: "daniel.barak" Date: Tue, 25 Feb 2025 12:08:31 +0200 Subject: [PATCH 5/8] checking file types before compression --- .../apps/kaltura/lib/storage/kFileSyncUtils.class.php | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php index 4ffd7a518fc..93d1dddff38 100644 --- a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php +++ b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php @@ -297,17 +297,16 @@ public static function file_put_contents ( FileSyncKey $key , $content , $strict protected static function setInSmallFileCache($key, $content) { - $contentToPut = gzcompress($content) ?? $content; - - //If the file is under a specific threshold in size, save it to redis cache - $contentSize = strlen($contentToPut); $fileSyncFilesToCache = kConf::get(self::FILE_SYNC_TYPES_TO_CACHE, kConfMapNames::RUNTIME_CONFIG, array()); $maxFileSizeToCache = kConf::get(self::MAX_FILE_SIZE_TO_CACHE, kConfMapNames::RUNTIME_CONFIG); - if (!$fileSyncFilesToCache || !$maxFileSizeToCache) + if (!$fileSyncFilesToCache || !$maxFileSizeToCache || !in_array("{$key->object_type}_{$key->object_sub_type}", $fileSyncFilesToCache)) { return; } - if ($contentSize <= $maxFileSizeToCache && in_array("{$key->object_type}_{$key->object_sub_type}", $fileSyncFilesToCache)) + + $contentToPut = gzcompress($content) ?? $content; + $contentSize = strlen($contentToPut); + if ($contentSize <= $maxFileSizeToCache) { $redisWrapper = self::initSmallFileRedisInstance(); if (!$redisWrapper) From 0038e0c4f12dcb690fbec24700d4a065cd9a10ba Mon Sep 17 00:00:00 2001 From: "daniel.barak" Date: Tue, 25 Feb 2025 14:44:52 +0200 Subject: [PATCH 6/8] cache handling more generic --- .../lib/storage/kFileSyncUtils.class.php | 68 ++++++++----------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php index 93d1dddff38..2e7cb92976d 100644 --- a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php +++ b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php @@ -177,49 +177,37 @@ protected static function getCacheKey(FileSyncKey $key) return self::CACHE_KEY_PREFIX . "{$key->object_id}_{$key->object_type}_{$key->object_sub_type}_{$key->version}"; } - private static function getFromSmallFileCache($cacheKey) + protected static function getCacheType(FileSyncKey $key) { - $redisCacheStore = self::initSmallFileRedisInstance(); - if (!$redisCacheStore) - { - return; - } + $fileSyncFilesToCache = kConf::get(self::FILE_SYNC_TYPES_TO_CACHE, kConfMapNames::RUNTIME_CONFIG, array()); + $objectKeys = array ( + $key->getObjectType() . ':' . $key->getObjectSubType(), + $key->getObjectType() . ':' . '*', + '*' // WildCard + ); - try + if ($fileSyncFilesToCache && array_intersect($objectKeys, $fileSyncFilesToCache)) { - $result = $redisCacheStore->doGet($cacheKey); - if ($result) - { - $uncompressedResult = gzuncompress($result); - $result = $uncompressedResult ?? $result; - KalturaLog::info("returning from redis cache, key [$cacheKey] size [" . strlen($result) . "]"); - return $result; - } + return kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC; } - catch(Exception $e) + else { - KalturaLog::info('File not found on Redis cache. [' . $e->getCode() . ' : ' . $e->getMessage() . ']'); + return kCacheManager::CACHE_TYPE_FILE_SYNC; } } public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote_if_no_local = true , $strict = true , $max_file_size = 0 ) { $cacheKey = self::getCacheKey($key); - - $resultFromSmallCache = self::getFromSmallFileCache($cacheKey); - if ($resultFromSmallCache) - { - return $resultFromSmallCache; - } - - $cacheStore = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_FILE_SYNC); + $cacheType = self::getCacheType($key); + $cacheStore = kCacheManager::getSingleLayerCache($cacheType); if ($cacheStore) { $result = $cacheStore->get($cacheKey); if ($result) { KalturaLog::info("returning from cache, key [$cacheKey] size [" . strlen($result) . "]"); - return $result; + return ($cacheType == kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC) ? gzuncompress($result) : $result; } } @@ -299,33 +287,31 @@ protected static function setInSmallFileCache($key, $content) { $fileSyncFilesToCache = kConf::get(self::FILE_SYNC_TYPES_TO_CACHE, kConfMapNames::RUNTIME_CONFIG, array()); $maxFileSizeToCache = kConf::get(self::MAX_FILE_SIZE_TO_CACHE, kConfMapNames::RUNTIME_CONFIG); - if (!$fileSyncFilesToCache || !$maxFileSizeToCache || !in_array("{$key->object_type}_{$key->object_sub_type}", $fileSyncFilesToCache)) + $objectKeys = array ( + $key->getObjectType() . ':' . $key->getObjectSubType(), + $key->getObjectType() . ':' . '*', + '*' // WildCard + ); + if (!$fileSyncFilesToCache || !$maxFileSizeToCache || !array_intersect($objectKeys, $fileSyncFilesToCache)) { return; } - $contentToPut = gzcompress($content) ?? $content; - $contentSize = strlen($contentToPut); + $contentSize = strlen($content); if ($contentSize <= $maxFileSizeToCache) { - $redisWrapper = self::initSmallFileRedisInstance(); - if (!$redisWrapper) + $cacheStore = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC); + if (!$cacheStore) { return; } $cacheKey = self::getCacheKey($key); - try - { - $fileAdd = $redisWrapper->doSet($cacheKey, $contentToPut); - if ($fileAdd === false) - { - KalturaLog::err("Failed to add file content with key [$key]"); - } - } - catch (Exception $e) + $contentToPut = gzcompress($content) ?? $content; + $result = $cacheStore->set($cacheKey, $contentToPut, self::FILE_SYNC_CACHE_EXPIRY); + if ($result === false) { - KalturaLog::err("Failed to add file content with key [$key] - [" . $e->getCode() . " : " . $e->getMessage() . "]"); + KalturaLog::err("Failed to add file content with key [$key]"); } } } From 19ca5ed0a550a848961c1b238917de71264cfaae Mon Sep 17 00:00:00 2001 From: "daniel.barak" Date: Tue, 25 Feb 2025 15:08:40 +0200 Subject: [PATCH 7/8] handling uncompressed results --- .../kaltura/lib/storage/kFileSyncUtils.class.php | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php index 2e7cb92976d..b108820baf6 100644 --- a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php +++ b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php @@ -207,7 +207,7 @@ public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote if ($result) { KalturaLog::info("returning from cache, key [$cacheKey] size [" . strlen($result) . "]"); - return ($cacheType == kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC) ? gzuncompress($result) : $result; + return gzuncompress($result) ?? $result; } } @@ -316,17 +316,6 @@ protected static function setInSmallFileCache($key, $content) } } - protected static function initSmallFileRedisInstance() - { - $redisWrapper = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC); - if (!$redisWrapper) - { - return null; - } - - return $redisWrapper; - } - protected static function setPermissions($filePath) { From 47018143b38ee9571a129f883ebac85f9ddb0813 Mon Sep 17 00:00:00 2001 From: "daniel.barak" Date: Sun, 2 Mar 2025 09:09:13 +0200 Subject: [PATCH 8/8] adjustments --- .../lib/storage/kFileSyncUtils.class.php | 93 +++++++++++-------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php index b108820baf6..85d45758d14 100644 --- a/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php +++ b/alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php @@ -21,6 +21,7 @@ class kFileSyncUtils implements kObjectChangedEventConsumer, kObjectAddedEventCo const SOURCE_TYPE_HTTP = 'http'; const FILE_SYNC_TYPES_TO_CACHE = 'file_sync_types_to_cache'; const MAX_FILE_SIZE_TO_CACHE = 'max_file_size_to_cache'; + const COMPRESSED_PREFIX = '#COMPRESS'; /** * Contain all object types and sub types that should not be synced @@ -196,21 +197,72 @@ protected static function getCacheType(FileSyncKey $key) } } - public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote_if_no_local = true , $strict = true , $max_file_size = 0 ) + protected static function setInSmallFileCache($key, $content) { - $cacheKey = self::getCacheKey($key); $cacheType = self::getCacheType($key); - $cacheStore = kCacheManager::getSingleLayerCache($cacheType); + $maxFileSizeToCache = kConf::get(self::MAX_FILE_SIZE_TO_CACHE, kConfMapNames::RUNTIME_CONFIG); + if (!$maxFileSizeToCache || $cacheType != kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC) + { + return; + } + + self::storeData($content, $maxFileSizeToCache, $key, $cacheType); + } + + protected static function storeData($content, $maxFileSizeToCache, $key, $cacheType) + { + $contentSize = strlen($content); + if ($contentSize <= $maxFileSizeToCache) + { + $cacheStore = kCacheManager::getSingleLayerCache($cacheType); + if (!$cacheStore) + { + return; + } + + $cacheKey = self::getCacheKey($key); + $compressedContent = gzcompress($content); + $contentToPut = $compressedContent ? (self::COMPRESSED_PREFIX . $compressedContent) : $content; + $result = $cacheStore->set($cacheKey, $contentToPut, self::FILE_SYNC_CACHE_EXPIRY); + if ($result === false) + { + KalturaLog::err("Failed to add file content with key [$key]"); + } + } + } + + protected static function getDataFromCacheStore($cacheStore, $cacheKey) + { if ($cacheStore) { $result = $cacheStore->get($cacheKey); if ($result) { + if (str_starts_with($result, self::COMPRESSED_PREFIX)) + { + $result = gzuncompress(substr($result, strlen(self::COMPRESSED_PREFIX))); + } KalturaLog::info("returning from cache, key [$cacheKey] size [" . strlen($result) . "]"); - return gzuncompress($result) ?? $result; + return $result; } } + return false; + } + + public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote_if_no_local = true , $strict = true , $max_file_size = 0 ) + { + $cacheKey = self::getCacheKey($key); + $cacheType = self::getCacheType($key); + $cacheStore = kCacheManager::getSingleLayerCache($cacheType); + + $cachedResult = self::getDataFromCacheStore($cacheStore, $cacheKey); + if ($cachedResult) + { + return $cachedResult; + } + + KalturaLog::debug("key [$key], fetch_from_remote_if_no_local [$fetch_from_remote_if_no_local], strict [$strict]"); list ( $file_sync , $local ) = self::getReadyFileSyncForKey( $key , $fetch_from_remote_if_no_local , $strict ); if($file_sync) @@ -283,39 +335,6 @@ public static function file_put_contents ( FileSyncKey $key , $content , $strict self::setInSmallFileCache($key, $content); } - protected static function setInSmallFileCache($key, $content) - { - $fileSyncFilesToCache = kConf::get(self::FILE_SYNC_TYPES_TO_CACHE, kConfMapNames::RUNTIME_CONFIG, array()); - $maxFileSizeToCache = kConf::get(self::MAX_FILE_SIZE_TO_CACHE, kConfMapNames::RUNTIME_CONFIG); - $objectKeys = array ( - $key->getObjectType() . ':' . $key->getObjectSubType(), - $key->getObjectType() . ':' . '*', - '*' // WildCard - ); - if (!$fileSyncFilesToCache || !$maxFileSizeToCache || !array_intersect($objectKeys, $fileSyncFilesToCache)) - { - return; - } - - $contentSize = strlen($content); - if ($contentSize <= $maxFileSizeToCache) - { - $cacheStore = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC); - if (!$cacheStore) - { - return; - } - - $cacheKey = self::getCacheKey($key); - $contentToPut = gzcompress($content) ?? $content; - $result = $cacheStore->set($cacheKey, $contentToPut, self::FILE_SYNC_CACHE_EXPIRY); - if ($result === false) - { - KalturaLog::err("Failed to add file content with key [$key]"); - } - } - } - protected static function setPermissions($filePath) {