caching of small files in redis

alpha/apps/kaltura/lib/cache/kCacheManager.php

@@ -9,6 +9,7 @@ class kCacheManager
 	// cache types
 	const CACHE_TYPE_PLAY_MANIFEST = 'playManifest';
 	const CACHE_TYPE_FILE_SYNC = 'fileSync';
+	const CACHE_TYPE_SMALL_FILE_SYNC = 'smallFileSync';
 	const CACHE_TYPE_PERMISSION_MANAGER = 'permissionManager';
 	const CACHE_TYPE_QUERY_CACHE_KEYS = 'queryCacheKeys';
 	const CACHE_TYPE_QUERY_CACHE_QUERIES = 'queryCacheQueries';

alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php

@@ -19,6 +19,9 @@ class kFileSyncUtils implements kObjectChangedEventConsumer, kObjectAddedEventCo
 
 	const SOURCE_TYPE_FILE = 'file';
 	const SOURCE_TYPE_HTTP = 'http';
+	const FILE_SYNC_TYPES_TO_CACHE = 'file_sync_types_to_cache';
+	const MAX_FILE_SIZE_TO_CACHE = 'max_file_size_to_cache';
+	const COMPRESSED_PREFIX = '#COMPRESS';
 
 	/**
 	 * Contain all object types and sub types that should not be synced
@@ -170,20 +173,96 @@ public static function dir_get_files(FileSyncKey $key, $strict = true)
 		return $files;
 	}
 
-	public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote_if_no_local = true , $strict = true , $max_file_size = 0 )
+	protected static function getCacheKey(FileSyncKey $key)
+	{
+		return self::CACHE_KEY_PREFIX . "{$key->object_id}_{$key->object_type}_{$key->object_sub_type}_{$key->version}";
+	}
+
+	protected static function getCacheType(FileSyncKey $key)
+	{
+		$fileSyncFilesToCache = kConf::get(self::FILE_SYNC_TYPES_TO_CACHE, kConfMapNames::RUNTIME_CONFIG, array());
+		$objectKeys = array (
+			$key->getObjectType() . ':' . $key->getObjectSubType(),
+			$key->getObjectType() . ':' . '*',
+			'*' // WildCard
+		);
+
+		if ($fileSyncFilesToCache && array_intersect($objectKeys, $fileSyncFilesToCache))
+		{
+			return kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC;
+		}
+		else
+		{
+			return kCacheManager::CACHE_TYPE_FILE_SYNC;
+		}
+	}
+
+	protected static function setInSmallFileCache($key, $content)
+	{
+		$cacheType = self::getCacheType($key);
+		$maxFileSizeToCache = kConf::get(self::MAX_FILE_SIZE_TO_CACHE, kConfMapNames::RUNTIME_CONFIG);
+		if (!$maxFileSizeToCache || $cacheType != kCacheManager::CACHE_TYPE_SMALL_FILE_SYNC)
+		{
+			return;
+		}
+
+		self::storeData($content, $maxFileSizeToCache, $key, $cacheType);
+	}
+
+	protected static function storeData($content, $maxFileSizeToCache, $key, $cacheType)
+	{
+		$contentSize = strlen($content);
+		if ($contentSize <= $maxFileSizeToCache)
+		{
+			$cacheStore = kCacheManager::getSingleLayerCache($cacheType);
+			if (!$cacheStore)
+			{
+				return;
+			}
+
+			$cacheKey = self::getCacheKey($key);
+			$compressedContent = gzcompress($content);
+			$contentToPut = $compressedContent ? (self::COMPRESSED_PREFIX . $compressedContent) : $content;
+			$result = $cacheStore->set($cacheKey, $contentToPut, self::FILE_SYNC_CACHE_EXPIRY);
+			if ($result === false)
+			{
+				KalturaLog::err("Failed to add file content with key [$key]");
+			}
+		}
+	}
+
+	protected static function getDataFromCacheStore($cacheStore, $cacheKey)
 	{
-		$cacheStore = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_FILE_SYNC);
 		if ($cacheStore)
 		{
-			$cacheKey = self::CACHE_KEY_PREFIX . "{$key->object_id}_{$key->object_type}_{$key->object_sub_type}_{$key->version}";
 			$result = $cacheStore->get($cacheKey);
 			if ($result)
 			{
-				KalturaLog::info("returning from cache, key [$cacheKey] size [".strlen($result)."]");
+				if (str_starts_with($result, self::COMPRESSED_PREFIX))
+				{
+					$result = gzuncompress(substr($result, strlen(self::COMPRESSED_PREFIX)));
+				}
+				KalturaLog::info("returning from cache, key [$cacheKey] size [" . strlen($result) . "]");
 				return $result;
 			}
 		}
 
+		return false;
+	}
+
+	public static function file_get_contents ( FileSyncKey $key , $fetch_from_remote_if_no_local = true , $strict = true , $max_file_size = 0 )
+	{
+		$cacheKey = self::getCacheKey($key);
+		$cacheType = self::getCacheType($key);
+		$cacheStore = kCacheManager::getSingleLayerCache($cacheType);
+
+		$cachedResult = self::getDataFromCacheStore($cacheStore, $cacheKey);
+		if ($cachedResult)
+		{
+			return $cachedResult;
+		}
+
+
 		KalturaLog::debug("key [$key], fetch_from_remote_if_no_local [$fetch_from_remote_if_no_local], strict [$strict]");
 		list ( $file_sync , $local ) = self::getReadyFileSyncForKey( $key , $fetch_from_remote_if_no_local , $strict );
 		if($file_sync)
@@ -253,6 +332,7 @@ public static function file_put_contents ( FileSyncKey $key , $content , $strict
 		self::setPermissions($fullPath);
 		self::createSyncFileForKey($rootPath, $filePath,  $key , $strict , !is_null($res), false, md5($content), kPathManager::getStorageProfileIdForKey($key));
 		self::encryptByFileSyncKey($key, kPathManager::getStorageProfileIdForKey($key));
+		self::setInSmallFileCache($key, $content);
 	}
 
 	protected static function setPermissions($filePath)