From b19475e38daf82da73d887a3e1fa8bc751e90c59 Mon Sep 17 00:00:00 2001
From: Antuan <antuan@leil.io>
Date: Thu, 18 Jul 2024 13:59:52 +0200
Subject: [PATCH 01/27] fix: Allow legacy metadata download

After Shadow node downloads metadata from master server, it verifies its
version. This commits allows this verification to process legacy tag
`LIZM` and `MFS` to allow easy migration from Legacy metadata versions.
---
 src/common/metadata.cc         |  6 ++++++
 src/master/filesystem_store.cc | 11 ++++++++---
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/common/metadata.cc b/src/common/metadata.cc
index 1e5470202..d8ad001cf 100644
--- a/src/common/metadata.cc
+++ b/src/common/metadata.cc
@@ -92,9 +92,15 @@ uint64_t metadataGetVersion(const std::string& file) {
 	std::string signature = std::string(chkbuff, 8);
 	std::string sfsSignature = std::string(SFSSIGNATURE "M 2.9");
 	std::string sauSignature = std::string(SAUSIGNATURE "M 2.9");
+	std::string legacySignature = std::string("LIZM 2.9");
 
 	if (signature == sfsSignature || signature == sauSignature) {
 		memcpy(eofmark,"[SFS EOF MARKER]",16);
+	} else if (memcmp(chkbuff, legacySignature.data(), legacySignature.size()) == 0) {
+		safs_pretty_syslog(LOG_WARNING,
+		                   "Legacy metadata section header %s, was detected in the metadata file %s",
+		                   legacySignature.c_str(), file.c_str());
+		memcpy(eofmark,"[MFS EOF MARKER]",16);
 	} else {
 		close(fd);
 		throw MetadataCheckException("Bad EOF MARKER in the metadata file.");
diff --git a/src/master/filesystem_store.cc b/src/master/filesystem_store.cc
index 651877363..9c7aef520 100644
--- a/src/master/filesystem_store.cc
+++ b/src/master/filesystem_store.cc
@@ -1011,8 +1011,11 @@ static const std::vector<MetadataSection> kMetadataSections = {
 bool isEndOfMetadata(const uint8_t *sectionPtr) {
 	static constexpr std::string_view kMetadataTrailer("[" SFSSIGNATURE
 	                                                   " EOF MARKER]");
-	return memcmp(sectionPtr, kMetadataTrailer.data(),
-	              ::kMetadataSectionHeaderSize) == kOpSuccess;
+	static constexpr std::string_view kMetadataLegacyTrailer("[MFS EOF MARKER]");
+	return ((memcmp(sectionPtr, kMetadataTrailer.data(),
+	              ::kMetadataSectionHeaderSize) == kOpSuccess) ||
+	        (memcmp(sectionPtr, kMetadataLegacyTrailer.data(),
+	              ::kMetadataSectionHeaderSize) == kOpSuccess));
 }
 
 
@@ -1329,6 +1332,7 @@ bool isNewMetadataFile([[maybe_unused]]const uint8_t *headerPtr) {
 bool checkMetadataSignature(const std::shared_ptr<MemoryMappedFile> &metadataFile) {
 	static constexpr std::string_view kMetadataHeaderNewV2_9(SFSSIGNATURE "M 2.9");
 	static constexpr std::string_view kMetadataHeaderOldV2_9(SAUSIGNATURE "M 2.9");
+	static constexpr std::string_view kMetadataHeaderLegacy("LIZM 2.9");
 	static constexpr uint8_t kMetadataHeaderSize = 8;
 	size_t kMetadataHeaderOffset{0};
 	uint8_t *headerPtr;
@@ -1343,7 +1347,8 @@ bool checkMetadataSignature(const std::shared_ptr<MemoryMappedFile> &metadataFil
 		return false;
 	}
 	if ((memcmp(headerPtr, kMetadataHeaderNewV2_9.data(), kMetadataHeaderSize) != kOpSuccess) &&
-	    (memcmp(headerPtr, kMetadataHeaderOldV2_9.data(), kMetadataHeaderSize) != kOpSuccess)) {
+	    (memcmp(headerPtr, kMetadataHeaderOldV2_9.data(), kMetadataHeaderSize) != kOpSuccess) &&
+	    (memcmp(headerPtr, kMetadataHeaderLegacy.data(), kMetadataHeaderSize) != kOpSuccess)) {
 		throw MetadataConsistencyException("wrong metadata header version");
 	}
 	return true;

From b9df1b11ce0af7db7571c2e83b7f32505603349f Mon Sep 17 00:00:00 2001
From: Antuan <antuan@leil.io>
Date: Thu, 18 Jul 2024 19:43:51 +0200
Subject: [PATCH 02/27] fix: Allow legacy sessions download

---
 src/master/matoclserv.cc | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/master/matoclserv.cc b/src/master/matoclserv.cc
index 9a2384b96..aadc59e7c 100644
--- a/src/master/matoclserv.cc
+++ b/src/master/matoclserv.cc
@@ -685,6 +685,7 @@ int matoclserv_load_sessions() {
 	uint32_t i,statsinfile;
 	int r;
 	FILE *fd;
+	std::string legacySessionSignature = std::string("MFS");
 
 	fd = fopen(kSessionsFilename, "r");
 	if (fd==NULL) {
@@ -700,19 +701,23 @@ int matoclserv_load_sessions() {
 		fclose(fd);
 		return -1;
 	}
-	if (memcmp(hdr,SFSSIGNATURE "S 1.5",8)==0) {
+	if (memcmp(hdr,SFSSIGNATURE "S 1.5",8)==0 ||
+	    memcmp(hdr, (legacySessionSignature + "S 1.5").c_str(), 8) == 0) {
 		mapalldata = 0;
 		goaltrashdata = 0;
 		statsinfile = 16;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\001",8)==0) {
+	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\001",8)==0 ||
+	           memcmp(hdr, (legacySessionSignature + "S \001\006\001").c_str(), 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		statsinfile = 16;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\002",8)==0) {
+	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\002",8)==0 ||
+	           memcmp(hdr, (legacySessionSignature + "S \001\006\002").c_str(), 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		statsinfile = 21;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\003",8)==0) {
+	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\003",8)==0 ||
+	           memcmp(hdr, (legacySessionSignature + "S \001\006\003").c_str(), 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		if (fread(hdr,2,1,fd)!=1) {
@@ -722,7 +727,8 @@ int matoclserv_load_sessions() {
 		}
 		ptr = hdr;
 		statsinfile = get16bit(&ptr);
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\004",8)==0) {
+	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\004",8)==0 ||
+	           memcmp(hdr, (legacySessionSignature + "S \001\006\004").c_str(), 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 1;
 		if (fread(hdr,2,1,fd)!=1) {

From bb8671dba0b3b928a9669c8e9d65fa95df17418f Mon Sep 17 00:00:00 2001
From: Antuan <antuan@leil.io>
Date: Fri, 19 Jul 2024 14:37:41 +0200
Subject: [PATCH 03/27] feat: Load legacy metadata file

The master server now supports loading legacy `metadata.mfs` file. The
new logic priority is to search for new metadata format and loaded if it
is found. If not found it search for legacy format, load it and save it
in new format. If both are present, it load new metadata `metadata.sfs`.

- fix: Apply code review suggestions
---
 src/common/metadata.cc   |  5 ++++-
 src/common/metadata.h    |  1 +
 src/master/filesystem.cc | 25 ++++++++++++++++++++++---
 src/master/matoclserv.cc | 23 ++++++++++++-----------
 4 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/src/common/metadata.cc b/src/common/metadata.cc
index d8ad001cf..46892d0c2 100644
--- a/src/common/metadata.cc
+++ b/src/common/metadata.cc
@@ -41,6 +41,9 @@ const char kMetadataFilename[] = METADATA_FILENAME_TEMPL;
 const char kMetadataTmpFilename[] = METADATA_FILENAME_TEMPL ".tmp";
 const char kMetadataEmergencyFilename[] = METADATA_FILENAME_TEMPL ".emergency";
 #undef METADATA_FILENAME_TEMPL
+#define METADATA_LEGACY_FILENAME_TEMPL "metadata.mfs"
+const char kMetadataLegacyFilename[] = METADATA_LEGACY_FILENAME_TEMPL;
+#undef METADATA_LEGACY_FILENAME_TEMPL
 #define METADATA_ML_FILENAME_TEMPL "metadata_ml.sfs"
 const char kMetadataMlFilename[] = METADATA_ML_FILENAME_TEMPL;
 const char kMetadataMlTmpFilename[] = METADATA_ML_FILENAME_TEMPL ".tmp";
@@ -96,7 +99,7 @@ uint64_t metadataGetVersion(const std::string& file) {
 
 	if (signature == sfsSignature || signature == sauSignature) {
 		memcpy(eofmark,"[SFS EOF MARKER]",16);
-	} else if (memcmp(chkbuff, legacySignature.data(), legacySignature.size()) == 0) {
+	} else if (signature == legacySignature) {
 		safs_pretty_syslog(LOG_WARNING,
 		                   "Legacy metadata section header %s, was detected in the metadata file %s",
 		                   legacySignature.c_str(), file.c_str());
diff --git a/src/common/metadata.h b/src/common/metadata.h
index 2a912ca76..c48dc944d 100644
--- a/src/common/metadata.h
+++ b/src/common/metadata.h
@@ -31,6 +31,7 @@
 
 extern const char kMetadataFilename[];
 extern const char kMetadataTmpFilename[];
+extern const char kMetadataLegacyFilename[];
 extern const char kMetadataMlFilename[];
 extern const char kMetadataMlTmpFilename[];
 extern const char kMetadataEmergencyFilename[];
diff --git a/src/master/filesystem.cc b/src/master/filesystem.cc
index fd333f3b7..6825fefd6 100644
--- a/src/master/filesystem.cc
+++ b/src/master/filesystem.cc
@@ -216,9 +216,28 @@ int fs_loadall(void) {
 	changelogsMigrateFrom_1_6_29("changelog");
 	if (fs::exists(kMetadataTmpFilename)) {
 		throw MetadataFsConsistencyException(
-				"temporary metadata file (" + std::string(kMetadataTmpFilename) + ") exists, metadata directory is in dirty state");
+		    "temporary metadata file (" + std::string(kMetadataTmpFilename) + ") exists,"
+		    " metadata directory is in dirty state");
 	}
-	if ((metadataserver::isMaster()) && !fs::exists(kMetadataFilename)) {
+	std::string metadataFile;
+	bool metadataFileExists = fs::exists(kMetadataFilename);
+	bool legacyMetadataFileExists = fs::exists(kMetadataLegacyFilename);
+	if (metadataFileExists) {
+		metadataFile = kMetadataFilename;
+	}
+	if(metadataFileExists && legacyMetadataFileExists) {
+		metadataFile = kMetadataFilename;
+		safs_pretty_syslog(LOG_WARNING, "There are two metadata files in the data path: %s and %s."
+		                   " Please remove the legacy one (%s) to avoid damage to your storage.",
+		                   kMetadataFilename, kMetadataLegacyFilename, kMetadataLegacyFilename);
+	}
+	if (!metadataFileExists && legacyMetadataFileExists) {
+		metadataFile = kMetadataLegacyFilename;
+		safs_pretty_syslog(LOG_WARNING, "Only Legacy metadata file %s found and will be loaded instead."
+		                   " You should delete legacy metadata %s on next restart after new metadata %s is created ",
+		                   metadataFile.c_str(), kMetadataLegacyFilename, kMetadataFilename);
+	}
+	if (metadataserver::isMaster() && !metadataFileExists && !legacyMetadataFileExists) {
 		fs_unlock();
 		std::string currentPath = fs::getCurrentWorkingDirectoryNoThrow();
 		throw FilesystemException("can't open metadata file "+ currentPath + "/" + kMetadataFilename
@@ -229,7 +248,7 @@ int fs_loadall(void) {
 
 	{
 		auto scopedTimer = util::ScopedTimer("metadata load time");
-		fs_loadall(kMetadataFilename, 0);
+		fs_loadall(metadataFile, 0);
 	}
 
 	bool autoRecovery = fs_can_do_auto_recovery();
diff --git a/src/master/matoclserv.cc b/src/master/matoclserv.cc
index aadc59e7c..556f31f1e 100644
--- a/src/master/matoclserv.cc
+++ b/src/master/matoclserv.cc
@@ -674,6 +674,7 @@ void matoclserv_store_sessions() {
 	}
 }
 
+#define MFSSIGNATURE "MFS"
 int matoclserv_load_sessions() {
 	session *asesdata;
 	uint32_t ileng;
@@ -685,7 +686,6 @@ int matoclserv_load_sessions() {
 	uint32_t i,statsinfile;
 	int r;
 	FILE *fd;
-	std::string legacySessionSignature = std::string("MFS");
 
 	fd = fopen(kSessionsFilename, "r");
 	if (fd==NULL) {
@@ -701,23 +701,23 @@ int matoclserv_load_sessions() {
 		fclose(fd);
 		return -1;
 	}
-	if (memcmp(hdr,SFSSIGNATURE "S 1.5",8)==0 ||
-	    memcmp(hdr, (legacySessionSignature + "S 1.5").c_str(), 8) == 0) {
+	if (memcmp(hdr, SFSSIGNATURE "S 1.5", 8) == 0 ||
+	    memcmp(hdr, MFSSIGNATURE "S 1.5", 8) == 0) {
 		mapalldata = 0;
 		goaltrashdata = 0;
 		statsinfile = 16;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\001",8)==0 ||
-	           memcmp(hdr, (legacySessionSignature + "S \001\006\001").c_str(), 8) == 0) {
+	} else if (memcmp(hdr, SFSSIGNATURE "S \001\006\001", 8) == 0 ||
+	           memcmp(hdr, MFSSIGNATURE "S \001\006\001", 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		statsinfile = 16;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\002",8)==0 ||
-	           memcmp(hdr, (legacySessionSignature + "S \001\006\002").c_str(), 8) == 0) {
+	} else if (memcmp(hdr, SFSSIGNATURE "S \001\006\002", 8) == 0 ||
+	           memcmp(hdr, MFSSIGNATURE "S \001\006\002", 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		statsinfile = 21;
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\003",8)==0 ||
-	           memcmp(hdr, (legacySessionSignature + "S \001\006\003").c_str(), 8) == 0) {
+	} else if (memcmp(hdr, SFSSIGNATURE "S \001\006\003", 8) == 0 ||
+	           memcmp(hdr, MFSSIGNATURE "S \001\006\003", 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 0;
 		if (fread(hdr,2,1,fd)!=1) {
@@ -727,8 +727,8 @@ int matoclserv_load_sessions() {
 		}
 		ptr = hdr;
 		statsinfile = get16bit(&ptr);
-	} else if (memcmp(hdr,SFSSIGNATURE "S \001\006\004",8)==0 ||
-	           memcmp(hdr, (legacySessionSignature + "S \001\006\004").c_str(), 8) == 0) {
+	} else if (memcmp(hdr, SFSSIGNATURE "S \001\006\004", 8) == 0 ||
+	           memcmp(hdr, MFSSIGNATURE "S \001\006\004", 8) == 0) {
 		mapalldata = 1;
 		goaltrashdata = 1;
 		if (fread(hdr,2,1,fd)!=1) {
@@ -821,6 +821,7 @@ int matoclserv_load_sessions() {
 	fclose(fd);
 	return 1;
 }
+#undef MFSSIGNATURE
 
 int matoclserv_insert_openfile(session* cr,uint32_t inode) {
 	filelist *ofptr,**ofpptr;

From aa18beff0b539c6b5bfc820a0f0e764555a9d34a Mon Sep 17 00:00:00 2001
From: Antuan <antuan@leil.io>
Date: Fri, 19 Jul 2024 01:16:51 +0200
Subject: [PATCH 04/27] test: Verify master loading legacy metadata

---
 .../test_metadata_load_legacy.sh              | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 tests/test_suites/ShortSystemTests/test_metadata_load_legacy.sh

diff --git a/tests/test_suites/ShortSystemTests/test_metadata_load_legacy.sh b/tests/test_suites/ShortSystemTests/test_metadata_load_legacy.sh
new file mode 100644
index 000000000..3874b305f
--- /dev/null
+++ b/tests/test_suites/ShortSystemTests/test_metadata_load_legacy.sh
@@ -0,0 +1,77 @@
+timeout_set 30 seconds
+master_cfg+="MAGIC_DEBUG_LOG = $TEMP_DIR/syslog|LOG_FLUSH_ON=DEBUG"
+touch "$TEMP_DIR/syslog"
+
+CHUNKSERVERS=0 \
+	MASTERSERVERS=2 \
+	USE_RAMDISK="YES" \
+	MASTER_EXTRA_CONFIG="${master_cfg}" \
+	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER" \
+	AUTO_SHADOW_MASTER="YES" \
+	setup_local_empty_saunafs info
+
+metadata_file="${info[master0_data_path]}/metadata.sfs"
+
+ACTUAL_HEADER_TAG="SFSM 2.9"
+ACTUAL_TRAILER_TAG="[SFS EOF MARKER]"
+LEGACY_HEADER_TAG="LIZM 2.9"
+LEGACY_TRAILER_TAG="[MFS EOF MARKER]"
+
+# Add metalogger service to the configuration
+saunafs_metalogger_daemon start
+
+# Access mountpoint to create metadata
+cd "${info[mount0]}"
+for i in {1..100}; do
+	touch file$i
+	mkdir dir$i
+done
+
+# Exit mount point and save metadata by stopping master server
+cd "${TEMP_DIR}"
+assert_success saunafs_master_n 0 stop
+assert_success saunafs_master_n 1 stop
+
+# Expected Values
+assert_equals "$(head -c8 ${metadata_file})" "${ACTUAL_HEADER_TAG}"
+assert_equals "$(tail -c16 ${metadata_file})" "${ACTUAL_TRAILER_TAG}"
+
+# Modify metadata signature for legacy and reload metadata by starting master
+echo -n "LIZ" | dd of=${metadata_file} bs=1 seek=0 count=3 conv=notrunc
+echo -n "MFS" | dd of=${metadata_file} bs=1 seek=$(( $(stat --print="%s" ${metadata_file}) - 15)) count=3 conv=notrunc
+assert_equals "$(head -c8 ${metadata_file})" "${LEGACY_HEADER_TAG}"
+assert_equals "$(tail -c16 ${metadata_file})" "${LEGACY_TRAILER_TAG}"
+
+mv -iv "${info[master0_data_path]}/metadata.sfs" "${info[master0_data_path]}/metadata.mfs"
+assertlocal_file_exists "${info[master0_data_path]}/metadata.mfs"
+
+# Start master server to reload metadata legacy format
+assertlocal_success saunafs_master_n 0 start
+# Restart master server to load metadata in new format when both are present
+assertlocal_success saunafs_master_n 0 restart
+assertlocal_file_exists "${info[master0_data_path]}/metadata.sfs"
+assertlocal_file_exists "${info[master0_data_path]}/metadata.mfs"
+
+# Shadows has not  synchronize metadata at this point
+assertlocal_file_not_exists "${info[master1_data_path]}/metadata.sfs"
+#start shadow server to synchronize metadata
+assert_success saunafs_master_n 1 start
+assert_eventually "saunafs_shadow_synchronized 1"
+assertlocal_file_exists "${info[master1_data_path]}/metadata.sfs"
+
+# Stop master server to save metadata in new format
+assert_success saunafs_master_n 0 stop
+
+# Synchronized metadata in shadows has proper Tags
+SHADOW_METADATA_FILE="${info[master1_data_path]}/metadata.sfs"
+assert_equals "$(head -c8 ${SHADOW_METADATA_FILE})" "${ACTUAL_HEADER_TAG}"
+assert_equals "$(tail -c16 ${SHADOW_METADATA_FILE})" "${ACTUAL_TRAILER_TAG}"
+
+# Stop metalogger service and verify created changelogs
+saunafs_metalogger_daemon stop
+assertlocal_file_exists "${info[master0_data_path]}/changelog_ml.sfs"
+
+# Expected Values after reload
+metadata_file="${info[master0_data_path]}/metadata.sfs"
+assert_equals "$(head -c8 ${metadata_file})" "${ACTUAL_HEADER_TAG}"
+assert_equals "$(tail -c16 ${metadata_file})" "${ACTUAL_TRAILER_TAG}"

From 22bd6e8d23598bf2eaf78a35b056a9ee4648544d Mon Sep 17 00:00:00 2001
From: Luis Guillermo Silva Rojas <guillex@leil.io>
Date: Fri, 19 Jul 2024 10:06:30 -0400
Subject: [PATCH 05/27] feat(chunkserver): Extend IDiskManager interface (#143)

This change adds two new abstract functions to IDiskManager, to
increase the flexibility for the plugins.
---
 .../chunkserver-common/default_disk_manager.h            | 9 +++++++++
 .../chunkserver-common/disk_manager_interface.h          | 6 ++++++
 src/chunkserver/hddspacemgr.cc                           | 4 ++++
 3 files changed, 19 insertions(+)

diff --git a/src/chunkserver/chunkserver-common/default_disk_manager.h b/src/chunkserver/chunkserver-common/default_disk_manager.h
index 0f4d4b09a..17d00e755 100644
--- a/src/chunkserver/chunkserver-common/default_disk_manager.h
+++ b/src/chunkserver/chunkserver-common/default_disk_manager.h
@@ -47,6 +47,11 @@ class DefaultDiskManager : public IDiskManager {
 	/// Virtual destructor needed for correct polymorphism
 	virtual ~DefaultDiskManager() = default;
 
+	/// Reload the configuration from the configuration file.
+	/// No need to reload the configuration so far for this implementation.
+	void reloadConfiguration() override {};
+
+	/// Parse a configuration line for a disk.
 	virtual int parseCfgLine(std::string hddCfgLine);
 
 	/// Reload the disks from the configuration and populates the gDisks vector.
@@ -60,4 +65,8 @@ class DefaultDiskManager : public IDiskManager {
 	 */
 	IDisk *getDiskForNewChunk(
 	    [[maybe_unused]] const ChunkPartType &chunkType) override;
+
+	/// Update the space usage of the disks.
+	/// No need to update the space usage here for this implementation.
+	void updateSpaceUsage() override {};
 };
diff --git a/src/chunkserver/chunkserver-common/disk_manager_interface.h b/src/chunkserver/chunkserver-common/disk_manager_interface.h
index ce51d325c..9bfec908c 100644
--- a/src/chunkserver/chunkserver-common/disk_manager_interface.h
+++ b/src/chunkserver/chunkserver-common/disk_manager_interface.h
@@ -44,6 +44,9 @@ class IDiskManager {
 	/// Virtual destructor needed for correct polymorphism
 	virtual ~IDiskManager() = default;
 
+	/// Reload the configuration from the configuration file.
+	virtual void reloadConfiguration() = 0;
+
 	/// Reload the disks from the configuration and populates the gDisks vector.
 	virtual void reloadDisksFromCfg() = 0;
 
@@ -51,4 +54,7 @@ class IDiskManager {
 	/// different strategies.
 	virtual IDisk *getDiskForNewChunk(
 	    [[maybe_unused]] const ChunkPartType &chunkType) = 0;
+
+	/// Update the space usage of the disks.
+	virtual void updateSpaceUsage() = 0;
 };
diff --git a/src/chunkserver/hddspacemgr.cc b/src/chunkserver/hddspacemgr.cc
index bca68241a..6c525b0d7 100644
--- a/src/chunkserver/hddspacemgr.cc
+++ b/src/chunkserver/hddspacemgr.cc
@@ -394,6 +394,8 @@ void hddCheckDisks() {
 		}
 	}
 
+	gDiskManager->updateSpaceUsage();
+
 	disksUniqueLock.unlock();
 
 	if (changed) {
@@ -2536,6 +2538,7 @@ void hddReload(void) {
 	safs_pretty_syslog(LOG_NOTICE,"reloading hdd data ...");
 
 	try {
+		gDiskManager->reloadConfiguration();
 		gDiskManager->reloadDisksFromCfg();
 	} catch (const Exception& ex) {
 		safs_pretty_syslog(LOG_ERR, "%s", ex.what());
@@ -2635,6 +2638,7 @@ int hddInit() {
 	}
 
 	try {
+		gDiskManager->reloadConfiguration();
 		gDiskManager->reloadDisksFromCfg();
 	} catch (const Exception& ex) {
 		safs_pretty_syslog(LOG_ERR, "%s", ex.what());

From 573a812ee2f1171558804b748842ac80a06b4b18 Mon Sep 17 00:00:00 2001
From: Urmas Rist <urmas@leil.io>
Date: Mon, 22 Jul 2024 18:28:46 +0300
Subject: [PATCH 06/27] fix: Release compilation on GCC 14 (#148)

When building with "Release" CMAKE_BUILD_TYPE, the compilation fails on
Arch Linux with GCC 14.1.1, complaining about uninitialized array's.

This small commit fixes that and two linter errors.
---
 src/common/goal.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/common/goal.cc b/src/common/goal.cc
index bcae7efbe..43f8fb793 100644
--- a/src/common/goal.cc
+++ b/src/common/goal.cc
@@ -63,8 +63,8 @@ void Goal::Slice::mergeIn(const Slice &other) {
 	assert(type_ == other.type_);
 	assert(size() == other.size());
 
-	std::array<std::array<int, kMaxPartsCount>, kMaxPartsCount> cost;
-	std::array<int, kMaxPartsCount> assignment;
+	std::array<std::array<int, kMaxPartsCount>, kMaxPartsCount> cost{};
+	std::array<int, kMaxPartsCount> assignment{};
 
 	Labels tmp_union;
 

From e6022982e01cafb7a07ba5b05616862b4cc07e56 Mon Sep 17 00:00:00 2001
From: Antuan <antuan@leil.io>
Date: Wed, 24 Jul 2024 14:05:33 +0200
Subject: [PATCH 07/27] fix: Update uraft service to restart on failure (#153)

fix: Update uraft service to restart on failure

The service was not attempting to automatically restart if a failure occurred.
The retry time slapsed was set to 60 seconds.
---
 rpm/service-files/saunafs-uraft.service | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rpm/service-files/saunafs-uraft.service b/rpm/service-files/saunafs-uraft.service
index 4fff0e297..48036fdb0 100644
--- a/rpm/service-files/saunafs-uraft.service
+++ b/rpm/service-files/saunafs-uraft.service
@@ -10,7 +10,8 @@ PIDFile=/var/run/saunafs-uraft.pid
 TimeoutSec=0
 ExecStart=/usr/sbin/saunafs-uraft
 ExecStopPost=/usr/sbin/saunafs-uraft-helper demote
-Restart=no
+Restart=on-failure
+RestartSec=60
 User=saunafs
 
 [Install]

From 9078ae809d859b25aeafb395f05c8102294485c8 Mon Sep 17 00:00:00 2001
From: aNeutrino <piotr.modrzyk@gmail.com>
Date: Fri, 26 Jul 2024 14:25:43 +0200
Subject: [PATCH 08/27] fix: Update services to wait for network (#151)

network.target is not giving warranty that IP adresses are set
network-online.target gives sush assurances
https://systemd.io/NETWORK_ONLINE/

Co-authored-by: Antuan <antuan@leil.io>
Co-authored-by: Crash <crash@leil.io>
---
 rpm/service-files/saunafs-cgiserv.service     | 2 +-
 rpm/service-files/saunafs-chunkserver.service | 2 +-
 rpm/service-files/saunafs-ha-master.service   | 2 +-
 rpm/service-files/saunafs-master.service      | 2 +-
 rpm/service-files/saunafs-metalogger.service  | 2 +-
 rpm/service-files/saunafs-uraft.service       | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/rpm/service-files/saunafs-cgiserv.service b/rpm/service-files/saunafs-cgiserv.service
index 5c49b1f07..27790651b 100644
--- a/rpm/service-files/saunafs-cgiserv.service
+++ b/rpm/service-files/saunafs-cgiserv.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=SaunaFS CGI server daemon
 Documentation=man:saunafs-cgiserver
-After=network.target
+After=network-online.target network.target
 
 [Service]
 Environment=BIND_HOST=0.0.0.0
diff --git a/rpm/service-files/saunafs-chunkserver.service b/rpm/service-files/saunafs-chunkserver.service
index 2b714353e..9681e7777 100644
--- a/rpm/service-files/saunafs-chunkserver.service
+++ b/rpm/service-files/saunafs-chunkserver.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=SaunaFS chunkserver daemon
 Documentation=man:sfschunkserver
-After=network.target
+After=network-online.target network.target
 
 [Service]
 Type=forking
diff --git a/rpm/service-files/saunafs-ha-master.service b/rpm/service-files/saunafs-ha-master.service
index 036f58420..d824bc5ae 100644
--- a/rpm/service-files/saunafs-ha-master.service
+++ b/rpm/service-files/saunafs-ha-master.service
@@ -1,6 +1,6 @@
 [Unit]
 Description=SaunaFS master server daemon
-After=syslog.target network.target
+After=syslog.target network-online.target network.target
 PartOf=saunafs-uraft.service
 
 [Service]
diff --git a/rpm/service-files/saunafs-master.service b/rpm/service-files/saunafs-master.service
index 0015095e3..07280b6fc 100644
--- a/rpm/service-files/saunafs-master.service
+++ b/rpm/service-files/saunafs-master.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=SaunaFS master server daemon
 Documentation=man:sfsmaster
-After=network.target
+After=network-online.target network.target
 
 [Service]
 Type=forking
diff --git a/rpm/service-files/saunafs-metalogger.service b/rpm/service-files/saunafs-metalogger.service
index 885f90197..4b93b488c 100644
--- a/rpm/service-files/saunafs-metalogger.service
+++ b/rpm/service-files/saunafs-metalogger.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=SaunaFS metalogger server daemon
 Documentation=man:sfsmetalogger
-After=network.target
+After=network-online.target network.target
 
 [Service]
 Type=forking
diff --git a/rpm/service-files/saunafs-uraft.service b/rpm/service-files/saunafs-uraft.service
index 48036fdb0..94486e7a4 100644
--- a/rpm/service-files/saunafs-uraft.service
+++ b/rpm/service-files/saunafs-uraft.service
@@ -1,7 +1,7 @@
 [Unit]
 Description=SaunaFS uraft high availability daemon
 Requires=saunafs-ha-master.service
-After=network.target
+After=network-online.target network.target
 After=saunafs-ha-master.service
 
 [Service]

From 4e17835f004b64544e52a79dab6db2c3be08db5d Mon Sep 17 00:00:00 2001
From: Walter Gomez <baldor@leil.io>
Date: Thu, 8 Aug 2024 11:58:05 +0200
Subject: [PATCH 09/27] fix(tests): Amend upgrade tests (#146)

- Add authentication to APT

If the host has Auth set, it will copy and use those credentials.

- Update Long Upgrade tests

Some tests are not required anymore since the legacy chunk formats where
removed from the project.

- Update some paths not to require sudo

-  Move back the tests to the original suite

Co-authored-by: Guillex <guillex@leil.io>
---
 tests/data/LongSystemTests.txt                |  3 ---
 .../test_upgrade_overwrite_file.sh}           |  1 -
 .../test_upgrade_undergoal_chunks_ec21.sh}    |  1 -
 .../test_upgrade_undergoal_chunks_ec32.sh}    |  1 -
 ...grade_overwrite_file_legacy_chunkformat.sh |  3 ---
 ...ndergoal_chunks_ec21_legacy_chunkformat.sh |  3 ---
 ...ndergoal_chunks_ec32_legacy_chunkformat.sh |  3 ---
 .../test_upgrade_ec_basic.sh}                 |  0
 .../test_upgrade_general.sh}                  |  0
 ...est_upgrade_new_mount_with_old_saunafs.sh} |  0
 .../test_saunafs_upgrade_overwrite_file.inc   |  3 ---
 .../test_saunafs_upgrade_undergoal_chunks.inc |  4 ---
 tests/tools/saunafsXX.sh                      | 27 ++++++++++++++-----
 13 files changed, 20 insertions(+), 29 deletions(-)
 rename tests/test_suites/{LongUpgradeTests/test_saunafs_upgrade_overwrite_file.sh => LongSystemTests/test_upgrade_overwrite_file.sh} (77%)
 rename tests/test_suites/{LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec21.sh => LongSystemTests/test_upgrade_undergoal_chunks_ec21.sh} (78%)
 rename tests/test_suites/{LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec32.sh => LongSystemTests/test_upgrade_undergoal_chunks_ec32.sh} (78%)
 delete mode 100644 tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_overwrite_file_legacy_chunkformat.sh
 delete mode 100644 tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec21_legacy_chunkformat.sh
 delete mode 100644 tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec32_legacy_chunkformat.sh
 rename tests/test_suites/{ShortUpgradeTests/test_saunafs_upgrade_ec_basic.sh => ShortSystemTests/test_upgrade_ec_basic.sh} (100%)
 rename tests/test_suites/{ShortUpgradeTests/test_saunafs_upgrade_general.sh => ShortSystemTests/test_upgrade_general.sh} (100%)
 rename tests/test_suites/{ShortUpgradeTests/test_saunafs_upgrade_new_mount_with_old_saunafs.sh => ShortSystemTests/test_upgrade_new_mount_with_old_saunafs.sh} (100%)

diff --git a/tests/data/LongSystemTests.txt b/tests/data/LongSystemTests.txt
index 41a4eb061..adea372c7 100644
--- a/tests/data/LongSystemTests.txt
+++ b/tests/data/LongSystemTests.txt
@@ -34,11 +34,8 @@ test_data_integrity=2700
 test_dbench=2160
 test_disk_to_delete_replication=10
 test_saunafs_upgrade_overwrite_file=2010
-test_saunafs_upgrade_overwrite_file_legacy_chunkformat=1950
 test_saunafs_upgrade_undergoal_chunks_ec21=970
-test_saunafs_upgrade_undergoal_chunks_ec21_legacy_chunkformat=1020
 test_saunafs_upgrade_undergoal_chunks_ec32=1080
-test_saunafs_upgrade_undergoal_chunks_ec32_legacy_chunkformat=1000
 test_punching_holes=5
 test_xor_overwriting_faulty_chunkservers=660
 test_xor_overwriting_faulty_master=1080
diff --git a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_overwrite_file.sh b/tests/test_suites/LongSystemTests/test_upgrade_overwrite_file.sh
similarity index 77%
rename from tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_overwrite_file.sh
rename to tests/test_suites/LongSystemTests/test_upgrade_overwrite_file.sh
index 63f89a070..83a593507 100644
--- a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_overwrite_file.sh
+++ b/tests/test_suites/LongSystemTests/test_upgrade_overwrite_file.sh
@@ -1,3 +1,2 @@
-LEGACY_CHUNK_FORMAT="0"
 GOAL=ec21
 source test_suites/TestTemplates/test_saunafs_upgrade_overwrite_file.inc
diff --git a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec21.sh b/tests/test_suites/LongSystemTests/test_upgrade_undergoal_chunks_ec21.sh
similarity index 78%
rename from tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec21.sh
rename to tests/test_suites/LongSystemTests/test_upgrade_undergoal_chunks_ec21.sh
index 4f3d82d67..39dfb87e2 100644
--- a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec21.sh
+++ b/tests/test_suites/LongSystemTests/test_upgrade_undergoal_chunks_ec21.sh
@@ -1,3 +1,2 @@
-LEGACY_CHUNK_FORMAT="0"
 GOAL="ec21"
 source test_suites/TestTemplates/test_saunafs_upgrade_undergoal_chunks.inc
diff --git a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec32.sh b/tests/test_suites/LongSystemTests/test_upgrade_undergoal_chunks_ec32.sh
similarity index 78%
rename from tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec32.sh
rename to tests/test_suites/LongSystemTests/test_upgrade_undergoal_chunks_ec32.sh
index f291fc206..545e88e86 100644
--- a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec32.sh
+++ b/tests/test_suites/LongSystemTests/test_upgrade_undergoal_chunks_ec32.sh
@@ -1,3 +1,2 @@
-LEGACY_CHUNK_FORMAT="0"
 GOAL="ec32"
 source test_suites/TestTemplates/test_saunafs_upgrade_undergoal_chunks.inc
diff --git a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_overwrite_file_legacy_chunkformat.sh b/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_overwrite_file_legacy_chunkformat.sh
deleted file mode 100644
index b6ab1cc1e..000000000
--- a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_overwrite_file_legacy_chunkformat.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-LEGACY_CHUNK_FORMAT="1"
-GOAL=ec21
-source test_suites/TestTemplates/test_saunafs_upgrade_overwrite_file.inc
diff --git a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec21_legacy_chunkformat.sh b/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec21_legacy_chunkformat.sh
deleted file mode 100644
index 17be83903..000000000
--- a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec21_legacy_chunkformat.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-LEGACY_CHUNK_FORMAT="1"
-GOAL="ec21"
-source test_suites/TestTemplates/test_saunafs_upgrade_undergoal_chunks.inc
diff --git a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec32_legacy_chunkformat.sh b/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec32_legacy_chunkformat.sh
deleted file mode 100644
index 947dd8cf4..000000000
--- a/tests/test_suites/LongUpgradeTests/test_saunafs_upgrade_undergoal_chunks_ec32_legacy_chunkformat.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-LEGACY_CHUNK_FORMAT="1"
-GOAL="ec32"
-source test_suites/TestTemplates/test_saunafs_upgrade_undergoal_chunks.inc
diff --git a/tests/test_suites/ShortUpgradeTests/test_saunafs_upgrade_ec_basic.sh b/tests/test_suites/ShortSystemTests/test_upgrade_ec_basic.sh
similarity index 100%
rename from tests/test_suites/ShortUpgradeTests/test_saunafs_upgrade_ec_basic.sh
rename to tests/test_suites/ShortSystemTests/test_upgrade_ec_basic.sh
diff --git a/tests/test_suites/ShortUpgradeTests/test_saunafs_upgrade_general.sh b/tests/test_suites/ShortSystemTests/test_upgrade_general.sh
similarity index 100%
rename from tests/test_suites/ShortUpgradeTests/test_saunafs_upgrade_general.sh
rename to tests/test_suites/ShortSystemTests/test_upgrade_general.sh
diff --git a/tests/test_suites/ShortUpgradeTests/test_saunafs_upgrade_new_mount_with_old_saunafs.sh b/tests/test_suites/ShortSystemTests/test_upgrade_new_mount_with_old_saunafs.sh
similarity index 100%
rename from tests/test_suites/ShortUpgradeTests/test_saunafs_upgrade_new_mount_with_old_saunafs.sh
rename to tests/test_suites/ShortSystemTests/test_upgrade_new_mount_with_old_saunafs.sh
diff --git a/tests/test_suites/TestTemplates/test_saunafs_upgrade_overwrite_file.inc b/tests/test_suites/TestTemplates/test_saunafs_upgrade_overwrite_file.inc
index 79c36e40d..3ee102693 100644
--- a/tests/test_suites/TestTemplates/test_saunafs_upgrade_overwrite_file.inc
+++ b/tests/test_suites/TestTemplates/test_saunafs_upgrade_overwrite_file.inc
@@ -12,14 +12,11 @@ timeout_set 120 minutes
 # - SAFS_MOUNT_COMMAND_AFTER_UPGRADE (mount command to use after
 #                                     upgrade to current SaunaFS version was done)
 
-export SAFS_MOUNT_COMMAND="sfsmount"
-
 source test_utils/upgrade.sh
 
 CHUNKSERVERS=3
 START_WITH_LEGACY_SAUNAFS=YES \
 	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER" \
-	CHUNKSERVER_EXTRA_CONFIG="CREATE_NEW_CHUNKS_IN_LEGACY_FORMAT = $LEGACY_CHUNK_FORMAT" \
 	MASTER_EXTRA_CONFIG="CHUNKS_LOOP_MIN_TIME = 1|OPERATIONS_DELAY_INIT = 0" \
 	setup_local_empty_saunafs info
 
diff --git a/tests/test_suites/TestTemplates/test_saunafs_upgrade_undergoal_chunks.inc b/tests/test_suites/TestTemplates/test_saunafs_upgrade_undergoal_chunks.inc
index 6652a528c..2caec8700 100644
--- a/tests/test_suites/TestTemplates/test_saunafs_upgrade_undergoal_chunks.inc
+++ b/tests/test_suites/TestTemplates/test_saunafs_upgrade_undergoal_chunks.inc
@@ -9,7 +9,6 @@ timeout_set 120 minutes
 # - GOAL (e.g. 2, ec21, ec32)
 # - CHUNKSERVERS_GOAL_COVER (compatible with GOAL, e.g. goal 2 -> 2, ec32 -> 5)
 # - CHUNKSERVERS_REDUNDANT (how many we turn off, e.g. GOAL ec32 -> 2, ec21 -> 1)
-# - LEGACY_CHUNK_FORMAT ("1" when old chunk format, "0" otherwise)
 CHUNKSERVERS_GOAL_COVER=$(goal_to_part_count ${GOAL})
 CHUNKSERVERS_REDUNDANT=$(redundant_parts ${GOAL})
 CHUNKSERVERS_MINIMUM=$((CHUNKSERVERS_GOAL_COVER - ${CHUNKSERVERS_REDUNDANT}))
@@ -17,12 +16,9 @@ REPLICATION_SPEED="5000000" # Bytes per second
 
 source test_utils/upgrade.sh
 
-export SAFS_MOUNT_COMMAND="sfsmount"
-
 CHUNKSERVERS=$CHUNKSERVERS_GOAL_COVER
 	START_WITH_LEGACY_SAUNAFS=YES \
 	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER" \
-	CHUNKSERVER_EXTRA_CONFIG="CREATE_NEW_CHUNKS_IN_LEGACY_FORMAT = $LEGACY_CHUNK_FORMAT" \
 	MASTER_EXTRA_CONFIG="CHUNKS_LOOP_MIN_TIME = 1|OPERATIONS_DELAY_INIT = 0" \
 	setup_local_empty_saunafs info
 
diff --git a/tests/tools/saunafsXX.sh b/tests/tools/saunafsXX.sh
index bd3de7c45..3a8e4cb37 100644
--- a/tests/tools/saunafsXX.sh
+++ b/tests/tools/saunafsXX.sh
@@ -8,7 +8,10 @@
 SAUNAFSXX_TAG="4.1.0"
 
 install_saunafsXX() {
-	rm -rf "${SAUNAFSXX_DIR}"
+	if [ "${SAUNAFSXX_DIR:?}" == "/" ]; then
+		test_fail "Cowardly refusing to perform potentially destructive operation on /"
+	fi
+	rm -rf "${SAUNAFSXX_DIR:?}"
 	mkdir -p "${SAUNAFSXX_DIR}"
 	local distro="$(lsb_release -si)"
 	case "${distro}" in
@@ -17,20 +20,30 @@ install_saunafsXX() {
 		local codename="$(lsb_release -sc | tail -1)"
 		local release="$(lsb_release -sr | tail -1)"
 		mkdir -p "${TEMP_DIR}/apt/apt.conf.d"
-		mkdir -p "${TEMP_DIR}/apt/var/lib/apt/partial"
+		mkdir -p "${TEMP_DIR}/apt/auth.conf.d"
+		mkdir -p "${TEMP_DIR}/apt/sources.list.d"
 		mkdir -p "${TEMP_DIR}/apt/var/cache/apt/archives/partial"
+		mkdir -p "${TEMP_DIR}/apt/var/lib/apt/partial"
 		mkdir -p "${TEMP_DIR}/apt/var/lib/dpkg"
 		mkdir -p "${TEMP_DIR}/usr/share/keyrings"
 		cp /var/lib/dpkg/status "${TEMP_DIR}/apt/var/lib/dpkg/status"
+		if [ -f /tmp/saunafs-test-auth.conf ]; then
+			cp /tmp/saunafs-test-auth.conf "${TEMP_DIR}/apt/auth.conf.d/saunafs.conf"
+		fi
 		cat >"${TEMP_DIR}/apt/apt.conf" <<END
-Dir::State "${TEMP_DIR}/apt/var/lib/apt";
-Dir::State::status "${TEMP_DIR}/apt/var/lib/dpkg/status";
-Dir::Etc::SourceList "${TEMP_DIR}/apt/saunafs.list";
 Dir::Cache "${TEMP_DIR}/apt/var/cache/apt";
 Dir::Etc::Parts "${TEMP_DIR}/apt/apt.conf.d";
+Dir::Etc::netrc "${TEMP_DIR}/apt/auth.conf";
+Dir::Etc::netrcparts "${TEMP_DIR}/apt/auth.conf.d";
+Dir::Etc::SourceList "${TEMP_DIR}/apt/saunafs.list";
+Dir::Etc::SourceParts "${TEMP_DIR}/apt/sources.list.d";
+Dir::State "${TEMP_DIR}/apt/var/lib/apt";
+Dir::State::status "${TEMP_DIR}/apt/var/lib/dpkg/status";
 END
+		curl -sSL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xa80b96e2c79457d4" | \
+			gpg --dearmor -o "${TEMP_DIR}/apt/saunafs-archive-keyring.gpg"
 		local destdir="${TEMP_DIR}/apt/var/cache/apt/archives"
-		echo "deb [arch=amd64 signed-by=/usr/share/keyrings/saunafs-archive-keyring.gpg] https://repo.saunafs.com/repository/saunafs-${distro_id}-${release}/ ${codename} main" >"${TEMP_DIR}/apt/saunafs.list"
+		echo "deb [arch=amd64 signed-by=${TEMP_DIR}/apt/saunafs-archive-keyring.gpg] https://repo.saunafs.com/repository/saunafs-${distro_id}-${release}/ ${codename} main" >"${TEMP_DIR}/apt/saunafs.list"
 		env APT_CONFIG="${TEMP_DIR}/apt/apt.conf" apt-get update
 		env APT_CONFIG="${TEMP_DIR}/apt/apt.conf" apt-get install --yes libyaml-cpp*
 		SAUNAFSXX_TAG_APT="4.1.0-20240509-152518-stable-main-a7cb5669"
@@ -115,7 +128,7 @@ test_saunafsXX_executables() {
 
 saunafsXX_chunkserver_daemon() {
 	# shellcheck disable=SC2154
-	"${SAUNAFSXX_DIR}/sbin/sfschunkserver" -c "${saunafs_info_[chunkserver${1_cfg}]}" "${2}" | cat
+	"${SAUNAFSXX_DIR}/sbin/sfschunkserver" -c "${saunafs_info_["chunkserver${1}_cfg"]}" "${2}" | cat
 	return "${PIPESTATUS[0]}"
 }
 

From 091b755c899b7c4f83b6ae789d3d63cdb3d69712 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Manuel=20Garc=C3=ADa=20Aguilera?= <dave@leil.io>
Date: Thu, 8 Aug 2024 22:03:42 -0700
Subject: [PATCH 10/27] fix(mount): Fix IIS permissions mess (#154)

fix(mount): Fix IIS permissions mess

This commit aims to provide the expected behavior of our system while
dealing with the IIS tool. This tool apparently uses a very specific
user (17) to access the underlaying files. The user 17 was not expected
to perform these kinds of operations, thus causing the permission denied
errors. Solution was to make a breach at some checks to allow it using
the UIDs provided by the ```sfsallowedusers``` option.
---
 src/mount/sauna_client.cc | 22 ++++++++++++----------
 src/mount/sauna_client.h  |  2 ++
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/mount/sauna_client.cc b/src/mount/sauna_client.cc
index 077d3d4e9..5ddfc50c4 100644
--- a/src/mount/sauna_client.cc
+++ b/src/mount/sauna_client.cc
@@ -233,6 +233,7 @@ uint8_t get_session_flags() { return session_flags; }
 bool isMasterDisconnected() { return gIsDisconnectedFromMaster.load(); }
 
 #define LOCAL_USERS_THRESHOLD 0x30000
+std::unordered_set<uint32_t> allowed_users;
 static int32_t mounting_uid = USE_LOCAL_ID;
 static int32_t mounting_gid = USE_LOCAL_ID;
 uint32_t winfsp_uid = 0;
@@ -242,13 +243,13 @@ std::mutex winfsp_context_mutex;
 void update_last_winfsp_context(const uint32_t uid, const uint32_t gid) {
 	std::lock_guard<std::mutex> winfsp_context_lock(winfsp_context_mutex);
 
-	if (uid > LOCAL_USERS_THRESHOLD) {
+	if (uid > LOCAL_USERS_THRESHOLD || allowed_users.contains(uid)) {
 		if (debug_mode && winfsp_uid != uid) {
 			safs::log_debug("set WinFSP user id to {}", uid);
 		}
 		winfsp_uid = uid;
 	}
-	if (gid > LOCAL_USERS_THRESHOLD) {
+	if (gid > LOCAL_USERS_THRESHOLD || allowed_users.contains(gid)) {
 		if (debug_mode && winfsp_gid != gid) {
 			safs::log_debug("set WinFSP group id to {}", uid);
 		}
@@ -267,12 +268,12 @@ void convert_winfsp_context_to_master_context(uint32_t &uid, uint32_t &gid) {
 }
 
 void patch_uid_gid_fields(stat32 &st) {
-	if (((st.st_uid ==
-	      (mounting_uid == USE_LOCAL_ID ? winfsp_uid : mounting_uid)) ||
-	     (st.st_mode & S_IWOTH) ||
-	     (st.st_gid ==
-	          (mounting_gid == USE_LOCAL_ID ? winfsp_gid : mounting_gid) &&
-	      (st.st_mode & S_IWGRP)))) {
+	if ((st.st_uid ==
+	     (mounting_uid == USE_LOCAL_ID ? winfsp_uid : mounting_uid)) ||
+	    (st.st_mode & S_IWOTH) ||
+	    (st.st_gid ==
+	         (mounting_gid == USE_LOCAL_ID ? winfsp_gid : mounting_gid) &&
+	     (st.st_mode & S_IWGRP))) {
 		st.st_uid = winfsp_uid;
 		st.st_gid = winfsp_gid;
 	}
@@ -3391,13 +3392,14 @@ void init(int debug_mode_, int keep_cache_, double direntry_cache_timeout_, unsi
 		SugidClearMode sugid_clear_mode_, bool use_rwlock_,
 		double acl_cache_timeout_, unsigned acl_cache_size_, bool direct_io
 #ifdef _WIN32
-		, int mounting_uid_, int mounting_gid_
+		, int mounting_uid_, int mounting_gid_, std::unordered_set<uint32_t> &allowed_users_
 #endif
 		, bool ignore_flush_
 		) {
 #ifdef _WIN32
 	mounting_uid = mounting_uid_;
 	mounting_gid = mounting_gid_;
+	allowed_users = allowed_users_;
 #endif
 	ignore_flush = ignore_flush_;
 	debug_mode = debug_mode_;
@@ -3493,7 +3495,7 @@ void fs_init(FsInitParams &params) {
 		params.sugid_clear_mode, params.use_rw_lock,
 		params.acl_cache_timeout, params.acl_cache_size, params.direct_io
 #ifdef _WIN32
-		, params.mounting_uid, params.mounting_gid
+		, params.mounting_uid, params.mounting_gid, params.allowed_users
 #endif
 		, params.ignore_flush
 		);
diff --git a/src/mount/sauna_client.h b/src/mount/sauna_client.h
index 068b9aafd..9484274cb 100644
--- a/src/mount/sauna_client.h
+++ b/src/mount/sauna_client.h
@@ -27,6 +27,7 @@
 #include <unistd.h>
 
 #include <string>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -226,6 +227,7 @@ struct FsInitParams {
 #ifdef _WIN32
 	int mounting_uid;
 	int mounting_gid;
+	std::unordered_set<uint32_t> allowed_users;
 #endif
 
 	bool ignore_flush;

From c597cfe503b6d255dc17bd86df5c5c3358fef538 Mon Sep 17 00:00:00 2001
From: Baldor <baldor@leil.io>
Date: Wed, 17 Jul 2024 19:46:31 +0200
Subject: [PATCH 11/27] fix(tests): Fix typo in setup-machine script

Co-authored-by: Guillex <guillex@leil.io>
---
 tests/setup_machine.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/setup_machine.sh b/tests/setup_machine.sh
index 8fd252a0e..d10893581 100755
--- a/tests/setup_machine.sh
+++ b/tests/setup_machine.sh
@@ -267,7 +267,7 @@ fi
 # Add VIRTUAL_ENV to /etc/environment if it's not there
 sed '\@VIRTUAL_ENV="'"${VIRTUAL_ENV}"'"@!s@$@\nVIRTUAL_ENV="'"${VIRTUAL_ENV}"'"@' -zi /etc/environment
 python3 -m venv "${VIRTUAL_ENV}"
-"${VIRTUAL_ENV}/bin/python3" -m pip install install "${python_packages[@]}"
+"${VIRTUAL_ENV}/bin/python3" -m pip install "${python_packages[@]}"
 
 echo ; echo 'Add group fuse'
 groupadd -f fuse

From 99eaa88f51fc97ff71b2103609e90348c8e8b197 Mon Sep 17 00:00:00 2001
From: Ictus <ictus4u@gmail.com>
Date: Wed, 7 Aug 2024 13:28:29 +0200
Subject: [PATCH 12/27] ci: Update de base docker image on build

---
 tests/ci_build/docker-build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci_build/docker-build.sh b/tests/ci_build/docker-build.sh
index f41a87cd4..08b5d60aa 100755
--- a/tests/ci_build/docker-build.sh
+++ b/tests/ci_build/docker-build.sh
@@ -178,7 +178,7 @@ if parse_true "${DOCKER_ENABLE_PULL_CACHE_IMAGE}"; then
 	build_extra_args+=(--cache-from "${DOCKER_INTERNAL_REGISTRY}/${DOCKER_IMAGE}:${latest_tag}")
 fi
 
-run docker buildx build --load --progress=plain \
+run docker buildx build --pull --load --progress=plain \
 	--build-arg GROUP_ID="${DOCKER_IMAGE_GROUP_ID}" \
 	--build-arg USER_ID="${DOCKER_IMAGE_USER_ID}" \
 	--build-arg USERNAME="${DOCKER_IMAGE_USERNAME}" \

From ca04785019486c100317837e5fe8b7c734eabd60 Mon Sep 17 00:00:00 2001
From: Ictus <ictus4u@gmail.com>
Date: Fri, 9 Aug 2024 14:45:53 +0200
Subject: [PATCH 13/27] ci: Abort previous concurrent tests

No need of building from outdated code.
---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 98531346d..963e96adf 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -8,6 +8,7 @@ pipeline {
         ansiColor("xterm")
         parallelsAlwaysFailFast()
         preserveStashes(buildCount: 2)
+        disableConcurrentBuilds(abortPrevious: true)
     }
     stages {
         stage('Build Images') {

From adbb95eb605843b4695391a13fc2ef7a941566fc Mon Sep 17 00:00:00 2001
From: guillex <guillex@leil.io>
Date: Mon, 12 Aug 2024 17:11:40 +0200
Subject: [PATCH 14/27] chore: Changelog for v4.5.0

Co-authored-by: Crash <crash@leil.io>
Co-authored-by: Dave <dave@leil.io>
---
 CMakeLists.txt   |  2 +-
 NEWS             | 14 ++++++++++++++
 debian/changelog | 16 ++++++++++++++++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 09c1bb914..c688023c5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -189,7 +189,7 @@ endif()
 project(saunafs)
 
 if(NOT PACKAGE_VERSION)
-  set(DEFAULT_MIN_VERSION "4.4.0")
+  set(DEFAULT_MIN_VERSION "4.5.0")
   execute_process(COMMAND "git" "rev-parse" "--short" "HEAD"
     OUTPUT_VARIABLE GIT_SHA1_SHORT RESULT_VARIABLE GIT_SHA1_SHORT_RETURN_VALUE
     OUTPUT_STRIP_TRAILING_WHITESPACE)
diff --git a/NEWS b/NEWS
index 4e6aef21c..7d1c89aa6 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,19 @@
 This file lists noteworthy changes in SaunaFS.
 
+ * SaunaFS (4.5.0) (2024-08-12)
+ - ci: Abort previous concurrent tests
+ - ci: Update de base docker image on build
+ - fix(tests): Fix typo in setup-machine script
+ - fix(tests): Amend upgrade tests (#146)
+ - fix: Update services to wait for network (#151)
+ - fix: Update uraft service to restart on failure (#153)
+ - fix: Release compilation on GCC 14 (#148)
+ - feat(chunkserver): Extend IDiskManager interface (#143)
+ - test: Verify master loading legacy metadata
+ - feat: Load legacy metadata file
+ - fix: Allow legacy sessions download
+ - fix: Allow legacy metadata download
+
  * SaunaFS (4.4.0) (2024-07-31)
  - ci(.github): Add check for Conventional Commits (#131)
  - feat(metrics): Add initial prometheus support (#98)
diff --git a/debian/changelog b/debian/changelog
index 17e76779e..a12f9d596 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,19 @@
+saunafs (4.5.0) stable; urgency=medium
+  * ci: Abort previous concurrent tests
+  * ci: Update de base docker image on build
+  * fix(tests): Fix typo in setup-machine script
+  * fix(tests): Amend upgrade tests (#146)
+  * fix: Update services to wait for network (#151)
+  * fix: Update uraft service to restart on failure (#153)
+  * fix: Release compilation on GCC 14 (#148)
+  * feat(chunkserver): Extend IDiskManager interface (#143)
+  * test: Verify master loading legacy metadata
+  * feat: Load legacy metadata file
+  * fix: Allow legacy sessions download
+  * fix: Allow legacy metadata download
+
+ -- SaunaFS Team <contact@saunafs.com>  Mon, 12 Aug 2024 15:11:39 +0000
+
 saunafs (4.4.0) stable; urgency=medium
   * ci(.github): Add check for Conventional Commits (#131)
   * feat(metrics): Add initial prometheus support (#98)

From 6842e9758415aded80b13bfe576f2fe176dca1c0 Mon Sep 17 00:00:00 2001
From: Luis Guillermo Silva Rojas <guillex@leil.io>
Date: Wed, 14 Aug 2024 20:47:44 +0200
Subject: [PATCH 15/27] feat(admin): Add list-disk-groups command (#155)

The saunafs-admin was extended to show the disk groups configuration
from the chunkservers. The default disk manager does not support this
feature yet and prints 'Not supported', but it is already available
(and used) from the plugins by reimplementing the default behavior.
---
 src/admin/list_disk_groups_command.cc         | 98 +++++++++++++++++++
 src/admin/list_disk_groups_command.h          | 30 ++++++
 src/admin/main.cc                             |  2 +
 .../chunkserver-common/default_disk_manager.h |  4 +
 .../disk_manager_interface.h                  |  3 +
 src/chunkserver/hddspacemgr.cc                |  6 ++
 src/chunkserver/hddspacemgr.h                 |  2 +
 src/chunkserver/network_worker_thread.cc      | 19 ++++
 src/protocol/SFSCommunication.h               |  8 ++
 9 files changed, 172 insertions(+)
 create mode 100644 src/admin/list_disk_groups_command.cc
 create mode 100644 src/admin/list_disk_groups_command.h

diff --git a/src/admin/list_disk_groups_command.cc b/src/admin/list_disk_groups_command.cc
new file mode 100644
index 000000000..579a508d0
--- /dev/null
+++ b/src/admin/list_disk_groups_command.cc
@@ -0,0 +1,98 @@
+/*
+   Copyright 2023 Leil Storage OÜ
+
+   This file is part of SaunaFS.
+
+   SaunaFS is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, version 3.
+
+   SaunaFS is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with SaunaFS. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "common/platform.h"
+
+#include <yaml-cpp/yaml.h>
+#include <iostream>
+
+#include "admin/list_chunkservers_command.h"
+#include "admin/list_disk_groups_command.h"
+#include "common/saunafs_version.h"
+#include "common/serialization.h"
+#include "common/server_connection.h"
+
+std::string ListDiskGroupsCommand::name() const { return "list-disk-groups"; }
+
+void ListDiskGroupsCommand::usage() const {
+	std::cerr << name() << " <master ip> <master port>\n";
+	std::cerr << "    Prints disk groups configuration in chunkservers.\n";
+}
+
+void ListDiskGroupsCommand::run(const Options &options) const {
+	if (options.arguments().size() != 2) {
+		throw WrongUsageException(
+		    "Expected <master ip> and <master port> for " + name());
+	}
+
+	auto chunkservers = ListChunkserversCommand::getChunkserversList(
+	    options.argument(0), options.argument(1));
+
+	YAML::Emitter yaml;
+	yaml << YAML::BeginMap;  // start root map
+
+	yaml << YAML::Key << "chunkservers";
+	yaml << YAML::Value << YAML::BeginSeq;  // start chunkservers list
+
+	for (const auto &chunkserver : chunkservers) {
+		if (chunkserver.version == kDisconnectedChunkserverVersion) {
+			continue;  // skip disconnected chunkservers
+		}
+
+		std::vector<uint8_t> request;
+		std::vector<uint8_t> response;
+		serializeLegacyPacket(request, CLTOCS_ADMIN_LIST_DISK_GROUPS);
+
+		auto csAddress =
+		    NetworkAddress(chunkserver.servip, chunkserver.servport);
+		ServerConnection connection(csAddress);
+		response = connection.sendAndReceive(request,
+		                                     CSTOCL_ADMIN_LIST_DISK_GROUPS);
+
+		std::string info;
+
+		try {
+			deserializeAllLegacyPacketDataNoHeader(response, info);
+		} catch (const IncorrectDeserializationException &e) {
+			std::cerr << e.what() << "\n";
+			continue;
+		}
+
+		yaml << YAML::BeginMap;  // start chunkserver map
+		yaml << YAML::Key << "chunkserver";
+		yaml << YAML::Value << csAddress.toString();
+
+		static constexpr const char *kDiskGroupsKey = "disk_groups";
+		yaml << YAML::Key << kDiskGroupsKey;
+
+		YAML::Node diskGroups = YAML::Load(info);
+
+		if (diskGroups.IsMap() && diskGroups[kDiskGroupsKey]) {
+			yaml << YAML::Value << diskGroups[kDiskGroupsKey];
+		} else {
+			yaml << YAML::Value << info;
+		}
+
+		yaml << YAML::EndMap;  // end chunkserver map
+	}
+
+	yaml << YAML::EndSeq;  // end chunkservers list
+	yaml << YAML::EndMap;  // end root map
+
+	std::cout << yaml.c_str() << "\n";
+}
diff --git a/src/admin/list_disk_groups_command.h b/src/admin/list_disk_groups_command.h
new file mode 100644
index 000000000..12e35c60c
--- /dev/null
+++ b/src/admin/list_disk_groups_command.h
@@ -0,0 +1,30 @@
+/*
+   Copyright 2023 Leil Storage OÜ
+
+   This file is part of SaunaFS.
+
+   SaunaFS is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, version 3.
+
+   SaunaFS is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with SaunaFS. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/platform.h"
+
+#include "admin/saunafs_admin_command.h"
+
+class ListDiskGroupsCommand : public SaunaFsAdminCommand {
+public:
+	std::string name() const override;
+	void usage() const override;
+	void run(const Options& options) const override;
+};
diff --git a/src/admin/main.cc b/src/admin/main.cc
index 3d9c7cf98..f8057f2ef 100644
--- a/src/admin/main.cc
+++ b/src/admin/main.cc
@@ -29,6 +29,7 @@
 #include "admin/list_chunkservers_command.h"
 #include "admin/list_defective_files_command.h"
 #include "admin/list_disks_command.h"
+#include "admin/list_disk_groups_command.h"
 #include "admin/list_goals_command.h"
 #include "admin/list_metadataservers_command.h"
 #include "admin/list_mounts_command.h"
@@ -58,6 +59,7 @@ int main(int argc, const char** argv) {
 			new ListChunkserversCommand(),
 			new ListDefectiveFilesCommand(),
 			new ListDisksCommand(),
+			new ListDiskGroupsCommand(),
 			new ListGoalsCommand(),
 			new ListMountsCommand(),
 			new ListMetadataserversCommand(),
diff --git a/src/chunkserver/chunkserver-common/default_disk_manager.h b/src/chunkserver/chunkserver-common/default_disk_manager.h
index 17d00e755..41d8c4c3f 100644
--- a/src/chunkserver/chunkserver-common/default_disk_manager.h
+++ b/src/chunkserver/chunkserver-common/default_disk_manager.h
@@ -69,4 +69,8 @@ class DefaultDiskManager : public IDiskManager {
 	/// Update the space usage of the disks.
 	/// No need to update the space usage here for this implementation.
 	void updateSpaceUsage() override {};
+
+	/// Gets the disk groups information.
+	/// Not supported by the default disk manager.
+	std::string getDiskGroupsInfo() override { return "Not supported"; }
 };
diff --git a/src/chunkserver/chunkserver-common/disk_manager_interface.h b/src/chunkserver/chunkserver-common/disk_manager_interface.h
index 9bfec908c..88bd3dfb4 100644
--- a/src/chunkserver/chunkserver-common/disk_manager_interface.h
+++ b/src/chunkserver/chunkserver-common/disk_manager_interface.h
@@ -57,4 +57,7 @@ class IDiskManager {
 
 	/// Update the space usage of the disks.
 	virtual void updateSpaceUsage() = 0;
+
+	/// Gets the disk groups information in YAML format
+	virtual std::string getDiskGroupsInfo() = 0;
 };
diff --git a/src/chunkserver/hddspacemgr.cc b/src/chunkserver/hddspacemgr.cc
index 6c525b0d7..80762e70c 100644
--- a/src/chunkserver/hddspacemgr.cc
+++ b/src/chunkserver/hddspacemgr.cc
@@ -189,6 +189,12 @@ void hddSerializeAllDiskInfosV2(uint8_t *buff) {
 	gDisksMutex.unlock();  //Locked by hddGetSerializedSizeOfAllDiskInfosV2
 }
 
+std::string hddGetDiskGroups() {
+	TRACETHIS();
+
+	return gDiskManager->getDiskGroupsInfo();
+}
+
 void hddDiskInfoRotateStats() {
 	TRACETHIS();
 
diff --git a/src/chunkserver/hddspacemgr.h b/src/chunkserver/hddspacemgr.h
index 4ead75a76..3d7c71cf3 100644
--- a/src/chunkserver/hddspacemgr.h
+++ b/src/chunkserver/hddspacemgr.h
@@ -41,6 +41,8 @@ void hddGetNewChunks(std::vector<ChunkWithVersionAndType>& chunks,
 uint32_t hddGetSerializedSizeOfAllDiskInfosV2();
 void hddSerializeAllDiskInfosV2(uint8_t *buff);
 
+std::string hddGetDiskGroups();
+
 const std::size_t kChunkBulkSize = 1000;
 using BulkFunction = std::function<void(std::vector<ChunkWithVersionAndType>&)>;
 /// Executes the callback for each bulk of at most \p kChunkBulkSize chunks.
diff --git a/src/chunkserver/network_worker_thread.cc b/src/chunkserver/network_worker_thread.cc
index 410c9d2db..940ff8883 100644
--- a/src/chunkserver/network_worker_thread.cc
+++ b/src/chunkserver/network_worker_thread.cc
@@ -887,6 +887,22 @@ void worker_hdd_list_v2(csserventry *eptr, const uint8_t *data,
 	hddSerializeAllDiskInfosV2(ptr); // unlock
 }
 
+void worker_list_disk_groups(csserventry *eptr,
+                             [[maybe_unused]] const uint8_t *data,
+                             [[maybe_unused]] uint32_t length) {
+	TRACETHIS();
+
+	std::string diskGroups = hddGetDiskGroups();
+
+	// 4 bytes for the size of the string + 1 byte for the null character
+	static constexpr uint8_t kSerializedSizePlusNullChar = 5;
+
+	uint8_t *ptr = worker_create_attached_packet(
+	    eptr, CSTOCL_ADMIN_LIST_DISK_GROUPS,
+	    diskGroups.size() + kSerializedSizePlusNullChar);
+	serialize(&ptr, diskGroups);
+}
+
 void worker_chart(csserventry *eptr, const uint8_t *data, uint32_t length) {
 	TRACETHIS();
 	uint32_t chartid;
@@ -1022,6 +1038,9 @@ void worker_gotpacket(csserventry *eptr, uint32_t type, const uint8_t *data, uin
 		case CLTOCS_HDD_LIST_V2:
 			worker_hdd_list_v2(eptr, data, length);
 			break;
+		case CLTOCS_ADMIN_LIST_DISK_GROUPS:
+			worker_list_disk_groups(eptr, data, length);
+			break;
 		case CLTOAN_CHART:
 			worker_chart(eptr, data, length);
 			break;
diff --git a/src/protocol/SFSCommunication.h b/src/protocol/SFSCommunication.h
index 68c16d78a..f4943303c 100644
--- a/src/protocol/SFSCommunication.h
+++ b/src/protocol/SFSCommunication.h
@@ -1751,3 +1751,11 @@ enum class SugidClearMode {
 // 0x0259
 #define CSTOCL_HDD_LIST_V2 (PROTO_BASE+601)
 // N*[ entrysize:16 path:NAME flags:8 errchunkid:64 errtime:32 used:64 total:64 chunkscount:32 bytesread:64 usecread:64 usecreadmax:64 byteswriten:64 usecwrite:64 usecwritemax:64]
+
+// 0x025D
+#define CLTOCS_ADMIN_LIST_DISK_GROUPS (PROTO_BASE + 602)
+/// -
+
+// 0x025E
+#define CSTOCL_ADMIN_LIST_DISK_GROUPS (PROTO_BASE + 603)
+/// config:STDSTRING

From 62e4e99d6764b5a7e8c4a2500917aaf489a90a7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Manuel=20Garc=C3=ADa=20Aguilera?= <dave@leil.io>
Date: Fri, 16 Aug 2024 12:21:11 -0700
Subject: [PATCH 16/27] fix(mount): Fix create time (Windows) (#167)

The create time in Windows is got from the ```st_birthtime``` field,
therefore that field must be filled from the ```ctime``` field and
passed to the next layers.
---
 src/common/stat32.h       |  1 +
 src/mount/sauna_client.cc | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/common/stat32.h b/src/common/stat32.h
index 40a5db67d..06541fc86 100644
--- a/src/common/stat32.h
+++ b/src/common/stat32.h
@@ -48,5 +48,6 @@ struct stat32 {
     uint32_t st_atime;
     uint32_t st_mtime;
     uint32_t st_ctime;
+    uint32_t st_birthtime;
 };
 #endif
diff --git a/src/mount/sauna_client.cc b/src/mount/sauna_client.cc
index 5ddfc50c4..b35e095fc 100644
--- a/src/mount/sauna_client.cc
+++ b/src/mount/sauna_client.cc
@@ -82,6 +82,10 @@
 
 namespace SaunaClient {
 
+#ifdef _WIN32
+#define SAUNAFS_HAVE_STRUCT_STAT_ST_BIRTHTIME
+#endif
+
 #define MAX_FILE_SIZE (int64_t)(SFS_MAX_FILE_SIZE)
 
 #define PKGVERSION \
@@ -562,9 +566,15 @@ void attr_to_stat(uint32_t inode, const Attributes &attr, struct stat *stbuf) {
 	stbuf->st_gid = attrgid;
 	stbuf->st_atime = attratime;
 	stbuf->st_mtime = attrmtime;
+#ifdef _WIN32
+	// WinFSP understands this field as "Change Time"
+	stbuf->st_ctime = attrmtime;
+#else
 	stbuf->st_ctime = attrctime;
+#endif
 #ifdef SAUNAFS_HAVE_STRUCT_STAT_ST_BIRTHTIME
-	stbuf->st_birthtime = attrctime;        // for future use
+	// Used for Windows create time
+	stbuf->st_birthtime = attrctime;
 #endif
 	stbuf->st_nlink = attrnlink;
 }

From 161f889c10e438b2762610747755e790ca22767b Mon Sep 17 00:00:00 2001
From: Urmas Rist <urmas@leil.io>
Date: Mon, 12 Aug 2024 11:59:35 +0300
Subject: [PATCH 17/27] fix(uraft): Fix missing floating ip address

If something removes the floating ip address from the device,
saunafs-uraft will not bring it back. For example, on Ubuntu systems, if
systemd-networkd is restarted, it will reapply netplan config, which
inadvertently remove the floating ip address, and uraft has no way of
adding this back.

This fix adds a way for uraft to bring the IP address back. First by
exporting the IP address as an environment variable in the main program,
and then by using in the script to query iproute2 for the IP address,
and add it when it's missing.

Note that if two IP addresses with different subnet masks are added,
then it won't re-add the IP address with the correct subnet mask.

Co-authored-by: Crash <crash@leli.io>
Co-authored-by: Walter Gomez <baldor@leil.io>
---
 src/uraft/main.cc                 | 11 ++++++++++-
 src/uraft/saunafs-uraft-helper.in |  5 +++++
 src/uraft/uraftcontroller.h       |  1 +
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/uraft/main.cc b/src/uraft/main.cc
index 68ae94973..d46a4c89c 100644
--- a/src/uraft/main.cc
+++ b/src/uraft/main.cc
@@ -48,7 +48,8 @@ void parseOptions(int argc, char **argv, uRaftController::Options &opt, bool &ma
 	("URAFT_DEMOTE_TIMEOUT", po::value<int>()->default_value(1000000000), "demote timeout (ms)")
 	("URAFT_DEAD_HANDLER_TIMEOUT", po::value<int>()->default_value(1000000000), "metadata server dead handler timeout (ms)")
 	("URAFT_CHECK_CMD_PERIOD", po::value<int>()->default_value(100), "check command status period(ms)")
-	("URAFT_STATUS_PORT", po::value<int>()->default_value(9428), "node status port");
+	("URAFT_STATUS_PORT", po::value<int>()->default_value(9428), "node status port")
+	("URAFT_FLOATING_IP", po::value<std::string>(), "uraft floating ip address");
 
 	po::options_description cmdline_options;
 	cmdline_options.add(generic).add(config).add(hidden);
@@ -110,6 +111,9 @@ void parseOptions(int argc, char **argv, uRaftController::Options &opt, bool &ma
 	opt.local_master_server       = vm["LOCAL_MASTER_ADDRESS"].as<std::string>();
 	opt.local_master_port         = vm["LOCAL_MASTER_MATOCL_PORT"].as<int>();
 	opt.check_cmd_status_period   = vm["URAFT_CHECK_CMD_PERIOD"].as<int>();
+	if (opt.elector_mode == 0) {
+		opt.floating_ip           = vm["URAFT_FLOATING_IP"].as<std::string>();
+	}
 	make_daemon                   = vm["start-daemon"].as<bool>();
 
 	if (vm.count("id")) {
@@ -220,6 +224,11 @@ int main(int argc, char **argv) {
 		return EXIT_FAILURE;
 	}
 
+	if (!opt.floating_ip.empty()) {
+		syslog(LOG_INFO, "Setting URAFT_FLOATING_IP to %s", opt.floating_ip.c_str());
+		setenv("URAFT_FLOATING_IP", opt.floating_ip.c_str(), 0);
+	}
+
 	boost::asio::io_service  io_service;
 	uRaftController          server(io_service);
 #if (BOOST_VERSION >= 104700)
diff --git a/src/uraft/saunafs-uraft-helper.in b/src/uraft/saunafs-uraft-helper.in
index a8b057e0a..57585d0bf 100755
--- a/src/uraft/saunafs-uraft-helper.in
+++ b/src/uraft/saunafs-uraft-helper.in
@@ -155,6 +155,11 @@ saunafs_isalive() {
 	saunafs_master isalive
 	if [[ $? == 0 ]] ; then
 		echo -n alive
+		if ! ip addr | grep -w -q "${URAFT_FLOATING_IP}"; then
+			log "floating ip address ${URAFT_FLOATING_IP} was lost, adding it again..."
+			load_config
+			saunafs_assign_ip
+		fi
 	else
 		echo -n dead
 	fi
diff --git a/src/uraft/uraftcontroller.h b/src/uraft/uraftcontroller.h
index 9794d05ab..fa633b172 100644
--- a/src/uraft/uraftcontroller.h
+++ b/src/uraft/uraftcontroller.h
@@ -30,6 +30,7 @@ class uRaftController : public uRaftStatus {
 		int         promote_timeout;          //!< Time after which we kill promote script. //
 		int         demote_timeout;           //!< Time after which we kill demote script. //
 		int         dead_handler_timeout;     //!< Time after which we kill dead script. //
+		std::string floating_ip;        //!< Floating IP addresss used by uraft. //
 	};
 
 public:

From d1dc1d5321486dca161761349583cb4e925a5b99 Mon Sep 17 00:00:00 2001
From: Walter Gomez <baldor@leil.io>
Date: Wed, 21 Aug 2024 17:31:42 +0200
Subject: [PATCH 18/27] ci: Fix tests behavior (#172)

- Make the environment variables available for tests.
- Elevate privileges for cleanup.
---
 tests/ci_build/run-ganesha-tests.sh | 4 ++--
 tests/ci_build/run-sanity-check.sh  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/ci_build/run-ganesha-tests.sh b/tests/ci_build/run-ganesha-tests.sh
index e3cdb2409..01befaf13 100755
--- a/tests/ci_build/run-ganesha-tests.sh
+++ b/tests/ci_build/run-ganesha-tests.sh
@@ -17,7 +17,7 @@ export TERM=xterm
 killall -9 saunafs-tests || true
 mkdir -m 777 -p "${TEST_OUTPUT_DIR}"
 rm -rf "${TEST_OUTPUT_DIR:?}"/* || true
-rm -rf /mnt/ramdisk/* || true
+sudo rm -rf /mnt/ramdisk/* || true
 [ -f "${SAUNAFS_ROOT}/bin/saunafs-tests" ] || \
 	die "${SAUNAFS_ROOT}/bin/saunafs-tests" not found, did you build the project?
 export PATH="${SAUNAFS_ROOT}/bin:${PATH}"
@@ -27,4 +27,4 @@ echo ": \${SAUNAFS_ROOT:=${SAUNAFS_ROOT}}" | sudo tee -a /etc/saunafs_tests.conf
 sudo ln -sf ${SAUNAFS_ROOT}/lib/ganesha/libfsalsaunafs.so /usr/lib/ganesha/libfsalsaunafs.so
 sudo mkdir -p /usr/lib/x86_64-linux-gnu/ganesha
 sudo ln -sf ${SAUNAFS_ROOT}/lib/ganesha/libfsalsaunafs.so /usr/lib/x86_64-linux-gnu/ganesha/libfsalsaunafs.so
-sudo "${SAUNAFS_ROOT}/bin/saunafs-tests" --gtest_color=yes --gtest_filter="${test_filter}" --gtest_output=xml:"${TEST_OUTPUT_DIR}/ganesha_test_results.xml"
+sudo --preserve-env "${SAUNAFS_ROOT}/bin/saunafs-tests" --gtest_color=yes --gtest_filter="${test_filter}" --gtest_output=xml:"${TEST_OUTPUT_DIR}/ganesha_test_results.xml"
diff --git a/tests/ci_build/run-sanity-check.sh b/tests/ci_build/run-sanity-check.sh
index e2557632f..6dac5ba5c 100755
--- a/tests/ci_build/run-sanity-check.sh
+++ b/tests/ci_build/run-sanity-check.sh
@@ -27,10 +27,10 @@ export TERM=xterm
 killall -9 saunafs-tests || true
 mkdir -m 777 -p "${TEST_OUTPUT_DIR}"
 rm -rf "${TEST_OUTPUT_DIR:?}"/* || true
-rm -rf /mnt/ramdisk/* || true
+sudo rm -rf /mnt/ramdisk/* || true
 [ -f "${SAUNAFS_ROOT}/bin/saunafs-tests" ] || \
 	die "${SAUNAFS_ROOT}/bin/saunafs-tests" not found, did you build the project?
 export PATH="${SAUNAFS_ROOT}/bin:${PATH}"
 sudo sed -E -i '\,.*:\s+\$\{SAUNAFS_ROOT\s*:=.*,d' /etc/saunafs_tests.conf || true
 echo ": \${SAUNAFS_ROOT:=${SAUNAFS_ROOT}}" | sudo tee -a /etc/saunafs_tests.conf >/dev/null || true
-sudo "${SAUNAFS_ROOT}/bin/saunafs-tests" --gtest_color=yes --gtest_output=xml:"${TEST_OUTPUT_DIR}/sanity_test_results.xml" "${test_extra_args[@]}"
+sudo --preserve-env "${SAUNAFS_ROOT}/bin/saunafs-tests" --gtest_color=yes --gtest_output=xml:"${TEST_OUTPUT_DIR}/sanity_test_results.xml" "${test_extra_args[@]}"

From 4879a635b109a303213e750f5500eba7e1b65998 Mon Sep 17 00:00:00 2001
From: Antuan <antuan@leil.io>
Date: Fri, 23 Aug 2024 13:00:45 +0200
Subject: [PATCH 19/27] fix(ci): Modify timeout from env (#165)

- Preserve environment variables from CI on testing framework.
- Avoid duplicate build on tests pipelines.

Co-authored-by: Baldor <baldor@leil.io>
---
 tests/ci_build/run-build.sh         | 11 +++++------
 tests/ci_build/run-ganesha-tests.sh |  4 +++-
 tests/ci_build/run-sanity-check.sh  |  6 ++++--
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/tests/ci_build/run-build.sh b/tests/ci_build/run-build.sh
index 0fe0e5a2b..aae616ee4 100755
--- a/tests/ci_build/run-build.sh
+++ b/tests/ci_build/run-build.sh
@@ -32,7 +32,7 @@ declare -a CMAKE_SAUNAFS_ARGUMENTS=(
 [ -n "${1:-}" ] || usage
 declare build_type="${1}"
 declare build_dir
-declare do_install=true
+declare -a make_extra_args=()
 shift
 case "${build_type,,}" in
 	debug)
@@ -45,6 +45,7 @@ case "${build_type,,}" in
 			-DENABLE_WERROR=ON
 		)
 		build_dir="${WORKSPACE}/build/saunafs-debug"
+		make_extra_args+=( 'install' )
 		;;
 	coverage)
 		CMAKE_SAUNAFS_ARGUMENTS+=(
@@ -56,6 +57,7 @@ case "${build_type,,}" in
 			-DENABLE_WERROR=OFF
 		)
 		build_dir="${WORKSPACE}/build/saunafs-coverage"
+		make_extra_args+=( 'install' )
 		;;
 	test)
 		CMAKE_SAUNAFS_ARGUMENTS+=(
@@ -67,6 +69,7 @@ case "${build_type,,}" in
 			-DENABLE_WERROR=ON
 		)
 		build_dir="${WORKSPACE}/build/saunafs"
+		make_extra_args+=( 'install' )
 		;;
 	release)
 		CMAKE_SAUNAFS_ARGUMENTS+=(
@@ -78,7 +81,6 @@ case "${build_type,,}" in
 			-DENABLE_WERROR=OFF
 		)
 		build_dir="${WORKSPACE}/build/saunafs-release"
-		do_install=false
 		;;
 	*) die "Unsupported build type: ${build_type}"
 		;;
@@ -95,10 +97,7 @@ cmake -B "${build_dir}" \
 	"${CMAKE_SAUNAFS_ARGUMENTS[@]}" \
 	"${EXTRA_ARGUMENTS[@]}" "${WORKSPACE}"
 
-nice make -C "${build_dir}" -j "$(nproc)"
-if [ "${do_install}" = true ]; then
-	nice make -C "${build_dir}" install
-fi
+nice make -C "${build_dir}" -j "$(nproc)" "${make_extra_args[@]}"
 
 if [ -f "${build_dir}/CPackConfig.cmake" ]; then
 	nice cpack -B "${build_dir}" --config "${build_dir}/CPackConfig.cmake" -j "$(nproc)"
diff --git a/tests/ci_build/run-ganesha-tests.sh b/tests/ci_build/run-ganesha-tests.sh
index 01befaf13..30a28a69e 100755
--- a/tests/ci_build/run-ganesha-tests.sh
+++ b/tests/ci_build/run-ganesha-tests.sh
@@ -27,4 +27,6 @@ echo ": \${SAUNAFS_ROOT:=${SAUNAFS_ROOT}}" | sudo tee -a /etc/saunafs_tests.conf
 sudo ln -sf ${SAUNAFS_ROOT}/lib/ganesha/libfsalsaunafs.so /usr/lib/ganesha/libfsalsaunafs.so
 sudo mkdir -p /usr/lib/x86_64-linux-gnu/ganesha
 sudo ln -sf ${SAUNAFS_ROOT}/lib/ganesha/libfsalsaunafs.so /usr/lib/x86_64-linux-gnu/ganesha/libfsalsaunafs.so
-sudo --preserve-env "${SAUNAFS_ROOT}/bin/saunafs-tests" --gtest_color=yes --gtest_filter="${test_filter}" --gtest_output=xml:"${TEST_OUTPUT_DIR}/ganesha_test_results.xml"
+sudo --preserve-env=SAUNAFS_TEST_TIMEOUT_MULTIPLIER \
+"${SAUNAFS_ROOT}/bin/saunafs-tests" --gtest_color=yes \
+--gtest_filter="${test_filter}" --gtest_output=xml:"${TEST_OUTPUT_DIR}/ganesha_test_results.xml"
diff --git a/tests/ci_build/run-sanity-check.sh b/tests/ci_build/run-sanity-check.sh
index 6dac5ba5c..10d7be6eb 100755
--- a/tests/ci_build/run-sanity-check.sh
+++ b/tests/ci_build/run-sanity-check.sh
@@ -3,7 +3,7 @@ set -eux -o pipefail
 PROJECT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/../..")"
 WORKSPACE=${WORKSPACE:-"${PROJECT_DIR}"}
 die() { echo "Error: $*" >&2; exit 1; }
-test_extra_args=()
+declare -a test_extra_args=()
 if [ -n "${1:-}" ]; then
 	test_extra_args+=("--gtest_filter=${1}")
 	shift 1
@@ -33,4 +33,6 @@ sudo rm -rf /mnt/ramdisk/* || true
 export PATH="${SAUNAFS_ROOT}/bin:${PATH}"
 sudo sed -E -i '\,.*:\s+\$\{SAUNAFS_ROOT\s*:=.*,d' /etc/saunafs_tests.conf || true
 echo ": \${SAUNAFS_ROOT:=${SAUNAFS_ROOT}}" | sudo tee -a /etc/saunafs_tests.conf >/dev/null || true
-sudo --preserve-env "${SAUNAFS_ROOT}/bin/saunafs-tests" --gtest_color=yes --gtest_output=xml:"${TEST_OUTPUT_DIR}/sanity_test_results.xml" "${test_extra_args[@]}"
+sudo --preserve-env=SAUNAFS_TEST_TIMEOUT_MULTIPLIER \
+"${SAUNAFS_ROOT}/bin/saunafs-tests" --gtest_color=yes \
+--gtest_output=xml:"${TEST_OUTPUT_DIR}/sanity_test_results.xml" "${test_extra_args[@]}"

From e7027b214f2a5decbac7406fc7c6153945c96c4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rub=C3=A9n=20Alcolea=20N=C3=BA=C3=B1ez?= <crash@leil.io>
Date: Sat, 24 Aug 2024 14:01:24 +0200
Subject: [PATCH 20/27] test: Improve `stop_ganesha()` function (#168)

The original `stop_ganesha()` function unmounted mountpoints and
attempted to kill the Ganesha daemon only once. If the `pkill`
operation failed, the Ganesha daemon could remain hanging, leading
to Ganesha CI containers becoming "unhealthy" and requiring Jenkins
nodes to be restarted.

This commit enhances the `stop_ganesha()` function to handle failed
`pkill` attempts. If the initial `pkill` fails, a new Ganesha daemon
is started, which should terminate the hanging one. This change aims
to ensure proper cleanup of resources and prevent Ganesha CI containers
from entering an "unhealthy" state, thus avoiding unnecessary Jenkins
nodes restarts.
---
 tests/run-test.sh | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/tests/run-test.sh b/tests/run-test.sh
index 7c16a7195..acc6db76e 100755
--- a/tests/run-test.sh
+++ b/tests/run-test.sh
@@ -56,11 +56,24 @@ unwrap_generators() {
 stop_ganesha() {
 	# umount all hanging SaunaFS mounts
 	for mount in $(mount | grep /tmp/SaunaFS | awk '{print $3}'); do
-		sudo umount -l $mount
+		sudo umount -l $mount || echo "Failed to umount $mount"
 	done
 
-	# kill ganesha daemon
-	sudo pkill -9 ganesha.nfsd
+	# Check if ganesha daemon is running and kill it
+	if pgrep ganesha.nfsd > /dev/null; then
+		sudo pkill -9 ganesha.nfsd
+		sleep 0.5
+	fi
+
+	# Check if ganesha daemon is still running
+	if pgrep ganesha.nfsd > /dev/null; then
+		# Start a new ganesha daemon to remove the hanging one
+		sudo /usr/bin/ganesha.nfsd -L /var/log/ganesha.log
+		sleep 2
+
+		# Kill the newly started ganesha daemon to avoid conflicts with tests
+		sudo pkill -9 ganesha.nfsd
+	fi
 }
 
 stop_tests() {

From 2f6002190fe68adb7a55cd6e069ec5880f430299 Mon Sep 17 00:00:00 2001
From: Luis Guillermo Silva Rojas <guillex@leil.io>
Date: Sun, 25 Aug 2024 13:46:57 +0200
Subject: [PATCH 21/27] feat(chunkserver): Add IDiskManager::getDiskForGC
 (#169)

This commit adds and extension point for the disk managers to be asked
about the next disk for garbage collection (GC). The default disk
manager returns the next zoned device in a round-robin approach.

It is up to the disk manager plugins to provide custom implementation
for this behavior.
---
 .../default_disk_manager.cc                   | 28 +++++++++++++++++++
 .../chunkserver-common/default_disk_manager.h |  7 +++++
 .../disk_manager_interface.h                  |  4 +++
 src/common/unique_queue.h                     |  6 ++++
 4 files changed, 45 insertions(+)

diff --git a/src/chunkserver/chunkserver-common/default_disk_manager.cc b/src/chunkserver/chunkserver-common/default_disk_manager.cc
index 08ac724e0..14d62a410 100644
--- a/src/chunkserver/chunkserver-common/default_disk_manager.cc
+++ b/src/chunkserver/chunkserver-common/default_disk_manager.cc
@@ -304,3 +304,31 @@ IDisk *DefaultDiskManager::getDiskForNewChunk(
 
 	return bestDisk;
 }
+
+IDisk *DefaultDiskManager::getDiskForGC() {
+	TRACETHIS();
+	IDisk *bestDisk = DiskNotFound;
+
+	std::lock_guard disksLockGuard(gDisksMutex);
+
+	if (gDisks.empty()) {
+		return DiskNotFound;
+	}
+
+	auto diskCount = gDisks.size();
+
+	for (size_t i = 0; i < diskCount; ++i) {
+		size_t index = (nextDiskIndexForGC_ + i) % diskCount;
+		const auto &disk = gDisks[index];
+
+		if (!disk->isZonedDevice() || !disk->isSelectableForNewChunk()) {
+			continue;
+		}
+
+		bestDisk = disk.get();
+		nextDiskIndexForGC_ = (index + 1) % diskCount;
+		break;
+	}
+
+	return bestDisk;
+}
diff --git a/src/chunkserver/chunkserver-common/default_disk_manager.h b/src/chunkserver/chunkserver-common/default_disk_manager.h
index 41d8c4c3f..15021bc4f 100644
--- a/src/chunkserver/chunkserver-common/default_disk_manager.h
+++ b/src/chunkserver/chunkserver-common/default_disk_manager.h
@@ -73,4 +73,11 @@ class DefaultDiskManager : public IDiskManager {
 	/// Gets the disk groups information.
 	/// Not supported by the default disk manager.
 	std::string getDiskGroupsInfo() override { return "Not supported"; }
+
+	/// Selects the disk to use for GC.
+	IDisk *getDiskForGC() override;
+
+private:
+	/// Next disk index for GC. Helps in the round-robin strategy.
+	uint32_t nextDiskIndexForGC_ = 0;
 };
diff --git a/src/chunkserver/chunkserver-common/disk_manager_interface.h b/src/chunkserver/chunkserver-common/disk_manager_interface.h
index 88bd3dfb4..c7dd8e733 100644
--- a/src/chunkserver/chunkserver-common/disk_manager_interface.h
+++ b/src/chunkserver/chunkserver-common/disk_manager_interface.h
@@ -60,4 +60,8 @@ class IDiskManager {
 
 	/// Gets the disk groups information in YAML format
 	virtual std::string getDiskGroupsInfo() = 0;
+
+	/// Selects the disk to use for Garbage Collection (GC).
+	/// Could return DiskNotFound if the disks does not need GC.
+	virtual IDisk *getDiskForGC() = 0;
 };
diff --git a/src/common/unique_queue.h b/src/common/unique_queue.h
index 74ddccda9..1828db1b1 100644
--- a/src/common/unique_queue.h
+++ b/src/common/unique_queue.h
@@ -56,6 +56,12 @@ class UniqueQueue {
 			return element;
 		}
 	}
+
+	size_t size() {
+		std::lock_guard lock(mutex_);
+		return queue_.size();
+	}
+
 private:
 	typedef std::set<T> Set;
 	typedef std::queue<typename Set::iterator> Queue;

From c8423410eec4c628045d0c8a5a2e4bfd60975951 Mon Sep 17 00:00:00 2001
From: Luis Guillermo Silva Rojas <guillex@leil.io>
Date: Mon, 26 Aug 2024 20:45:04 +0200
Subject: [PATCH 22/27] test: Move long build tests to LongFailingTests (#175)

The tests related to build SaunaFS inside a mount point are failing in
the CI in a non-deterministic way, but they are passing in the
development environments.

This change moves those tests to the LongFailingTests suite, to be
investigated separately.
---
 .../{LongSystemTests => LongFailingTests}/test_build_and_rsync.sh | 0
 .../{LongSystemTests => LongFailingTests}/test_build_saunafs.sh   | 0
 .../test_build_while_auto_recovery.sh                             | 0
 .../test_build_while_changing_master.sh                           | 0
 4 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/test_suites/{LongSystemTests => LongFailingTests}/test_build_and_rsync.sh (100%)
 rename tests/test_suites/{LongSystemTests => LongFailingTests}/test_build_saunafs.sh (100%)
 rename tests/test_suites/{LongSystemTests => LongFailingTests}/test_build_while_auto_recovery.sh (100%)
 rename tests/test_suites/{LongSystemTests => LongFailingTests}/test_build_while_changing_master.sh (100%)

diff --git a/tests/test_suites/LongSystemTests/test_build_and_rsync.sh b/tests/test_suites/LongFailingTests/test_build_and_rsync.sh
similarity index 100%
rename from tests/test_suites/LongSystemTests/test_build_and_rsync.sh
rename to tests/test_suites/LongFailingTests/test_build_and_rsync.sh
diff --git a/tests/test_suites/LongSystemTests/test_build_saunafs.sh b/tests/test_suites/LongFailingTests/test_build_saunafs.sh
similarity index 100%
rename from tests/test_suites/LongSystemTests/test_build_saunafs.sh
rename to tests/test_suites/LongFailingTests/test_build_saunafs.sh
diff --git a/tests/test_suites/LongSystemTests/test_build_while_auto_recovery.sh b/tests/test_suites/LongFailingTests/test_build_while_auto_recovery.sh
similarity index 100%
rename from tests/test_suites/LongSystemTests/test_build_while_auto_recovery.sh
rename to tests/test_suites/LongFailingTests/test_build_while_auto_recovery.sh
diff --git a/tests/test_suites/LongSystemTests/test_build_while_changing_master.sh b/tests/test_suites/LongFailingTests/test_build_while_changing_master.sh
similarity index 100%
rename from tests/test_suites/LongSystemTests/test_build_while_changing_master.sh
rename to tests/test_suites/LongFailingTests/test_build_while_changing_master.sh

From 8491ad1235b459bc0fab7d92bf829d58c67d15c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rolando=20S=C3=A1nchez=20Ramos?=
 <65413231+rolysr@users.noreply.github.com>
Date: Tue, 27 Aug 2024 05:49:45 -0400
Subject: [PATCH 23/27] fix(mount): Fix random case option parsing (#135)

feat(mount): Random case option parsing

For allowing the client users to input option with random case formats,
this change allowing that on the client argument processing.

It also enables the parsing of the following formats of options
written in the cfg file (if provided, obviusly): <-D value>-like
format, this is a single character keyword like -P, -D, -B or -S and
<-o opt1=val1,opt2=val2,...> format.

Plus, it enables the random casing in the names of the options and
forbids mount if cfg is provided and there are some unexpected (like
--help, -V, -u options) or wrong options.

The proposed changes enforce cfg files to be well-formed, thus causing
the mount to fail in such cases. Some tests were fixed due to having
such errors.

Two integration tests were also added:
- a test that check the expected random casing parsing in cfg files.
- another test to ensure the behavior of random case options parsing
from command line work properly.

Co-authored-by: Dave <dave@leil.io>
---
 src/mount/fuse/main.cc                        | 15 ++++
 src/mount/fuse/mount_config.cc                | 16 +++-
 src/mount/option_casing_normalization.h       | 76 +++++++++++++++++++
 .../option_casing_normalization_unittest.cc   | 40 ++++++++++
 .../test_basic_performance_estimation.sh      |  2 +-
 .../test_options_random_casing.sh             | 45 +++++++++++
 ...test_options_random_casing_command_line.sh | 24 ++++++
 ...errooms_with_endangered_chunks_priority.sh |  3 +-
 .../test_read_write_during_scan.sh            |  2 +-
 tests/tools/saunafs.sh                        | 39 +++++++---
 10 files changed, 247 insertions(+), 15 deletions(-)
 create mode 100644 src/mount/option_casing_normalization.h
 create mode 100644 src/mount/option_casing_normalization_unittest.cc
 create mode 100644 tests/test_suites/SanityChecks/test_options_random_casing.sh
 create mode 100644 tests/test_suites/SanityChecks/test_options_random_casing_command_line.sh

diff --git a/src/mount/fuse/main.cc b/src/mount/fuse/main.cc
index 582914cff..0e6e7b14e 100644
--- a/src/mount/fuse/main.cc
+++ b/src/mount/fuse/main.cc
@@ -38,6 +38,7 @@
 #include "mount/g_io_limiters.h"
 #include "mount/mastercomm.h"
 #include "mount/masterproxy.h"
+#include "mount/option_casing_normalization.h"
 #include "mount/readdata.h"
 #include "mount/stats.h"
 #include "mount/symlinkcache.h"
@@ -506,6 +507,8 @@ int main(int argc, char *argv[]) try {
 	struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
 	struct fuse_args defaultargs = FUSE_ARGS_INIT(0, NULL);
 
+	normalize_options_casing(args.argc, args.argv);
+
 	fuse_opt_add_arg(&defaultargs, "fakeappname");
 
 	if (read_masterhost_if_present(&args))
@@ -517,9 +520,21 @@ int main(int argc, char *argv[]) try {
 	if (!gCustomCfg)
 		sfs_opt_parse_cfg_file(DEFAULT_SFSMOUNT_CONFIG_PATH, 1, &defaultargs);
 
+	normalize_options_casing(defaultargs.argc, defaultargs.argv);
+
 	if (fuse_opt_parse(&defaultargs, &gMountOptions, gSfsOptsStage2, sfs_opt_proc_stage2))
 		exit(1);
 
+	if (defaultargs.argc > 1) {
+		fprintf(stderr, "Unexpected/wrong option(s) at cfg file:\n");
+		for (int i = 1; i < defaultargs.argc; i++) {
+			if (strcmp(defaultargs.argv[i], "-o") != 0) {
+				fprintf(stderr, "%s\n", defaultargs.argv[i]);
+			}
+		}
+		exit(1);
+	}
+
 	if (fuse_opt_parse(&args, &gMountOptions, gSfsOptsStage2, sfs_opt_proc_stage2))
 		exit(1);
 
diff --git a/src/mount/fuse/mount_config.cc b/src/mount/fuse/mount_config.cc
index 8dd683662..e02fd900f 100644
--- a/src/mount/fuse/mount_config.cc
+++ b/src/mount/fuse/mount_config.cc
@@ -340,12 +340,14 @@ void sfs_opt_parse_cfg_file(const char *filename,int optional,struct fuse_args *
 	}
 	gCustomCfg = 1;
 	while (fgets(lbuff, N - 1, fd)) {
+		// Skip a comment line
 		if (lbuff[0] == '#' || lbuff[0] == ';')
 			continue;
 
 		lbuff[N - 1] = 0;
 		gCfgString += lbuff;
 
+		// Go to the end of the cfg file line
 		for (p = lbuff; *p; p++) {
 			if (*p == '\r' || *p == '\n') {
 				*p = 0;
@@ -355,6 +357,7 @@ void sfs_opt_parse_cfg_file(const char *filename,int optional,struct fuse_args *
 
 		p--;
 
+		// Skip last blank spaces in the line
 		while (p >= lbuff && (*p == ' ' || *p == '\t')) {
 			*p = 0;
 			p--;
@@ -362,18 +365,25 @@ void sfs_opt_parse_cfg_file(const char *filename,int optional,struct fuse_args *
 
 		p = lbuff;
 
+		// Skip first blank spaces in the line
 		while (*p == ' ' || *p == '\t') {
 			p++;
 		}
 
 		if (*p) {
 			if (*p == '-') {
-				fuse_opt_add_arg(outargs,p);
-			} else if (*p == '/') {
+				// "-o opt1=val1,opt2=val2,..." or "-D S"-like formats
+				if (p[1] != ' ' && p[1] != 0 && p[2] == ' ') {
+					fuse_opt_add_arg(outargs, std::string(p, p + 2).c_str());
+					fuse_opt_add_arg(outargs, p + 3);
+				} else { // "-oopt1=val1,opt2=val2,..." format
+					fuse_opt_add_arg(outargs, p);
+				}
+			} else if (*p == '/') { // default mountpoint
 				if (gDefaultMountpoint)
 					free(gDefaultMountpoint);
 				gDefaultMountpoint = strdup(p);
-			} else {
+			} else { // "opt1=val1,opt2=val2,..." format
 				fuse_opt_add_arg(outargs,"-o");
 				fuse_opt_add_arg(outargs,p);
 			}
diff --git a/src/mount/option_casing_normalization.h b/src/mount/option_casing_normalization.h
new file mode 100644
index 000000000..f73a09734
--- /dev/null
+++ b/src/mount/option_casing_normalization.h
@@ -0,0 +1,76 @@
+/*
+   Copyright 2023      Leil Storage OÜ
+
+   This file is part of SaunaFS.
+
+   SaunaFS is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, version 3.
+
+   SaunaFS is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with SaunaFS  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/platform.h"
+
+#include <cstring>
+#include <string>
+
+/**
+ * @brief Normalize a single argument casing by converting uppercase characters
+ * in the options names to its lowercase equivalent.
+ */
+void normalize_argument_casing(char *arg) {
+	// A single argument looks like this: arg1=val1,arg2=val2,...,argn=valn
+	// So, case change should be applied at the begining and between a comma
+	// appearance and the next equal sign
+
+	std::string argument(arg);
+	bool should_apply_case_change = true;
+
+	for (auto &c : argument) {
+		if (c == ',') {
+			should_apply_case_change = true;
+		} else if (c == '=') {
+			should_apply_case_change = false;
+		} else if (should_apply_case_change) {
+			if (std::isupper(c)) { c = std::tolower(c); }
+		}
+	}
+
+	// Copy the modified string back to the original C-string argument
+	std::copy(argument.begin(), argument.end(), arg);
+	
+	// Null-terminate the modified C-string
+	arg[argument.size()] = '\0';
+}
+
+/**
+ * @brief Normalize options casing by converting uppercase characters in the
+ * options names to its lowercase equivalent.
+ */
+void normalize_options_casing(int argc, char *argv[]) {
+	bool is_previous_dash_o = false;
+	for (int index = 1; index < argc; index++) {
+		size_t current_arg_len = strlen(argv[index]);
+		if (is_previous_dash_o ||
+		    (current_arg_len > 2 && argv[index][0] == '-' &&
+		     (argv[index][1] == 'o' || argv[index][1] == '-'))) {
+			if (is_previous_dash_o) {
+				normalize_argument_casing(argv[index]);
+			} else {
+				normalize_argument_casing(argv[index] + 2);
+			}
+			is_previous_dash_o = false;
+			continue;
+		}
+		is_previous_dash_o = (strcmp(argv[index], "-o") == 0);
+	}
+}
diff --git a/src/mount/option_casing_normalization_unittest.cc b/src/mount/option_casing_normalization_unittest.cc
new file mode 100644
index 000000000..ff4fe5938
--- /dev/null
+++ b/src/mount/option_casing_normalization_unittest.cc
@@ -0,0 +1,40 @@
+/*
+   Copyright 2023      Leil Storage OÜ
+
+   This file is part of SaunaFS.
+
+   SaunaFS is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, version 3.
+
+   SaunaFS is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with SaunaFS. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "common/platform.h"
+
+#include "mount/option_casing_normalization.h"
+
+#include <gtest/gtest.h>
+
+TEST(ArgumentCasingTest, LowercaseOptionNamesOnly) {
+	char input[] =
+	    "sfSWriteCACHESize=4096,sfscacHeperinodePERcentAGE=100,"
+	    "SFSCHunKSeRVERwavereadto=2000,sfschunkservertotalreadto=8000,"
+	    "MAXreadaheadrequests=1,cacheexpirationtime=10000,READWORKERS=42,"
+	    "sfswriteWORKERS=42";
+	char expected[] =
+	    "sfswritecachesize=4096,sfscacheperinodepercentage=100,"
+	    "sfschunkserverwavereadto=2000,sfschunkservertotalreadto=8000,"
+	    "maxreadaheadrequests=1,cacheexpirationtime=10000,readworkers=42,"
+	    "sfswriteworkers=42";
+
+	normalize_argument_casing(input);
+
+	EXPECT_EQ(strcmp(input, expected), 0);
+}
diff --git a/tests/test_suites/Benchmarks/test_basic_performance_estimation.sh b/tests/test_suites/Benchmarks/test_basic_performance_estimation.sh
index 5a22e88a0..a7b6930f0 100644
--- a/tests/test_suites/Benchmarks/test_basic_performance_estimation.sh
+++ b/tests/test_suites/Benchmarks/test_basic_performance_estimation.sh
@@ -3,7 +3,7 @@
 
 CHUNKSERVERS=1 \
 	USE_RAMDISK=YES \
-	MOUNT_EXTRA_CONFIG="mfscachemode=NEVER" \
+	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER" \
 	CHUNKSERVER_EXTRA_CONFIG="GARBAGE_COLLECTION_FREQ_MS = 0|HDD_TEST_FREQ = 100000" \
 	AUTO_SHADOW_MASTER="NO" \
 	setup_local_empty_saunafs info
diff --git a/tests/test_suites/SanityChecks/test_options_random_casing.sh b/tests/test_suites/SanityChecks/test_options_random_casing.sh
new file mode 100644
index 000000000..a3e128d69
--- /dev/null
+++ b/tests/test_suites/SanityChecks/test_options_random_casing.sh
@@ -0,0 +1,45 @@
+# The goal of this test is to make sure that client parsing is able to process option
+# names that use capital letters instead of the expected lowercase letters. At the end
+# it also checks that poorly written config files will make client to fail its mount
+# instead of ignoring them.
+
+CHUNKSERVERS=1 \
+	USE_RAMDISK=YES \
+	setup_local_empty_saunafs info
+
+cd "${info[mount0]}"
+
+# Expected values are current defaults
+assert_equals $(cat .saunafs_tweaks | egrep CacheExpirationTime | awk '{print $2}') 1000
+assert_equals $(cat .saunafs_tweaks | egrep WriteMaxRetries | awk '{print $2}') 30
+assert_equals $(cat .saunafs_tweaks | egrep MaxReadaheadRequests | awk '{print $2}') 5
+assert_equals $(cat .saunafs_tweaks | egrep ReadWaveTimeout | awk '{print $2}') 500
+
+cd /
+saunafs_mount_unmount 0
+
+# Must chose some values different than current defaults
+echo "-o CacheExpirationTime=2000" >> "${info[mount0_cfg]}"
+echo "sfsIORetries=20" >> "${info[mount0_cfg]}"
+echo "-oMaxReadaheadRequests=2,SFSChUnKSerVerWaVEREadtO=200" >> "${info[mount0_cfg]}"
+
+saunafs_mount_start 0
+cd "${info[mount0]}"
+
+assert_equals $(cat .saunafs_tweaks | egrep CacheExpirationTime | awk '{print $2}') 2000
+assert_equals $(cat .saunafs_tweaks | egrep WriteMaxRetries | awk '{print $2}') 20
+assert_equals $(cat .saunafs_tweaks | egrep MaxReadaheadRequests | awk '{print $2}') 2
+assert_equals $(cat .saunafs_tweaks | egrep ReadWaveTimeout | awk '{print $2}') 200
+
+cd /
+saunafs_mount_unmount 0
+
+# Try writing some error in a cfg file ...
+echo "-o foo=bar" >> "${info[mount0_cfg]}"
+
+assert_failure ${saunafs_info_[mntcall0]}
+
+# Let's substitute the wrong option for some other, which is not expected here (like --help) ...
+sed -i 's/-o foo=bar/--help/g' "${info[mount0_cfg]}"
+
+assert_failure ${saunafs_info_[mntcall0]}
diff --git a/tests/test_suites/SanityChecks/test_options_random_casing_command_line.sh b/tests/test_suites/SanityChecks/test_options_random_casing_command_line.sh
new file mode 100644
index 000000000..029b39f1d
--- /dev/null
+++ b/tests/test_suites/SanityChecks/test_options_random_casing_command_line.sh
@@ -0,0 +1,24 @@
+MOUNTS=2 \
+	USE_RAMDISK=YES \
+	FUSE_EXTRA_CONFIG="cACHEexpIRAtiONtIME=2000" \
+	FUSE_0_EXTRA_CONFIG="sfsIORetries=20`
+			`|MaxReadahEAdReqUests=2`
+			`|SFSChUnKSerVerWaVEREadtO=200" \
+	FUSE_1_EXTRA_CONFIG="sfsIORetries=60`
+			`|MaxReadahEAdReqUests=10`
+			`|SFSChUnKSerVerWaVEREadtO=300" \
+	setup_local_empty_saunafs info
+
+cd "${info[mount0]}"
+
+assert_equals $(cat .saunafs_tweaks | egrep CacheExpirationTime | awk '{print $2}') 2000
+assert_equals $(cat .saunafs_tweaks | egrep WriteMaxRetries | awk '{print $2}') 20
+assert_equals $(cat .saunafs_tweaks | egrep MaxReadaheadRequests | awk '{print $2}') 2
+assert_equals $(cat .saunafs_tweaks | egrep ReadWaveTimeout | awk '{print $2}') 200
+
+cd "${info[mount1]}"
+
+assert_equals $(cat .saunafs_tweaks | egrep CacheExpirationTime | awk '{print $2}') 2000
+assert_equals $(cat .saunafs_tweaks | egrep WriteMaxRetries | awk '{print $2}') 60
+assert_equals $(cat .saunafs_tweaks | egrep MaxReadaheadRequests | awk '{print $2}') 10
+assert_equals $(cat .saunafs_tweaks | egrep ReadWaveTimeout | awk '{print $2}') 300
diff --git a/tests/test_suites/ShortSystemTests/test_many_serverrooms_with_endangered_chunks_priority.sh b/tests/test_suites/ShortSystemTests/test_many_serverrooms_with_endangered_chunks_priority.sh
index df0723178..45fade3d6 100644
--- a/tests/test_suites/ShortSystemTests/test_many_serverrooms_with_endangered_chunks_priority.sh
+++ b/tests/test_suites/ShortSystemTests/test_many_serverrooms_with_endangered_chunks_priority.sh
@@ -3,7 +3,8 @@ USE_RAMDISK=YES \
 	CHUNKSERVERS=9 \
 	CHUNKSERVER_LABELS="0,1,2:sr1|3,4,5:sr2|6,7,8:sr3" \
 	MASTER_CUSTOM_GOALS="10 three_serverrooms: sr1 sr2 sr3" \
-	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER|ENDANGERED_CHUNKS_PRIORITY=1" \
+	MASTER_EXTRA_CONFIG="ENDANGERED_CHUNKS_PRIORITY=1" \
+	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER" \
 	setup_local_empty_saunafs info
 
 # Create many files, each should have at least one copy in each server room
diff --git a/tests/test_suites/ShortSystemTests/test_read_write_during_scan.sh b/tests/test_suites/ShortSystemTests/test_read_write_during_scan.sh
index c9952da0d..50c76e9e6 100644
--- a/tests/test_suites/ShortSystemTests/test_read_write_during_scan.sh
+++ b/tests/test_suites/ShortSystemTests/test_read_write_during_scan.sh
@@ -6,7 +6,7 @@ USE_RAMDISK=YES \
 	MOUNTS=1
 	CHUNKSERVERS=3 \
 	DISK_PER_CHUNKSERVER=3 \
-	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER|direct_io|DirectIO"
+	MOUNT_EXTRA_CONFIG="sfscachemode=NEVER|sfsdirectio=1"
 	setup_local_empty_saunafs info
 
 # Create a directory with many files on mountpoint
diff --git a/tests/tools/saunafs.sh b/tests/tools/saunafs.sh
index 33d1ca416..82b215232 100644
--- a/tests/tools/saunafs.sh
+++ b/tests/tools/saunafs.sh
@@ -597,6 +597,31 @@ add_chunkserver_() {
 	saunafs_info_[chunkserver${chunkserver_id}_hdd]=$hdd_cfg
 }
 
+function validate_and_append_fuse_options() {
+	local prefix=$1
+	local mount_cmd=$2
+	local this_fuse_options_variable="FUSE_${prefix}_EXTRA_CONFIG"
+	local fuse_options=""
+
+	for fuse_option in $(echo ${FUSE_EXTRA_CONFIG-} | tr '|' '\n'); do
+		fuse_option_name=$(echo $fuse_option | cut -f1 -d'=' |
+			tr '[:upper:]' '[:lower:]')
+		${mount_cmd} --help |& grep " -o ${fuse_option_name}[ =]" >/dev/null ||
+			test_fail "Your libfuse doesn't support $fuse_option_name flag"
+		fuse_options+="-o $fuse_option "
+	done
+
+	for fuse_option in $(echo ${!this_fuse_options_variable-} | tr '|' '\n'); do
+		fuse_option_name=$(echo $fuse_option | cut -f1 -d'=' |
+			tr '[:upper:]' '[:lower:]')
+		${mount_cmd} --help |& grep " -o ${fuse_option_name}[ =]" >/dev/null ||
+			test_fail "Your libfuse doesn't support $fuse_option_name flag"
+		fuse_options+="-o $fuse_option "
+	done
+
+	echo $fuse_options
+}
+
 create_sfsmount_cfg_() {
 	local this_mount_cfg_variable="MOUNT_${1}_EXTRA_CONFIG"
 	local this_mount_exports_variable="MOUNT_${1}_EXTRA_EXPORTS"
@@ -616,7 +641,9 @@ windows_do_mount_() {
 	local mount_dir=$3
 	local user_id=$4
 	local group_id=$5
-	sfsmount3.exe -c ${mount_cfg} -D ${drive}: --uid $user_id --gid $group_id &
+	local mount_id=$6
+	local fuse_options=$(validate_and_append_fuse_options "${mount_id}" "sfsmount3.exe")
+	sfsmount3.exe -c ${mount_cfg} -D ${drive}: --uid $user_id --gid $group_id ${fuse_options} &
 	disown $!
 
 	local counter=0
@@ -657,13 +684,7 @@ configure_mount_() {
 	local mount_cmd=$2
 	local mount_cfg=${saunafs_info_[mount${mount_id}_cfg]}
 	local mount_dir=${saunafs_info_[mount${mount_id}]}
-	local fuse_options=""
-	for fuse_option in $(echo ${FUSE_EXTRA_CONFIG-} | tr '|' '\n'); do
-		fuse_option_name=$(echo $fuse_option | cut -f1 -d'=')
-		${mount_cmd} --help |& grep " -o ${fuse_option_name}[ =]" >/dev/null ||
-			test_fail "Your libfuse doesn't support $fuse_option_name flag"
-		fuse_options+="-o $fuse_option "
-	done
+	local fuse_options=$(validate_and_append_fuse_options "${mount_id}" "${mount_cmd}")
 	local call="${command_prefix} ${mount_cmd} -c ${mount_cfg} ${mount_dir} ${fuse_options}"
 	saunafs_info_[mntcall${mount_id}]=$call
 	saunafs_info_[mnt${mount_id}_command]=${mount_cmd}
@@ -679,7 +700,7 @@ windows_prepare_mount_() {
 	local group_id=$3
 	create_sfsmount_cfg_ ${mount_id} > "$mount_cfg"
 
-	local mount_call="windows_do_mount_ $mount_cfg ${mount_big_dir_letter} ${mount_dir} $user_id $group_id"
+	local mount_call="windows_do_mount_ $mount_cfg ${mount_big_dir_letter} ${mount_dir} $user_id $group_id ${mount_id}"
 	local umount_call="windows_do_umount_ ${mount_big_dir_letter} ${mount_dir}"
 
 	if [ ! -d "$mount_dir" ]; then

From 3f61a50a70533e71ae9999c3708af3e06d03aa9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rolando=20S=C3=A1nchez=20Ramos?=
 <65413231+rolysr@users.noreply.github.com>
Date: Wed, 28 Aug 2024 18:01:27 -0400
Subject: [PATCH 24/27] feat(master): Add empty reserved files option (#176)

For avoiding reserved files to take system space indefinitely,
this change add a periodical empty procedure over reserved files.
---
 doc/sfsmount.1.adoc                           |  2 +-
 src/data/sfsmaster.cfg.in                     |  5 ++
 src/master/filesystem.cc                      |  1 +
 src/master/filesystem_periodic.cc             | 38 ++++++++++++
 src/master/filesystem_periodic.h              |  2 +
 .../test_create_empty_reserved_files.sh       | 58 +++++++++++++++++++
 6 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_suites/ShortSystemTests/test_create_empty_reserved_files.sh

diff --git a/doc/sfsmount.1.adoc b/doc/sfsmount.1.adoc
index d59cdc9a9..f50ccb7fc 100644
--- a/doc/sfsmount.1.adoc
+++ b/doc/sfsmount.1.adoc
@@ -192,7 +192,7 @@ new session is created without password.
 Specify number of retries before I/O error is returned (default: 30).
 
 *-o sfsreportreservedperiod=*'N'::
-Specify interval of reporting reserved inodes in seconds (default: 60).
+Specify interval for reporting reserved inodes in seconds (default: 30).
 
 *-o sfsiolimits=*'PATH'::
 Specify local I/O limiting configuration file (default: no I/O limiting).
diff --git a/src/data/sfsmaster.cfg.in b/src/data/sfsmaster.cfg.in
index 4506dce38..4f7abfe92 100644
--- a/src/data/sfsmaster.cfg.in
+++ b/src/data/sfsmaster.cfg.in
@@ -241,6 +241,11 @@
 ## (Default: 3600)
 # METADATA_DUMP_PERIOD_SECONDS = 3600
 
+## Interval for periodically cleaning of reserved files, in milliseconds. If set to 0, the reserved files 
+## deletion is disabled.
+## (Default: 0)
+# EMPTY_RESERVED_FILES_PERIOD_MSECONDS = 0
+
 # deprecated:
 # CHUNKS_DEL_LIMIT - use CHUNKS_SOFT_DEL_LIMIT instead
 
diff --git a/src/master/filesystem.cc b/src/master/filesystem.cc
index 6825fefd6..dc6ce03de 100644
--- a/src/master/filesystem.cc
+++ b/src/master/filesystem.cc
@@ -348,6 +348,7 @@ static void fs_read_config_file() {
 		safs_pretty_syslog(LOG_WARNING, "REPLICATIONS_DELAY_INIT and REPLICATION_DELAY_DISCONNECT"
 		" entries are deprecated. Use OPERATIONS_DELAY_INIT and OPERATIONS_DELAY_DISCONNECT instead.");
 	}
+	gEmptyReservedFilesPeriod = cfg_getuint32("EMPTY_RESERVED_FILES_PERIOD_MSECONDS", 0);
 
 	chunk_invalidate_goal_cache();
 	fs_read_goal_config_file(); // may throw
diff --git a/src/master/filesystem_periodic.cc b/src/master/filesystem_periodic.cc
index abc7c4ce5..d4d2663a1 100644
--- a/src/master/filesystem_periodic.cc
+++ b/src/master/filesystem_periodic.cc
@@ -586,11 +586,48 @@ uint8_t fs_apply_emptytrash_deprecated(uint32_t ts, uint32_t freeinodes, uint32_
 	return SAUNAFS_STATUS_OK;
 }
 
+#ifndef METARESTORE
+static void fs_do_emptyreserved(uint32_t ts) {
+	SignalLoopWatchdog watchdog;
+
+	auto it = gMetadata->reserved.begin();
+	watchdog.start();
+	while (it != gMetadata->reserved.end()) {
+		FSNodeFile *node = fsnodes_id_to_node_verify<FSNodeFile>((*it).first);
+
+		if (!node) {
+			gMetadata->reserved.erase(it);
+			it = gMetadata->reserved.begin();
+			continue;
+		}
+
+		assert(node->type == FSNode::kReserved);
+
+		uint32_t node_id = node->id;
+		fsnodes_purge(ts, node);
+
+		// Purge operation should be performed anyway
+		fs_changelog(ts, "PURGE(%" PRIu32 ")", node_id);
+
+		it = gMetadata->reserved.begin();
+
+		if (watchdog.expired()) {
+			break;
+		}
+	}
+}
+#endif
+
 uint8_t fs_apply_emptyreserved_deprecated(uint32_t /*ts*/,uint32_t /*freeinodes*/) {
 	return SAUNAFS_STATUS_OK;
 }
 
 #ifndef METARESTORE
+void fs_periodic_emptyreserved(void) {
+	uint32_t ts = eventloop_time();
+	fs_do_emptyreserved(ts);
+}
+
 void fs_read_periodic_config_file() {
 	gFileTestLoopTime = cfg_get_minmaxvalue<uint32_t>("FILE_TEST_LOOP_MIN_TIME", 3600, FILETESTSMINLOOPTIME, FILETESTSMAXLOOPTIME);
 }
@@ -601,5 +638,6 @@ void fs_periodic_master_init() {
 	eventloop_eachloopregister(fs_background_task_manager_work);
 	eventloop_eachloopregister(fs_background_file_test);
 	eventloop_timeregister_ms(100, fs_periodic_emptytrash);
+	eventloop_timeregister_ms(gEmptyReservedFilesPeriod, fs_periodic_emptyreserved);
 }
 #endif
diff --git a/src/master/filesystem_periodic.h b/src/master/filesystem_periodic.h
index dbf84c783..6dd86dcb7 100644
--- a/src/master/filesystem_periodic.h
+++ b/src/master/filesystem_periodic.h
@@ -24,6 +24,8 @@
 
 #include "common/defective_file_info.h"
 
+inline uint32_t gEmptyReservedFilesPeriod = 0;
+
 std::vector<DefectiveFileInfo> fs_get_defective_nodes_info(uint8_t requested_flags, uint64_t max_entries,
 	                                                   uint64_t &entry_index);
 
diff --git a/tests/test_suites/ShortSystemTests/test_create_empty_reserved_files.sh b/tests/test_suites/ShortSystemTests/test_create_empty_reserved_files.sh
new file mode 100644
index 000000000..0138d66d5
--- /dev/null
+++ b/tests/test_suites/ShortSystemTests/test_create_empty_reserved_files.sh
@@ -0,0 +1,58 @@
+CHUNKSERVERS=1 \
+	MOUNTS=2
+	USE_RAMDISK=YES \
+	MASTER_EXTRA_CONFIG="EMPTY_RESERVED_FILES_PERIOD_MSECONDS = 1000"\
+	MOUNT_0_EXTRA_CONFIG="sfscachemode=NEVER" \
+	MOUNT_1_EXTRA_CONFIG="sfsmeta" \
+	SFSEXPORTS_META_EXTRA_OPTIONS="nonrootmeta" \
+	setup_local_empty_saunafs info
+
+cd "${info[mount0]}"
+
+# Function to count the number of files in a given directory
+count_files() {
+    local dir_path="$1"
+    echo $(ls "$dir_path" | wc -l)
+}
+
+# set trash and reserved files folder
+trash="${info[mount1]}/trash"
+reserved="${info[mount1]}/reserved"
+
+mkdir folder
+
+# set folder trash time to 0 for redirecting 
+# files in use to reserved files folder
+saunafs settrashtime 0 folder
+
+# create 5 files and keep them in use
+for i in {1..5}; do
+    touch folder/file$i
+    while true; do echo "Data" >> folder/file$i; sleep 1; done &
+done
+
+# delete those 5 files and as trashtime = 0
+# then these files are marked as reserved in the system
+for i in {1..5}; do
+    rm folder/file$i
+done
+
+# check files are not in trash folder, 
+# just undel folder
+trash_files_count=$(count_files $trash)
+echo "number of trash files: $trash_files_count"
+assert_equals "1" "$trash_files_count"
+
+# check there are 5 files on reserved files
+reserved_files_count=$(count_files $reserved)
+echo "number of reserved files: $reserved_files_count"
+assert_equals "5" "$reserved_files_count"
+
+# sleep for a enough period to allow cleanup
+# of reserved files
+sleep 5
+
+# check reserved files were correctly deleted
+reserved_files_count_after=$(count_files $reserved)
+echo "number of reserved files after cleanup: $reserved_files_count_after"
+assert_equals "0" "$reserved_files_count_after"

From 55e346079eebe2e3b60de383d82c665dc389a789 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rolando=20S=C3=A1nchez=20Ramos?= <rolysr@leil.io>
Date: Fri, 30 Aug 2024 09:08:34 -0400
Subject: [PATCH 25/27] feat(mount): Add readcachemaxsize option (#145)

For allowing the client process to limit the expected memory
in system for reading operations, an option was added so that
the client also waits for availability of system memory, avoiding
the client process to crash.
---
 doc/sfsmount.1.adoc               |  3 ++
 src/mount/chunk_reader.h          |  3 ++
 src/mount/client/saunafs_c_api.cc |  1 +
 src/mount/client/saunafs_c_api.h  |  1 +
 src/mount/fuse/main.cc            |  1 +
 src/mount/fuse/mount_config.cc    |  4 ++
 src/mount/fuse/mount_config.h     |  2 +
 src/mount/memory_info.h           | 70 +++++++++++++++++++++++++++++++
 src/mount/readdata.cc             | 66 ++++++++++++++++++++++++++++-
 src/mount/readdata.h              |  2 +
 src/mount/readdata_cache.h        | 13 +++++-
 src/mount/sauna_client.cc         |  1 +
 src/mount/sauna_client.h          |  4 ++
 13 files changed, 168 insertions(+), 3 deletions(-)
 create mode 100644 src/mount/memory_info.h

diff --git a/doc/sfsmount.1.adoc b/doc/sfsmount.1.adoc
index f50ccb7fc..f528b5d95 100644
--- a/doc/sfsmount.1.adoc
+++ b/doc/sfsmount.1.adoc
@@ -162,6 +162,9 @@ disables cache (default: 0).
 *-o readaheadmaxwindowsize=*'KB'::
 Set max value of readahead window per single descriptor in kibibytes (default: 16384).
 
+*-o readcachemaxsize=*'MB'::
+Set max value of read cache size in mebibytes (default: 16384).
+
 *-o readworkers=*'N'::
 Define number of read workers (default: 30).
 
diff --git a/src/mount/chunk_reader.h b/src/mount/chunk_reader.h
index 8ba570f3a..d41bc0f77 100644
--- a/src/mount/chunk_reader.h
+++ b/src/mount/chunk_reader.h
@@ -71,6 +71,9 @@ class ChunkReader {
 	uint32_t version() const {
 		return location_->version;
 	}
+	uint64_t fileLength() const {
+		return location_->fileLength;
+	}
 
 	/// Counter for the .saunafs_tweaks file.
 	static std::atomic<uint64_t> preparations;
diff --git a/src/mount/client/saunafs_c_api.cc b/src/mount/client/saunafs_c_api.cc
index dc95dd70e..9459d76f3 100644
--- a/src/mount/client/saunafs_c_api.cc
+++ b/src/mount/client/saunafs_c_api.cc
@@ -60,6 +60,7 @@ void sau_set_default_init_params(struct sau_init_params *params,
 	params->total_read_timeout_ms = SaunaClient::FsInitParams::kDefaultChunkserverTotalReadTo;
 	params->cache_expiration_time_ms = SaunaClient::FsInitParams::kDefaultCacheExpirationTime;
 	params->readahead_max_window_size_kB = SaunaClient::FsInitParams::kDefaultReadaheadMaxWindowSize;
+	params->read_cache_max_size_mB = SaunaClient::FsInitParams::kDefaultReadCacheMaxSize;
 	params->prefetch_xor_stripes = SaunaClient::FsInitParams::kDefaultPrefetchXorStripes;
 	params->bandwidth_overuse = SaunaClient::FsInitParams::kDefaultBandwidthOveruse;
 
diff --git a/src/mount/client/saunafs_c_api.h b/src/mount/client/saunafs_c_api.h
index f18b43082..8bbfc63ea 100644
--- a/src/mount/client/saunafs_c_api.h
+++ b/src/mount/client/saunafs_c_api.h
@@ -56,6 +56,7 @@ typedef struct sau_init_params {
 	unsigned total_read_timeout_ms;
 	unsigned cache_expiration_time_ms;
 	unsigned readahead_max_window_size_kB;
+	unsigned read_cache_max_size_mB;
 	bool prefetch_xor_stripes;
 	double bandwidth_overuse;
 
diff --git a/src/mount/fuse/main.cc b/src/mount/fuse/main.cc
index 0e6e7b14e..5388d7c72 100644
--- a/src/mount/fuse/main.cc
+++ b/src/mount/fuse/main.cc
@@ -230,6 +230,7 @@ static int mainloop(struct fuse_args *args, struct fuse_cmdline_opts *fuse_opts,
 	params.total_read_timeout_ms = gMountOptions.chunkservertotalreadto;
 	params.cache_expiration_time_ms = gMountOptions.cacheexpirationtime;
 	params.readahead_max_window_size_kB = gMountOptions.readaheadmaxwindowsize;
+	params.read_cache_max_size_mB = gMountOptions.readcachemaxsize,
 	params.read_workers = gMountOptions.readworkers;
 	params.max_readahead_requests = gMountOptions.maxreadaheadrequests;
 	params.prefetch_xor_stripes = gMountOptions.prefetchxorstripes;
diff --git a/src/mount/fuse/mount_config.cc b/src/mount/fuse/mount_config.cc
index e02fd900f..6dbd80ee2 100644
--- a/src/mount/fuse/mount_config.cc
+++ b/src/mount/fuse/mount_config.cc
@@ -83,6 +83,7 @@ struct fuse_opt gSfsOptsStage2[] = {
 	SFS_OPT("sfschunkservertotalreadto=%d", chunkservertotalreadto, 0),
 	SFS_OPT("cacheexpirationtime=%d", cacheexpirationtime, 0),
 	SFS_OPT("readaheadmaxwindowsize=%d", readaheadmaxwindowsize, 4096),
+	SFS_OPT("readcachemaxsize=%d", readcachemaxsize, 16384),
 	SFS_OPT("readworkers=%d", readworkers, 1),
 	SFS_OPT("maxreadaheadrequests=%d", maxreadaheadrequests, 0),
 	SFS_OPT("sfsprefetchxorstripes", prefetchxorstripes, 1),
@@ -154,6 +155,8 @@ void usage(const char *progname) {
 				"of a xor chunk\n"
 "    -o bandwidthoveruse=N       define ratio of allowed bandwidth overuse "
 				"when fetching data (default: %.2f)\n"
+"    -o readcachemaxsize=MB      define the maximum cache memory size "
+				"for reading operations in mebibytes (default: %u)\n"
 "\n"
 "Write related options:\n"
 "    -o sfschunkserverwriteto=MSEC  set chunkserver response timeout during "
@@ -239,6 +242,7 @@ void usage(const char *progname) {
 		SaunaClient::FsInitParams::kDefaultChunkserverReadTo,
 		SaunaClient::FsInitParams::kDefaultChunkserverTotalReadTo,
 		SaunaClient::FsInitParams::kDefaultBandwidthOveruse,
+		SaunaClient::FsInitParams::kDefaultReadCacheMaxSize,
 		SaunaClient::FsInitParams::kDefaultChunkserverWriteTo,
 		SaunaClient::FsInitParams::kDefaultWriteCacheSize,
 		SaunaClient::FsInitParams::kDefaultCachePerInodePercentage,
diff --git a/src/mount/fuse/mount_config.h b/src/mount/fuse/mount_config.h
index 30bb617bb..72c13d8b3 100644
--- a/src/mount/fuse/mount_config.h
+++ b/src/mount/fuse/mount_config.h
@@ -106,6 +106,7 @@ struct sfsopts_ {
 	int chunkserverwriteto;
 	int cacheexpirationtime;
 	int readaheadmaxwindowsize;
+	int readcachemaxsize;
 	unsigned readworkers;
 	unsigned maxreadaheadrequests;
 	int prefetchxorstripes;
@@ -162,6 +163,7 @@ struct sfsopts_ {
 		chunkserverwriteto(SaunaClient::FsInitParams::kDefaultChunkserverWriteTo),
 		cacheexpirationtime(SaunaClient::FsInitParams::kDefaultCacheExpirationTime),
 		readaheadmaxwindowsize(SaunaClient::FsInitParams::kDefaultReadaheadMaxWindowSize),
+		readcachemaxsize(SaunaClient::FsInitParams::kDefaultReadCacheMaxSize),
 		readworkers(SaunaClient::FsInitParams::kDefaultReadWorkers),
 		maxreadaheadrequests(SaunaClient::FsInitParams::kDefaultMaxReadaheadRequests),
 		prefetchxorstripes(SaunaClient::FsInitParams::kDefaultPrefetchXorStripes),
diff --git a/src/mount/memory_info.h b/src/mount/memory_info.h
new file mode 100644
index 000000000..aa0fb0837
--- /dev/null
+++ b/src/mount/memory_info.h
@@ -0,0 +1,70 @@
+/*
+
+
+ Copyright 2023 Leil Storage OÜ
+
+ This file is part of SaunaFS.
+
+ SaunaFS is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, version 3.
+
+ SaunaFS is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with SaunaFS. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "common/platform.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#elif linux
+#include <sys/sysinfo.h>
+#endif
+
+class IMemoryInfo {
+public:
+    virtual ~IMemoryInfo() = default;
+    virtual unsigned long long getAvailableMemory() = 0;
+    virtual unsigned long long getTotalMemory() = 0;
+};
+
+#ifdef _WIN32
+class WindowsMemoryInfo : public IMemoryInfo {
+public:
+    unsigned long long getAvailableMemory() override {
+        MEMORYSTATUSEX memInfo;
+        memInfo.dwLength = sizeof(MEMORYSTATUSEX);
+        GlobalMemoryStatusEx(&memInfo);
+        return memInfo.ullAvailPhys;
+    }
+
+    unsigned long long getTotalMemory() override {
+        MEMORYSTATUSEX memInfo;
+        memInfo.dwLength = sizeof(MEMORYSTATUSEX);
+        GlobalMemoryStatusEx(&memInfo);
+        return memInfo.ullTotalPhys;
+    }
+};
+#elif __linux__
+class LinuxMemoryInfo : public IMemoryInfo {
+public:
+    unsigned long long getAvailableMemory() override {
+        struct sysinfo memInfo;
+        sysinfo(&memInfo);
+        return (memInfo.freeram * memInfo.mem_unit);
+    }
+
+    unsigned long long getTotalMemory() override {
+        struct sysinfo memInfo;
+        sysinfo(&memInfo);
+        return (memInfo.totalram * memInfo.mem_unit);
+    }
+};
+#endif
diff --git a/src/mount/readdata.cc b/src/mount/readdata.cc
index 41ae8ded7..418400eda 100644
--- a/src/mount/readdata.cc
+++ b/src/mount/readdata.cc
@@ -47,6 +47,7 @@
 #include "mount/mastercomm.h"
 #include "mount/readahead_adviser.h"
 #include "mount/readdata_cache.h"
+#include "mount/memory_info.h"
 #include "mount/tweaks.h"
 #include "protocol/SFSCommunication.h"
 
@@ -56,6 +57,49 @@
 #define EMPTY_REQUEST nullptr
 
 inline std::condition_variable readOperationsAvailable;
+inline int kMaxReadCacheRequestRetries = 10;
+
+std::unique_ptr<IMemoryInfo> createMemoryInfo() {
+    std::unique_ptr<IMemoryInfo> gMemoryInfo;
+    #ifdef _WIN32
+        gMemoryInfo = std::make_unique<WindowsMemoryInfo>();
+    #elif __linux__
+        gMemoryInfo = std::make_unique<LinuxMemoryInfo>();
+    #endif
+    return gMemoryInfo;
+}
+std::unique_ptr<IMemoryInfo> gMemoryInfo = createMemoryInfo();
+
+bool readShouldWaitForSystemMemory(size_t bytesToReadLeft) {
+	std::lock_guard lock(gUsedReadCacheMemoryMutex);
+	uint64_t avalaibleSystemMemory = gMemoryInfo->getAvailableMemory();
+	uint64_t virtualReadCacheAvailableMemory =
+		gReadCacheMaxSize.load() - gUsedReadCacheMemory;
+	return bytesToReadLeft > virtualReadCacheAvailableMemory ||
+			bytesToReadLeft > avalaibleSystemMemory;
+}
+
+void increaseUsedReadCacheMemory(size_t bytesToReadLeft) {
+	std::lock_guard lock(gUsedReadCacheMemoryMutex);
+	gUsedReadCacheMemory += bytesToReadLeft;
+}
+
+uint32_t getBytesToBeReadFromCS(uint32_t index, uint32_t offset, uint32_t size,
+                                const uint64_t fileLength) {
+	uint64_t offsetInFile = static_cast<uint64_t>(index) * SFSCHUNKSIZE + offset;
+	uint32_t availableSize = size;  // requested data may lie beyond end of file
+	if (offsetInFile >= fileLength) {
+		// read request entirely beyond EOF, can't read anything
+		availableSize = 0;
+	} else if (offsetInFile + availableSize > fileLength) {
+		// read request partially beyond EOF, truncate request to EOF
+		availableSize = fileLength - offsetInFile;
+	}
+	if (availableSize == 0) {
+		return 0;
+	}
+	return availableSize;
+}
 
 RequestConditionVariablePair
 ReadaheadRequests::append(ReadaheadRequestPtr readaheadRequestPtr) {
@@ -257,8 +301,8 @@ void ReadaheadOperationsManager::addExtraRequests_(
 			continue;
 		}
 
-		ReadCache::Entry *entry =
-		    rrec->cache.forceInsert(maximumRequestedOffset, satisfyingSize);
+		ReadCache::Entry *entry = rrec->cache.forceInsert(
+		    maximumRequestedOffset, satisfyingSize);
 
 		// we are not going to use the return value from the addRequest_ call
 		RequestConditionVariablePair _ = addRequest_(rrec, entry);
@@ -275,6 +319,7 @@ inline ChunkConnectorUsingPool gChunkConnector(gReadConnectionPool);
 inline ReadaheadOperationsManager gReadaheadOperationsManager;
 inline std::mutex gMutex;
 inline std::mutex gReadaheadOperationsManagerMutex;
+inline std::mutex gReadCacheMemoryMutex;
 inline ReadRecords gActiveReadRecords;
 inline pthread_t delayedOpsThread;
 inline std::vector<pthread_t> readOpsThreads;
@@ -453,6 +498,7 @@ void read_data_init(uint32_t retries,
 		uint32_t chunkserverTotalReadTimeout_ms,
 		uint32_t cache_expiration_time_ms,
 		uint32_t readahead_max_window_size_kB,
+		uint32_t read_chache_max_size_mB,
 		uint32_t read_workers,
 		uint32_t max_readahead_requests,
 		bool prefetchXorStripes,
@@ -469,6 +515,8 @@ void read_data_init(uint32_t retries,
 	gChunkserverTotalReadTimeout_ms = chunkserverTotalReadTimeout_ms;
 	gCacheExpirationTime_ms = cache_expiration_time_ms;
 	gReadaheadMaxWindowSize = readahead_max_window_size_kB * 1024;
+	gReadCacheMaxSize.store(std::min<uint64_t>(
+	    read_chache_max_size_mB * (1024LL * 1024LL), gMemoryInfo->getTotalMemory()));
 	gReadWorkers = read_workers;
 	gMaxReadaheadRequests = max_readahead_requests;
 	gPrefetchXorStripes = prefetchXorStripes;
@@ -490,6 +538,7 @@ void read_data_init(uint32_t retries,
 	gTweaks.registerVariable("ReadTotalTimeout", gChunkserverTotalReadTimeout_ms);
 	gTweaks.registerVariable("CacheExpirationTime", gCacheExpirationTime_ms);
 	gTweaks.registerVariable("ReadaheadMaxWindowSize", gReadaheadMaxWindowSize);
+	gTweaks.registerVariable("ReadCacheMaxSize", gReadCacheMaxSize);
 	gTweaks.registerVariable("MaxReadaheadRequests", gMaxReadaheadRequests);
 	gTweaks.registerVariable("ReadChunkPrepare", ChunkReader::preparations);
 	gTweaks.registerVariable("ReqExecutedTotal", ReadPlanExecutor::executions_total_);
@@ -513,6 +562,7 @@ void read_data_term(void) {
 	}
 
 	clear_active_read_records();
+	gMemoryInfo.reset();
 }
 
 void read_inode_ops(uint32_t inode) { // attributes of inode have been changed - force reconnect and clear cache
@@ -583,6 +633,17 @@ int read_to_buffer(ReadRecord *rrec, uint64_t current_offset,
 			if (size_in_chunk > bytes_to_read) {
 				size_in_chunk = bytes_to_read;
 			}
+
+			uint32_t read_cache_bytes_to_reserve =
+				getBytesToBeReadFromCS(reader.index(), offset_in_chunk,
+										size_in_chunk, reader.fileLength());
+			std::unique_lock lock(gReadCacheMemoryMutex);
+			if (readShouldWaitForSystemMemory(read_cache_bytes_to_reserve)) {
+				throw RecoverableReadException(
+				    "Not enough read cache memory available for reading");
+			}
+			increaseUsedReadCacheMemory(read_cache_bytes_to_reserve);
+			lock.unlock();
 			uint32_t bytes_read_from_chunk = reader.readData(
 					read_buffer, offset_in_chunk, size_in_chunk,
 					gChunkserverConnectTimeout_ms, gChunkserverWaveReadTimeout_ms,
@@ -649,6 +710,7 @@ int read_data(ReadRecord *rrec, off_t fuseOffset, size_t fuseSize,
 			ChunkReader reader(gChunkConnector, rrec->locator,
 			                   gBandwidthOveruse);
 			uint64_t bytesRead = 0;
+
 			int errorCode = read_to_buffer(rrec,
 			                               requestOffset,
 			                               bytesToReadLeft,
diff --git a/src/mount/readdata.h b/src/mount/readdata.h
index a7c472012..09fe86d22 100644
--- a/src/mount/readdata.h
+++ b/src/mount/readdata.h
@@ -34,6 +34,7 @@
 inline std::atomic<uint32_t> gReadaheadMaxWindowSize;
 inline std::atomic<uint32_t> gCacheExpirationTime_ms;
 inline std::atomic<uint32_t> gMaxReadaheadRequests;
+inline std::atomic<uint64_t> gReadCacheMaxSize;
 
 enum class ReadaheadRequestState {
 	kInqueued,
@@ -337,6 +338,7 @@ void read_data_init(uint32_t retries, uint32_t chunkserverRoundTripTime_ms,
                     uint32_t chunkserverTotalReadTimeout_ms,
                     uint32_t cache_expiration_time_ms,
                     uint32_t readahead_max_window_size_kB,
+					uint32_t read_cache_max_size_mB,
                     uint32_t read_workers, uint32_t max_readahead_requests,
                     bool prefetchXorStripes, double bandwidth_overuse);
 void read_data_term();
diff --git a/src/mount/readdata_cache.h b/src/mount/readdata_cache.h
index 4a8dd1697..615870213 100644
--- a/src/mount/readdata_cache.h
+++ b/src/mount/readdata_cache.h
@@ -38,6 +38,9 @@
 
 #define MISSING_OFFSET_PTR nullptr
 
+inline std::mutex gUsedReadCacheMemoryMutex;
+inline uint64_t gUsedReadCacheMemory;
+
 class ReadCache {
 public:
 	typedef uint64_t Offset;
@@ -286,7 +289,7 @@ class ReadCache {
 	 * \return cache query result
 	 */
 	void query(Offset offset, Size size, ReadCache::Result &result,
-	           bool insertPending = true) {
+			bool insertPending = true) {
 		collectGarbage();
 
 		auto it = entries_.upper_bound(offset, Entry::OffsetComp());
@@ -403,15 +406,23 @@ class ReadCache {
 			reserved_entries_.push_back(*e);
 		} else {
 			assert(e->refcount == 0);
+			std::unique_lock lock(gUsedReadCacheMemoryMutex);
+			gUsedReadCacheMemory -= e->buffer.size();
+			lock.unlock();
 			delete e;
 		}
 		return ret;
 	}
 
 	void clearReserved(unsigned count) {
+		std::unique_lock<std::mutex> lock(gUsedReadCacheMemoryMutex,
+		                                  std::defer_lock);
 		while (!reserved_entries_.empty() && count-- > 0) {
 			Entry *e = std::addressof(reserved_entries_.front());
 			if (e->refcount == 0) {
+				lock.lock();
+				gUsedReadCacheMemory -= e->buffer.size();
+				lock.unlock();
 				reserved_entries_.pop_front();
 				delete e;
 			} else {
diff --git a/src/mount/sauna_client.cc b/src/mount/sauna_client.cc
index b35e095fc..feb16ca82 100644
--- a/src/mount/sauna_client.cc
+++ b/src/mount/sauna_client.cc
@@ -3490,6 +3490,7 @@ void fs_init(FsInitParams &params) {
 			params.total_read_timeout_ms,
 			params.cache_expiration_time_ms,
 			params.readahead_max_window_size_kB,
+			params.read_cache_max_size_mB,
 			params.read_workers,
 			params.max_readahead_requests,
 			params.prefetch_xor_stripes,
diff --git a/src/mount/sauna_client.h b/src/mount/sauna_client.h
index 9484274cb..b7fa80d39 100644
--- a/src/mount/sauna_client.h
+++ b/src/mount/sauna_client.h
@@ -73,6 +73,7 @@ struct FsInitParams {
 	static constexpr unsigned kDefaultChunkserverTotalReadTo = 2000;
 	static constexpr unsigned kDefaultCacheExpirationTime = 1000;
 	static constexpr unsigned kDefaultReadaheadMaxWindowSize = 65536;
+	static constexpr unsigned kDefaultReadCacheMaxSize = 16384;
 	static constexpr unsigned kDefaultReadWorkers = 30;
 	static constexpr unsigned kDefaultMaxReadaheadRequests = 5;
 	static constexpr bool     kDefaultPrefetchXorStripes = false;
@@ -129,6 +130,7 @@ struct FsInitParams {
 	             total_read_timeout_ms(kDefaultChunkserverTotalReadTo),
 	             cache_expiration_time_ms(kDefaultCacheExpirationTime),
 	             readahead_max_window_size_kB(kDefaultReadaheadMaxWindowSize),
+				 read_cache_max_size_mB(kDefaultReadCacheMaxSize),
 	             read_workers(kDefaultReadWorkers),
 	             max_readahead_requests(kDefaultMaxReadaheadRequests),
 	             prefetch_xor_stripes(kDefaultPrefetchXorStripes),
@@ -161,6 +163,7 @@ struct FsInitParams {
 	             total_read_timeout_ms(kDefaultChunkserverTotalReadTo),
 	             cache_expiration_time_ms(kDefaultCacheExpirationTime),
 	             readahead_max_window_size_kB(kDefaultReadaheadMaxWindowSize),
+				 read_cache_max_size_mB(kDefaultReadCacheMaxSize),
 	             read_workers(kDefaultReadWorkers),
 	             max_readahead_requests(kDefaultMaxReadaheadRequests),
 	             prefetch_xor_stripes(kDefaultPrefetchXorStripes),
@@ -200,6 +203,7 @@ struct FsInitParams {
 	unsigned total_read_timeout_ms;
 	unsigned cache_expiration_time_ms;
 	unsigned readahead_max_window_size_kB;
+	unsigned read_cache_max_size_mB;
 	unsigned read_workers;
 	unsigned max_readahead_requests;
 	bool prefetch_xor_stripes;

From d2840610cf9e5b407ceaa22fe175b7863818bebe Mon Sep 17 00:00:00 2001
From: Urmas Rist <urmas@leil.io>
Date: Tue, 3 Sep 2024 23:46:39 +0300
Subject: [PATCH 26/27] fix(mount): Set default cacheExpirationTime to 0 (#180)

This is a temporary mitigation for #179, it does not fix it.
---
 doc/sfsmount.1.adoc                                          | 5 ++++-
 src/mount/sauna_client.h                                     | 2 +-
 tests/test_suites/SanityChecks/test_options_random_casing.sh | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/doc/sfsmount.1.adoc b/doc/sfsmount.1.adoc
index f528b5d95..9f1c81b4b 100644
--- a/doc/sfsmount.1.adoc
+++ b/doc/sfsmount.1.adoc
@@ -157,7 +157,10 @@ operation in milliseconds (default: 2000).
 
 *-o cacheexpirationtime=*'MSEC'::
 Set timeout for read cache entries to be considered valid in milliseconds. 0
-disables cache (default: 0).
+disables cache (default: 0). Due to a bug, this can sometimes result in invalid
+data being returned if reading the mount point from multiple processes/threads
+and it's not disabled. Verify data being sent using a checksum if using this
+feature.
 
 *-o readaheadmaxwindowsize=*'KB'::
 Set max value of readahead window per single descriptor in kibibytes (default: 16384).
diff --git a/src/mount/sauna_client.h b/src/mount/sauna_client.h
index b7fa80d39..5a982df31 100644
--- a/src/mount/sauna_client.h
+++ b/src/mount/sauna_client.h
@@ -71,7 +71,7 @@ struct FsInitParams {
 	static constexpr unsigned kDefaultChunkserverReadTo = 2000;
 	static constexpr unsigned kDefaultChunkserverWaveReadTo = 500;
 	static constexpr unsigned kDefaultChunkserverTotalReadTo = 2000;
-	static constexpr unsigned kDefaultCacheExpirationTime = 1000;
+	static constexpr unsigned kDefaultCacheExpirationTime = 0;
 	static constexpr unsigned kDefaultReadaheadMaxWindowSize = 65536;
 	static constexpr unsigned kDefaultReadCacheMaxSize = 16384;
 	static constexpr unsigned kDefaultReadWorkers = 30;
diff --git a/tests/test_suites/SanityChecks/test_options_random_casing.sh b/tests/test_suites/SanityChecks/test_options_random_casing.sh
index a3e128d69..1a6bb7d80 100644
--- a/tests/test_suites/SanityChecks/test_options_random_casing.sh
+++ b/tests/test_suites/SanityChecks/test_options_random_casing.sh
@@ -10,7 +10,7 @@ CHUNKSERVERS=1 \
 cd "${info[mount0]}"
 
 # Expected values are current defaults
-assert_equals $(cat .saunafs_tweaks | egrep CacheExpirationTime | awk '{print $2}') 1000
+assert_equals $(cat .saunafs_tweaks | egrep CacheExpirationTime | awk '{print $2}') 0
 assert_equals $(cat .saunafs_tweaks | egrep WriteMaxRetries | awk '{print $2}') 30
 assert_equals $(cat .saunafs_tweaks | egrep MaxReadaheadRequests | awk '{print $2}') 5
 assert_equals $(cat .saunafs_tweaks | egrep ReadWaveTimeout | awk '{print $2}') 500

From c2c96cd9daf3fb252540abaff569c470a12b48b0 Mon Sep 17 00:00:00 2001
From: Luis Guillermo Silva Rojas <guillex@leil.io>
Date: Fri, 6 Sep 2024 15:51:11 +0200
Subject: [PATCH 27/27] chore: Changelog for v4.5.0 (#183)

---
 NEWS             | 13 ++++++++++++-
 debian/changelog | 13 ++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 7d1c89aa6..fad3516f0 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,17 @@
 This file lists noteworthy changes in SaunaFS.
 
- * SaunaFS (4.5.0) (2024-08-12)
+ * SaunaFS (4.5.0) (2024-09-05)
+ - fix(mount): Set default cacheExpirationTime to 0 (#180)
+ - feat(mount): Add readcachemaxsize option (#145)
+ - feat(master): Add empty reserved files option (#176)
+ - fix(mount): Fix random case option parsing (#135)
+ - feat(chunkserver): Add IDiskManager::getDiskForGC (#169)
+ - test: Improve `stop_ganesha()` function (#168)
+ - fix(ci): Modify timeout from env (#165)
+ - ci: Fix tests behavior (#172)
+ - fix(uraft): Fix missing floating ip address
+ - fix(mount): Fix create time (Windows) (#167)
+ - feat(admin): Add list-disk-groups command (#155)
  - ci: Abort previous concurrent tests
  - ci: Update de base docker image on build
  - fix(tests): Fix typo in setup-machine script
diff --git a/debian/changelog b/debian/changelog
index a12f9d596..2d706f9cd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,15 @@
 saunafs (4.5.0) stable; urgency=medium
+  * fix(mount): Set default cacheExpirationTime to 0 (#180)
+  * feat(mount): Add readcachemaxsize option (#145)
+  * feat(master): Add empty reserved files option (#176)
+  * fix(mount): Fix random case option parsing (#135)
+  * feat(chunkserver): Add IDiskManager::getDiskForGC (#169)
+  * test: Improve `stop_ganesha()` function (#168)
+  * fix(ci): Modify timeout from env (#165)
+  * ci: Fix tests behavior (#172)
+  * fix(uraft): Fix missing floating ip address
+  * fix(mount): Fix create time (Windows) (#167)
+  * feat(admin): Add list-disk-groups command (#155)
   * ci: Abort previous concurrent tests
   * ci: Update de base docker image on build
   * fix(tests): Fix typo in setup-machine script
@@ -12,7 +23,7 @@ saunafs (4.5.0) stable; urgency=medium
   * fix: Allow legacy sessions download
   * fix: Allow legacy metadata download
 
- -- SaunaFS Team <contact@saunafs.com>  Mon, 12 Aug 2024 15:11:39 +0000
+ -- SaunaFS Team <contact@saunafs.com>  Thu, 05 Sep 2024 14:23:04 +0000
 
 saunafs (4.4.0) stable; urgency=medium
   * ci(.github): Add check for Conventional Commits (#131)