Store size limit #319 #321

storage/storage/src/main/java/org/eclipse/store/storage/exceptions/StorageExceptionCommitSizeExceeded.java

@@ -0,0 +1,31 @@
+package org.eclipse.store.storage.exceptions;
+
+/*-
+ * #%L
+ * EclipseStore Storage
+ * %%
+ * Copyright (C) 2023 - 2025 MicroStream Software
+ * %%
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ * #L%
+ */
+
+@SuppressWarnings("serial")
+public class StorageExceptionCommitSizeExceeded extends StorageException
+{
+
+	public StorageExceptionCommitSizeExceeded(String message)
+	{
+		super(message);
+	}
+
+	public StorageExceptionCommitSizeExceeded(int channelIndex, long commitSize)
+	{
+		super("Channel " + channelIndex + " store size " + commitSize  + " bytes exceeds technical limit of 2^31 bytes");
+	}
+
+}

storage/storage/src/main/java/org/eclipse/store/storage/types/StorageDataChunkValidator.java

@@ -18,18 +18,18 @@
 
 import java.nio.ByteBuffer;
 
-import org.eclipse.store.storage.exceptions.StorageExceptionConsistency;
 import org.eclipse.serializer.memory.XMemory;
 import org.eclipse.serializer.persistence.binary.types.Binary;
 import org.eclipse.serializer.persistence.binary.types.BinaryEntityRawDataIterator;
+import org.eclipse.store.storage.exceptions.StorageExceptionCommitSizeExceeded;
+import org.eclipse.store.storage.exceptions.StorageExceptionConsistency;
 
 
 public interface StorageDataChunkValidator
 {
 	public void validateDataChunk(Binary data);
 
-
-
+
 	public static StorageDataChunkValidator New(
 		final BinaryEntityRawDataIterator entityDataIterator ,
 		final StorageEntityDataValidator  entityDataValidator
@@ -321,5 +321,45 @@ public final Provider provideDataChunkValidatorProvider(
 		}
 
 	}
+
+	public class MaxFileSize implements StorageDataChunkValidator, Provider, Provider2
+	{
+		@Override
+		public Provider provideDataChunkValidatorProvider(
+				final StorageFoundation<?> foundation
+		)
+		{
+			return this;
+		}
+
+		@Override
+		public StorageDataChunkValidator provideDataChunkValidator(
+				final StorageTypeDictionary typeDictionary
+		)
+		{
+			return this;
+		}
+
+		@Override
+		public final void validateDataChunk(
+			final Binary data
+		)
+		{
+			for(int channelIndex = 0; channelIndex < data.channelCount(); channelIndex++)
+			{
+				long commitSize = 0;
+				for(ByteBuffer bb:data.channelChunk(channelIndex).buffers())
+				{
+					commitSize += bb.limit();
+				}
+
+				if(commitSize >= Integer.MAX_VALUE)
+				{
+					throw new StorageExceptionCommitSizeExceeded(channelIndex, commitSize);
+				}
+
+			}
+		}
+	}
 
 }

storage/storage/src/main/java/org/eclipse/store/storage/types/StorageFileManager.java

@@ -38,6 +38,7 @@
 import org.eclipse.serializer.util.X;
 import org.eclipse.serializer.util.logging.Logging;
 import org.eclipse.store.storage.exceptions.StorageException;
+import org.eclipse.store.storage.exceptions.StorageExceptionCommitSizeExceeded;
 import org.eclipse.store.storage.exceptions.StorageExceptionConsistency;
 import org.eclipse.store.storage.exceptions.StorageExceptionIoReading;
 import org.eclipse.store.storage.exceptions.StorageExceptionIoWriting;
@@ -594,6 +595,32 @@ private long ensureHeadFileTotalLength()
 
 			return physicalLength;
 		}
+
+		private void ensureHeadFileLoadableSize(final ByteBuffer[] dataBuffers)
+		{
+			//Must ensure that a file size never exceeds 2^31 bytes size.
+			//If that size is exceeded the storage is corrupted and can't be loaded any more because
+			//the current implementation use only one byte buffer to load a file.
+			//This buffer size is limited to an Integer.MAX_VALUE value.
+
+			long commitSize = 0;
+			for(int i = 0; i < dataBuffers.length; i++)
+			{
+				commitSize += dataBuffers[i].limit();
+				if(commitSize > Integer.MAX_VALUE)
+				{
+					//Should never be reached, this case should be already handled by
+					//org.eclipse.store.storage.types.StorageDataChunkValidator.MaxFileSize.
+					//But keep exception to have second guard as the validator might be replaced.
+					throw new StorageExceptionCommitSizeExceeded(this.channelIndex(), commitSize);
+				}
+				if(commitSize + this.headFile.totalLength() > Integer.MAX_VALUE)
+				{
+					logger.debug("creating new storage file because appending would exceed the file size limit of 2^31 bytes");
+					this.createNextStorageFile();
+				}
+			}
+		}
 
 		@Override
 		public final long[] storeChunks(final long timestamp, final ByteBuffer[] dataBuffers)
@@ -605,6 +632,8 @@ public final long[] storeChunks(final long timestamp, final ByteBuffer[] dataBuf
 			}
 
 			this.checkForNewFile();
+			this.ensureHeadFileLoadableSize(dataBuffers);
+
 			final long   oldTotalLength   = this.ensureHeadFileTotalLength();
 			final long[] storagePositions = allChunksStoragePositions(dataBuffers, oldTotalLength);
 			final long   writeCount       = this.writer.writeStore(this.headFile, X.ArrayView(dataBuffers));

storage/storage/src/main/java/org/eclipse/store/storage/types/StorageFoundation.java

@@ -973,7 +973,7 @@ protected StorageDataChunkValidator.Provider ensureDataChunkValidatorProvider()
 
 		protected StorageDataChunkValidator.Provider2 ensureDataChunkValidatorProvider2()
 		{
-			return new StorageDataChunkValidator.NoOp();
+			return new StorageDataChunkValidator.MaxFileSize();
 		}
 
 		protected StorageChannelsCreator ensureChannelCreator()