Add configurable option for target_uncompressed_size

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -602,7 +602,6 @@ def generic_start_worker(
         "-w", str(CONTAINER_CLP_HOME),
         "--name", container_name,
         "--log-driver", "local",
-        "-u", f"{os.getuid()}:{os.getgid()}",
         "-e", f"PYTHONPATH={clp_site_packages_dir}",
         "-e", (
             f"BROKER_URL=amqp://"
@@ -619,6 +618,7 @@ def generic_start_worker(
         "-e", f"CLP_LOGS_DIR={container_logs_dir}",
         "-e", f"CLP_LOGGING_LEVEL={worker_config.logging_level}",
         "-e", f"CLP_STORAGE_ENGINE={clp_config.package.storage_engine}",
+        "-u", f"{os.getuid()}:{os.getgid()}",
     ]
     if worker_specific_env:
         for env_name, env_value in worker_specific_env.items():

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -330,13 +330,20 @@ def dump_to_primitive_dict(self):
 
 class IrOutput(BaseModel):
     directory: pathlib.Path = pathlib.Path("var") / "data" / "ir"
+    target_uncompressed_size: int = 128 * 1024 * 1024
 
     @validator("directory")
     def validate_directory(cls, field):
         if "" == field:
             raise ValueError("directory can not be empty")
         return field
 
+    @validator("target_uncompressed_size")
+    def validate_target_uncompressed_size(cls, field):
+        if field <= 0:
+            raise ValueError("target_uncompressed_size must be greater than 0")
+        return field
+
     def make_config_paths_absolute(self, clp_home: pathlib.Path):
         self.directory = make_config_path_absolute(clp_home, self.directory)
 

components/package-template/src/etc/clp-config.yml

@@ -82,6 +82,10 @@
 #ir_output:
 #  directory: "var/data/ir"
 #
+#  # How large each IR chunk should be before being
+#  # split into a new IR chunk
+#  target_uncompressed_size: 134217728  # 128 MB
+#
 ## Location where other data (besides archives) are stored. It will be created if
 ## it doesn't exist.
 #data_directory: "var/data"