Disable GPU memory pre-allocation by default during training.

- This affects some systems with low memory GPUs.
talmolab · Aug 18, 2020 · 0aef828 · 0aef828
1 parent 5e1efa2
commit 0aef828
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/sleap/nn/training.py b/sleap/nn/training.py
@@ -1356,6 +1356,13 @@ def main():
     logger.info(job_config.to_json())
     logger.info("")
 
+    logger.info("System:")
+    if sleap.nn.system.is_gpu_system():
+        # Disable preallocation to handle Linux/low GPU memory issue.
+        sleap.nn.system.disable_preallocation()
+    sleap.nn.system.summary()
+    logger.info("")
+
     logger.info("Initializing trainer...")
     # Create a trainer and run!
     trainer = Trainer.from_config(

diff --git a/sleap/version.py b/sleap/version.py
@@ -10,4 +10,4 @@
 
 Must be a semver string, "aN" should be appended for alpha releases.
 """
-__version__ = "1.0.9a2"
+__version__ = "1.0.9a3"