From b394aecbee042153a88e2854ffec9824acd6e9e8 Mon Sep 17 00:00:00 2001 From: lucidrains Date: Thu, 5 Oct 2023 14:39:17 -0700 Subject: [PATCH] more barriers --- audiolm_pytorch/trainer.py | 3 +++ audiolm_pytorch/version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/audiolm_pytorch/trainer.py b/audiolm_pytorch/trainer.py index 8f177e7..62568dd 100644 --- a/audiolm_pytorch/trainer.py +++ b/audiolm_pytorch/trainer.py @@ -317,6 +317,7 @@ def __init__( if self.is_main and force_clear_prev_results is True or (not exists(force_clear_prev_results) and len([*self.results_folder.glob('**/*')]) > 0 and yes_or_no('do you want to clear previous experiment checkpoints and results?')): rmtree(str(self.results_folder)) + self.accelerator.wait_for_everyone() self.results_folder.mkdir(parents = True, exist_ok = True) # Initialize experiment trackers if an external Accelerator is not passed in @@ -705,6 +706,7 @@ def __init__( if self.is_main and force_clear_prev_results is True or (not exists(force_clear_prev_results) and len([*self.results_folder.glob('**/*')]) > 0 and yes_or_no('do you want to clear previous experiment checkpoints and results?')): rmtree(str(self.results_folder)) + self.accelerator.wait_for_everyone() self.results_folder.mkdir(parents = True, exist_ok = True) hps = {"num_train_steps": num_train_steps, "data_max_length": data_max_length, "learning_rate": lr} @@ -1239,6 +1241,7 @@ def __init__( if force_clear_prev_results is True or (not exists(force_clear_prev_results) and len([*self.results_folder.glob('**/*')]) > 0 and yes_or_no('do you want to clear previous experiment checkpoints and results?')): rmtree(str(self.results_folder)) + self.accelerator.wait_for_everyone() self.results_folder.mkdir(parents = True, exist_ok = True) hps = {"num_train_steps": num_train_steps, "data_max_length": data_max_length, "learning_rate": lr} diff --git a/audiolm_pytorch/version.py b/audiolm_pytorch/version.py index c24ed73..63a5878 100644 --- a/audiolm_pytorch/version.py +++ b/audiolm_pytorch/version.py @@ -1 +1 @@ -__version__ = '1.5.4' +__version__ = '1.5.5'