Skip to content

Commit

Permalink
rm unnecessary timer msg
Browse files Browse the repository at this point in the history
  • Loading branch information
haolin-nju committed Jan 3, 2025
1 parent 051db76 commit 6c8ebeb
Showing 1 changed file with 0 additions and 7 deletions.
7 changes: 0 additions & 7 deletions chatlearn/schedule/model_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import concurrent.futures
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
import time

import ray
import ray.experimental.state.api
Expand Down Expand Up @@ -175,24 +174,18 @@ def sync_parameters(self, episode_offset=0, requires_grad=None, validate=False):
episode_offset % sync_group.frequency == 0:
sync_group: ParameterSyncGroup = sync_group

start = time.perf_counter()
src_model, dst_model = sync_group.src_model, sync_group.dst_model
refs = src_model.onload(to_build_grad_buffers=False, to_onload_main_weights=False, to_onload_optimizer_states=False)
future.wait(refs)
refs = dst_model.onload(to_build_grad_buffers=False, to_onload_main_weights=False, to_onload_optimizer_states=False)
future.wait(refs)
logger.info(f"============In sync_parameters, onload {sync_group} elapsed {time.perf_counter() - start} s")

start = time.perf_counter()
sync_group.sync(requires_grad, validate)
logger.info(f"============In sync_parameters, synchronizing {sync_group} elapsed {time.perf_counter() - start} s")

start = time.perf_counter()
refs = src_model.offload()
future.wait(refs)
refs = dst_model.offload()
future.wait(refs)
logger.info(f"============In sync_parameters, offload {sync_group} elapsed {time.perf_counter() - start} s")

def set_func_decorator(self, model):
if is_decorated(model.name):
Expand Down

0 comments on commit 6c8ebeb

Please sign in to comment.