From d92cd5ae037ae85ab9730499d99e5c1bd475eed2 Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 06 二月 2024 21:22:21 +0800
Subject: [PATCH] Funasr1.0 (#1362)
---
funasr/train_utils/trainer.py | 26 +++++++++++++++++++++++---
1 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/funasr/train_utils/trainer.py b/funasr/train_utils/trainer.py
index 62d6be8..d144019 100644
--- a/funasr/train_utils/trainer.py
+++ b/funasr/train_utils/trainer.py
@@ -148,6 +148,7 @@
self._train_epoch(epoch)
+
if self.use_ddp or self.use_fsdp:
dist.barrier()
@@ -156,8 +157,8 @@
if self.use_ddp or self.use_fsdp:
dist.barrier()
-
-
+
+
if self.rank == 0:
self._save_checkpoint(epoch)
@@ -172,7 +173,8 @@
if self.use_ddp or self.use_fsdp:
dist.barrier()
-
+
+
if self.writer:
self.writer.close()
@@ -202,7 +204,25 @@
my_context = self.model.no_sync if batch_idx % accum_grad != 0 else nullcontext
with my_context():
time2 = time.perf_counter()
+ print("before, GPU, memory: {:.1} MB, "
+ "{:.1} MB, "
+ "{:.1} MB, "
+ "{:.1} MB".format(torch.cuda.memory_allocated()/1024/1024/1024,
+ torch.cuda.max_memory_allocated()/1024/1024/1024,
+ torch.cuda.memory_reserved()/1024/1024/1024,
+ torch.cuda.max_memory_reserved()/1024/1024/1024,
+ ))
+
retval = self.model(**batch)
+ torch.cuda.empty_cache()
+ print("after, GPU, memory: {:.1} MB, "
+ "{:.1} MB, "
+ "{:.1} MB, "
+ "{:.1} MB".format(torch.cuda.memory_allocated()/1024/1024/1024,
+ torch.cuda.max_memory_allocated()/1024/1024/1024,
+ torch.cuda.memory_reserved()/1024/1024/1024,
+ torch.cuda.max_memory_reserved()/1024/1024/1024,
+ ))
time3 = time.perf_counter()
speed_stats["forward_time"] = f"{time3 - time2:0.3f}"
loss, stats, weight = retval
--
Gitblit v1.9.1