From d8154f5b5cfa04d6b1b735c732d5ce839f3d9442 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 18 四月 2023 17:30:37 +0800
Subject: [PATCH] Merge branch 'main' of github.com:alibaba-damo-academy/FunASR add
---
funasr/train/trainer.py | 18 ++++++++++--------
funasr/tasks/abs_task.py | 2 +-
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/funasr/tasks/abs_task.py b/funasr/tasks/abs_task.py
index 777513e..3d2004c 100644
--- a/funasr/tasks/abs_task.py
+++ b/funasr/tasks/abs_task.py
@@ -467,7 +467,7 @@
parser.add_argument(
"--batch_interval",
type=int,
- default=10000,
+ default=-1,
help="The batch interval for saving model.",
)
group.add_argument(
diff --git a/funasr/train/trainer.py b/funasr/train/trainer.py
index b12bded..9574a0d 100644
--- a/funasr/train/trainer.py
+++ b/funasr/train/trainer.py
@@ -571,8 +571,7 @@
#ouput dir
output_dir = Path(options.output_dir)
#batch interval
- batch_interval = options.batch_interval
- assert batch_interval > 0
+ batch_interval = options.batch_interval
start_time = time.perf_counter()
for iiter, (_, batch) in enumerate(
@@ -580,14 +579,17 @@
):
assert isinstance(batch, dict), type(batch)
- if rank == 0:
+ if batch_interval > 0 and (not distributed_option.distributed or rank == 0):
if hasattr(model, "num_updates") or (hasattr(model, "module") and hasattr(model.module, "num_updates")):
num_batch_updates = model.get_num_updates() if hasattr(model,"num_updates") else model.module.get_num_updates()
- if (num_batch_updates%batch_interval == 0) and (options.oss_bucket is not None) and options.use_pai:
- buffer = BytesIO()
- torch.save(model.state_dict(), buffer)
- options.oss_bucket.put_object(os.path.join(output_dir, f"{num_batch_updates}batch.pth"), buffer.getvalue())
-
+ if num_batch_updates % batch_interval == 0:
+ if options.use_pai and options.oss_bucket is not None:
+ buffer = BytesIO()
+ torch.save(model.state_dict(), buffer)
+ options.oss_bucket.put_object(os.path.join(output_dir, f"{num_batch_updates}step.pb"), buffer.getvalue())
+ else:
+ torch.save(model.state_dict(), os.path.join(output_dir, f"{num_batch_updates}step.pb"))
+
if distributed:
torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM)
if iterator_stop > 0:
--
Gitblit v1.9.1