From 83d644c899b262d453a802fa41cccba586db96ef Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 12 六月 2024 19:47:53 +0800
Subject: [PATCH] decoding
---
funasr/datasets/openai_datasets/index_ds.py | 5 -----
funasr/train_utils/trainer_ds.py | 4 +++-
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/funasr/datasets/openai_datasets/index_ds.py b/funasr/datasets/openai_datasets/index_ds.py
index cc518f8..9943e2a 100644
--- a/funasr/datasets/openai_datasets/index_ds.py
+++ b/funasr/datasets/openai_datasets/index_ds.py
@@ -15,11 +15,6 @@
def __init__(self, path: str, **kwargs):
super().__init__()
- self.max_source_length = kwargs.get("max_source_length", 2048)
- self.min_source_length = kwargs.get("min_source_length", 0)
- self.max_target_length = kwargs.get("max_target_length", 2048)
- self.min_target_length = kwargs.get("min_target_length", 0)
- self.max_token_length = kwargs.get("max_token_length", 2200)
is_training = kwargs.get("is_training", True)
if not (path.endswith(".jsonl") or path.endswith(".json")):
diff --git a/funasr/train_utils/trainer_ds.py b/funasr/train_utils/trainer_ds.py
index ba8dd16..0fbac96 100644
--- a/funasr/train_utils/trainer_ds.py
+++ b/funasr/train_utils/trainer_ds.py
@@ -287,7 +287,9 @@
elif self.use_fsdp:
pass
elif self.rank == 0:
- logging.info(f"Save checkpoint: {epoch}, rank: {self.local_rank}\n")
+ logging.info(
+ f"Save checkpoint: {epoch}, rank: {self.rank}, local_rank: {self.local_rank}\n"
+ )
# self.step_or_epoch += 1
state = {
"epoch": epoch,
--
Gitblit v1.9.1