From d5fa81d4dc11b3699b1a92e542b6160db42fd407 Mon Sep 17 00:00:00 2001
From: 嘉渊 <wangjiaming.wjm@alibaba-inc.com>
Date: 星期四, 18 五月 2023 14:40:47 +0800
Subject: [PATCH] update repo
---
funasr/datasets/small_datasets/dataset.py | 19 +++++++++++++++----
1 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/funasr/datasets/small_datasets/dataset.py b/funasr/datasets/small_datasets/dataset.py
index e14e4f1..7689a02 100644
--- a/funasr/datasets/small_datasets/dataset.py
+++ b/funasr/datasets/small_datasets/dataset.py
@@ -9,9 +9,7 @@
from typing import Collection
from typing import Dict
from typing import Mapping
-from typing import Optional
-from typing import Tuple
-from typing import Union
+from typing import Union, List, Tuple
import kaldiio
import numpy as np
@@ -145,7 +143,7 @@
def _build_loader(
self, path: str, loader_type: str
- ) -> Mapping[str, Union[np.ndarray, torch.Tensor, str, numbers.Number]]:
+ ) -> Mapping[str, Union[np.ndarray, torch.Tensor, str, List[int], numbers.Number]]:
"""Helper function to instantiate Loader.
Args:
@@ -175,6 +173,19 @@
raise RuntimeError(f"{k} is duplicated ({path}:{linenum})")
text_loader[k] = v
return text_loader
+ elif loader_type == "text_in":
+ text_in_loader = {}
+ with open(path, "r", encoding="utf-8") as f:
+ for linenum, line in enumerate(f, 1):
+ sps = line.rstrip().split(maxsplit=1)
+ if len(sps) == 1:
+ k, v = sps[0], ""
+ else:
+ k, v = sps
+ if k in text_in_loader:
+ raise RuntimeError(f"{k} is duplicated ({path}:{linenum})")
+ text_in_loader[k] = [int(i) for i in v.split()]
+ return text_in_loader
else:
raise RuntimeError(f"Not supported: loader_type={loader_type}")
--
Gitblit v1.9.1