From 55c09aeaa25b4bb88a50e09ba68fa6ff00a6d676 Mon Sep 17 00:00:00 2001
From: shixian.shi <shixian.shi@alibaba-inc.com>
Date: 星期一, 15 一月 2024 20:10:39 +0800
Subject: [PATCH] update readme, fix seaco bug
---
funasr/download/download_from_hub.py | 58 ++++++++++++++++++++++++++++++++++++++--------------------
1 files changed, 38 insertions(+), 20 deletions(-)
diff --git a/funasr/download/download_from_hub.py b/funasr/download/download_from_hub.py
index abf3ba0..57e8c41 100644
--- a/funasr/download/download_from_hub.py
+++ b/funasr/download/download_from_hub.py
@@ -7,24 +7,24 @@
def download_model(**kwargs):
model_hub = kwargs.get("model_hub", "ms")
if model_hub == "ms":
- kwargs = download_fr_ms(**kwargs)
+ kwargs = download_from_ms(**kwargs)
return kwargs
-def download_fr_ms(**kwargs):
+def download_from_ms(**kwargs):
model_or_path = kwargs.get("model")
if model_or_path in name_maps_ms:
model_or_path = name_maps_ms[model_or_path]
model_revision = kwargs.get("model_revision")
if not os.path.exists(model_or_path):
- model_or_path = get_or_download_model_dir(model_or_path, model_revision, is_training=kwargs.get("is_training"))
+ model_or_path = get_or_download_model_dir(model_or_path, model_revision, is_training=kwargs.get("is_training"), check_latest=kwargs.get("kwargs", True))
+ kwargs["model_path"] = model_or_path
config = os.path.join(model_or_path, "config.yaml")
if os.path.exists(config) and os.path.exists(os.path.join(model_or_path, "model.pb")):
- # config = os.path.join(model_or_path, "config.yaml")
- # assert os.path.exists(config), "{} is not exist!".format(config)
- cfg = OmegaConf.load(config)
- kwargs = OmegaConf.merge(cfg, kwargs)
+
+ config = OmegaConf.load(config)
+ kwargs = OmegaConf.merge(config, kwargs)
init_param = os.path.join(model_or_path, "model.pb")
kwargs["init_param"] = init_param
if os.path.exists(os.path.join(model_or_path, "tokens.txt")):
@@ -35,25 +35,43 @@
kwargs["tokenizer_conf"]["seg_dict"] = os.path.join(model_or_path, "seg_dict")
if os.path.exists(os.path.join(model_or_path, "bpe.model")):
kwargs["tokenizer_conf"]["bpemodel"] = os.path.join(model_or_path, "bpe.model")
- kwargs["model"] = cfg["model"]
+ kwargs["model"] = config["model"]
if os.path.exists(os.path.join(model_or_path, "am.mvn")):
kwargs["frontend_conf"]["cmvn_file"] = os.path.join(model_or_path, "am.mvn")
- else:# configuration.json
- assert os.path.exists(os.path.join(model_or_path, "configuration.json"))
+ if os.path.exists(os.path.join(model_or_path, "jieba_usr_dict")):
+ kwargs["jieba_usr_dict"] = os.path.join(model_or_path, "jieba_usr_dict")
+ elif os.path.exists(os.path.join(model_or_path, "configuration.json")):
with open(os.path.join(model_or_path, "configuration.json"), 'r', encoding='utf-8') as f:
conf_json = json.load(f)
- config = os.path.join(model_or_path, conf_json["model"]["model_config"])
- cfg = OmegaConf.load(config)
- kwargs = OmegaConf.merge(cfg, kwargs)
- init_param = os.path.join(model_or_path, conf_json["model"]["model_name"])
- kwargs["init_param"] = init_param
- kwargs["model"] = cfg["model"]
+ cfg = {}
+ add_file_root_path(model_or_path, conf_json["file_path_metas"], cfg)
+ cfg.update(kwargs)
+ config = OmegaConf.load(cfg["config"])
+ kwargs = OmegaConf.merge(config, cfg)
+ kwargs["model"] = config["model"]
return OmegaConf.to_container(kwargs, resolve=True)
+def add_file_root_path(model_or_path: str, file_path_metas: dict, cfg = {}):
+
+ if isinstance(file_path_metas, dict):
+ for k, v in file_path_metas.items():
+ if isinstance(v, str):
+ p = os.path.join(model_or_path, v)
+ if os.path.exists(p):
+ cfg[k] = p
+ elif isinstance(v, dict):
+ if k not in cfg:
+ cfg[k] = {}
+ return add_file_root_path(model_or_path, v, cfg[k])
+
+ return cfg
+
+
def get_or_download_model_dir(
- model,
- model_revision=None,
- is_training=False,
+ model,
+ model_revision=None,
+ is_training=False,
+ check_latest=True,
):
""" Get local model directory or download model if necessary.
@@ -69,7 +87,7 @@
key = Invoke.LOCAL_TRAINER if is_training else Invoke.PIPELINE
- if os.path.exists(model):
+ if os.path.exists(model) and check_latest:
model_cache_dir = model if os.path.isdir(
model) else os.path.dirname(model)
try:
--
Gitblit v1.9.1