From 9b4e9cc8a0311e5243d69b73ed073e7ea441982e Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 27 三月 2024 16:05:29 +0800
Subject: [PATCH] train update
---
funasr/download/download_from_hub.py | 36 ++++++++++++++++++++++++------------
1 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/funasr/download/download_from_hub.py b/funasr/download/download_from_hub.py
index 4f07daa..8255947 100644
--- a/funasr/download/download_from_hub.py
+++ b/funasr/download/download_from_hub.py
@@ -1,6 +1,6 @@
import os
import json
-from omegaconf import OmegaConf
+from omegaconf import OmegaConf, DictConfig
from funasr.download.name_maps_from_hub import name_maps_ms, name_maps_hf, name_maps_openai
@@ -13,18 +13,24 @@
pass
elif hub == "openai":
model_or_path = kwargs.get("model")
- if model_or_path in name_maps_openai:
- model_or_path = name_maps_openai[model_or_path]
- kwargs["model_path"] = model_or_path
-
+ if os.path.exists(model_or_path):
+ # local path
+ kwargs["model_path"] = model_or_path
+ kwargs["model"] = "WhisperWarp"
+ else:
+ # model name
+ if model_or_path in name_maps_openai:
+ model_or_path = name_maps_openai[model_or_path]
+ kwargs["model_path"] = model_or_path
+
return kwargs
def download_from_ms(**kwargs):
model_or_path = kwargs.get("model")
if model_or_path in name_maps_ms:
model_or_path = name_maps_ms[model_or_path]
- model_revision = kwargs.get("model_revision")
- if not os.path.exists(model_or_path):
+ model_revision = kwargs.get("model_revision", "master")
+ if not os.path.exists(model_or_path) and "model_path" not in kwargs:
try:
model_or_path = get_or_download_model_dir(model_or_path, model_revision,
is_training=kwargs.get("is_training"),
@@ -32,7 +38,7 @@
except Exception as e:
print(f"Download: {model_or_path} failed!: {e}")
- kwargs["model_path"] = model_or_path
+ kwargs["model_path"] = model_or_path if "model_path" not in kwargs else kwargs["model_path"]
if os.path.exists(os.path.join(model_or_path, "configuration.json")):
with open(os.path.join(model_or_path, "configuration.json"), 'r', encoding='utf-8') as f:
@@ -42,9 +48,10 @@
if "file_path_metas" in conf_json:
add_file_root_path(model_or_path, conf_json["file_path_metas"], cfg)
cfg.update(kwargs)
- config = OmegaConf.load(cfg["config"])
- kwargs = OmegaConf.merge(config, cfg)
- kwargs["model"] = config["model"]
+ if "config" in cfg:
+ config = OmegaConf.load(cfg["config"])
+ kwargs = OmegaConf.merge(config, cfg)
+ kwargs["model"] = config["model"]
elif os.path.exists(os.path.join(model_or_path, "config.yaml")) and os.path.exists(os.path.join(model_or_path, "model.pt")):
config = OmegaConf.load(os.path.join(model_or_path, "config.yaml"))
kwargs = OmegaConf.merge(config, kwargs)
@@ -63,8 +70,13 @@
kwargs["frontend_conf"]["cmvn_file"] = os.path.join(model_or_path, "am.mvn")
if os.path.exists(os.path.join(model_or_path, "jieba_usr_dict")):
kwargs["jieba_usr_dict"] = os.path.join(model_or_path, "jieba_usr_dict")
- if isinstance(kwargs, OmegaConf):
+ if isinstance(kwargs, DictConfig):
kwargs = OmegaConf.to_container(kwargs, resolve=True)
+ if os.path.exists(os.path.join(model_or_path, "requirements.txt")):
+ requirements = os.path.join(model_or_path, "requirements.txt")
+ print(f"Detect model requirements, begin to install it: {requirements}")
+ from funasr.utils.install_model_requirements import install_requirements
+ install_requirements(requirements)
return kwargs
def add_file_root_path(model_or_path: str, file_path_metas: dict, cfg = {}):
--
Gitblit v1.9.1