| | |
| | | import logging |
| | | import os |
| | | import json |
| | | from omegaconf import OmegaConf, DictConfig |
| | |
| | | |
| | | def download_model(**kwargs): |
| | | hub = kwargs.get("hub", "ms") |
| | | if hub == "ms": |
| | | if hub == "ms" or hub == "modelscope": |
| | | kwargs = download_from_ms(**kwargs) |
| | | elif hub == "hf": |
| | | elif hub == "hf" or hub == "huggingface": |
| | | kwargs = download_from_hf(**kwargs) |
| | | elif hub == "openai": |
| | | model_or_path = kwargs.get("model") |
| | |
| | | print(f"Download: {model_or_path} failed!: {e}") |
| | | |
| | | kwargs["model_path"] = model_or_path if "model_path" not in kwargs else kwargs["model_path"] |
| | | |
| | | model_or_path = kwargs["model_path"] |
| | | if os.path.exists(os.path.join(model_or_path, "configuration.json")): |
| | | with open(os.path.join(model_or_path, "configuration.json"), "r", encoding="utf-8") as f: |
| | | conf_json = json.load(f) |
| | |
| | | kwargs["jieba_usr_dict"] = os.path.join(model_or_path, "jieba_usr_dict") |
| | | if isinstance(kwargs, DictConfig): |
| | | kwargs = OmegaConf.to_container(kwargs, resolve=True) |
| | | if os.path.exists(os.path.join(model_or_path, "requirements.txt")): |
| | | logging.warning(f'trust_remote_code: {kwargs.get("trust_remote_code", False)}') |
| | | if os.path.exists(os.path.join(model_or_path, "requirements.txt")) and kwargs.get( |
| | | "trust_remote_code", False |
| | | ): |
| | | requirements = os.path.join(model_or_path, "requirements.txt") |
| | | print(f"Detect model requirements, begin to install it: {requirements}") |
| | | from funasr.utils.install_model_requirements import install_requirements |
| | |
| | | if isinstance(file_path_metas, dict): |
| | | if isinstance(cfg, list): |
| | | cfg.append({}) |
| | | |
| | | for k, v in file_path_metas.items(): |
| | | if isinstance(v, str): |
| | | p = os.path.join(model_or_path, v) |
| | |
| | | if k not in cfg: |
| | | cfg[k] = [] |
| | | if isinstance(vv, str): |
| | | p = os.path.join(model_or_path, v) |
| | | file_path_metas[i] = p |
| | | p = os.path.join(model_or_path, vv) |
| | | # file_path_metas[i] = p |
| | | if os.path.exists(p): |
| | | if isinstance(cfg[k], dict): |
| | | cfg[k] = p |