Merge pull request #1393 from alibaba-damo-academy/dev_gzf

Shi Xian

2024-02-27 ba589e05c1448d0487198f8603cba247f22d67e1

 funasr/auto/auto_model.py

@@ -162,8 +162,10 @@
            tokenizer_class = tables.tokenizer_classes.get(tokenizer)
            tokenizer = tokenizer_class(**kwargs["tokenizer_conf"])
            kwargs["tokenizer"] = tokenizer
            kwargs["token_list"] = tokenizer.token_list
            vocab_size = len(tokenizer.token_list)

            kwargs["token_list"] = tokenizer.token_list if hasattr(tokenizer, "token_list") else None
            kwargs["token_list"] = tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else kwargs["token_list"]
            vocab_size = len(kwargs["token_list"])
        else:
            vocab_size = -1
        
@@ -184,6 +186,7 @@
        # init_param
        init_param = kwargs.get("init_param", None)
        if init_param is not None:
            if os.path.exists(init_param):
            logging.info(f"Loading pretrained params from {init_param}")
            load_pretrained_model(
                model=model,
@@ -193,6 +196,8 @@
                scope_map=kwargs.get("scope_map", None),
                excludes=kwargs.get("excludes", None),
            )
            else:
                print(f"error, init_param does not exist!: {init_param}")
        
        return model, kwargs