EMO_UNK禁用和Merge VAD修复 (#1940)

gaochangfeng

2024-07-22 340c55838b56bb98508337f8404e0a74f22a20c2

EMO_UNK禁用和Merge VAD修复 (#1940)

* 添加富文本解码约束

* special token

* bug fix

* fix

* 增加unk score的参数

* emobaned

* kwargs2cfg

* merge_vad bug fix

---------

Co-authored-by: 常材 <gaochangfeng.gcf@alibaba-inc.com>

 funasr/auto/auto_model.py

@@ -337,7 +337,7 @@
        end_vad = time.time()

        #  FIX(gcf): concat the vad clips for sense vocie model for better aed
        if kwargs.get("merge_vad", False):
        if cfg.get("merge_vad", False):
            for i in range(len(res)):
                res[i]["value"] = merge_vad(
                    res[i]["value"], kwargs.get("merge_length_s", 15) * 1000

 funasr/models/sense_voice/model.py

@@ -644,6 +644,7 @@
        self.embed = torch.nn.Embedding(
            7 + len(self.lid_dict) + len(self.textnorm_dict), input_size
        )
        self.emo_dict = {"unk": 25009, "happy": 25001, "sad": 25002, "angry": 25003, "neutral": 25004}

        self.criterion_att = LabelSmoothingLoss(
            size=self.vocab_size,
@@ -870,6 +871,8 @@

        # c. Passed the encoder result and the beam search
        ctc_logits = self.ctc.log_softmax(encoder_out)
        if kwargs.get("ban_emo_unk", False):
            ctc_logits[:, :, self.emo_dict["unk"]] = -float("inf")

        results = []
        b, n, d = encoder_out.size()

 funasr/utils/vad_utils.py

@@ -32,8 +32,10 @@
    return speech_list, speech_lengths_list


def merge_vad(vad_result, max_length=15000):
def merge_vad(vad_result, max_length=15000, min_length=0):
    new_result = []
    if len(vad_result) <= 1:
        return vad_result
    time_step = [t[0] for t in vad_result] + [t[1] for t in vad_result]
    time_step = sorted(list(set(time_step)))
    if len(time_step) == 0:
@@ -43,13 +45,15 @@
        time = time_step[i]
        if time_step[i + 1] - bg < max_length:
            continue
        if time - bg < max_length * 1.5:
        if time - bg > min_length:
            new_result.append([bg, time])
        else:
            split_num = int(time - bg) // max_length + 1
            spl_l = int(time - bg) // split_num
            for j in range(split_num):
                new_result.append([bg + j * spl_l, bg + (j + 1) * spl_l])
        # if time - bg < max_length * 1.5:
        #     new_result.append([bg, time])
        # else:
        #     split_num = int(time - bg) // max_length + 1
        #     spl_l = int(time - bg) // split_num
        #     for j in range(split_num):
        #         new_result.append([bg + j * spl_l, bg + (j + 1) * spl_l])
        bg = time
    new_result.append([bg, time_step[-1]])
    return new_result