Dev gzf (#1475)

zhifu gao

2024-03-12 68f0603b10d13b8b8e6e090324c7e24e009df90b

Dev gzf (#1475)

* qwenaudio qwenaudiochat

* qwenaudio qwenaudiochat

* whisper

* whisper

* llm

* llm

* llm

* llm

* llm

* llm

* llm

* llm

* export onnx

* export onnx

* export onnx

* dingding

* dingding

* llm

* doc

* onnx

* onnx

* onnx

* onnx

* onnx

* onnx

* v1.0.15

* qwenaudio

* qwenaudio

* issue doc

 .github/ISSUE_TEMPLATE/ask_questions.md

@@ -4,8 +4,12 @@
labels: 'question, needs triage'
---

Notice: In order to resolve issues more efficiently, please raise issue following the template.
（注意：为了更加高效率解决您遇到的问题，请按照模板提问，补充细节）

## ❓ Questions and Help


### Before asking:
1. search the issues.
2. search the docs.

 .github/ISSUE_TEMPLATE/bug_report.md

@@ -4,6 +4,9 @@
labels: 'bug, needs triage'
---

Notice: In order to resolve issues more efficiently, please raise issue following the template.
（注意：为了更加高效率解决您遇到的问题，请按照模板提问，补充细节）

## 🐛 Bug

<!-- A clear and concise description of what the bug is. -->

 examples/industrial_data_pretraining/llm_asr/conf/whisper_qwen_linear.yaml

@@ -12,17 +12,16 @@
    length_normalized_loss: true

# encoder
audio_encoder: iic/Whisper-large-v2 #iic/Whisper-large-v3
audio_encoder: "/nfs/zhifu.gzf/init_model/Whisper-large-v3" #iic/Whisper-large-v3
audio_encoder_conf:
    hub: ms
    freeze: true
    init_param_path: "/nfs/maziyang.mzy/models/vicuna-7b-v1.5"

llm: Vicuna
llm: Qwen1.5-7b-chat
llm_conf:
  hub: hf
  freeze: true
  init_param_path: "/nfs/maziyang.mzy/models/vicuna-7b-v1.5"
  init_param_path: "/nfs/zhifu.gzf/init_model/qwen/Qwen1___5-7B-Chat"

audio_adaptor: Linear
audio_adaptor_conf:
@@ -34,7 +33,7 @@
frontend: WhisperFrontend
frontend_conf:
    fs: 16000
    whisper_model: large-v2
    whisper_model: large-v3
    do_pad_trim: true
    permute: true # true: [bs, frames, dims]; false: [bs, dims, frames]

@@ -66,8 +65,9 @@
optim: adamw
optim_conf:
   lr: 0.0001
   weight_decay: 0.000001
scheduler: warmuplr
   weight_decay: 0.000000

scheduler: custom_lambdalr
scheduler_conf:
   warmup_steps: 1000

@@ -83,12 +83,12 @@
    preprocessor_text: TextPreprocessRemovePunctuation
    audio_adaptor_downsample_rate: ${audio_adaptor_conf.downsample_rate}
    audio_encoder_downsample_rate: 2
    prompt: "<|startoftranscription|><|zh|><|transcribe|><|zh|><|notimestamps|><|wo_itn|>"
#    prompt: "<|startoftranscription|><|zh|><|transcribe|><|zh|><|notimestamps|><|wo_itn|>"



tokenizer: HuggingfaceTokenizer
tokenizer_conf:
  unk_symbol: <unk>
  init_param_path: "/nfs/maziyang.mzy/models/vicuna-7b-v1.5"
  init_param_path: "/nfs/zhifu.gzf/init_model/qwen/Qwen1___5-7B-Chat"


 funasr/datasets/audio_datasets/index_ds.py

@@ -99,7 +99,8 @@
                    target = data["target"]
                    source_len = data.get("source_len", 1)
                    target_len = data.get("target_len", 0)
                    
                    if "aishell" in source:
                        target = target.replace(" ", "")
                    contents.append({"source": source,
                                     "prompt": prompt,
                                     "target": target,