| | |
|---|
| | | from funasr.metrics.compute_acc import th_accuracy |
|---|
| | | # from funasr.models.e2e_asr_common import ErrorCalculator |
|---|
| | | from funasr.train_utils.device_funcs import force_gatherable |
|---|
| | | from funasr.datasets.audio_datasets.load_audio_extract_fbank import load_audio, extract_fbank |
|---|
| | | from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank |
|---|
| | | from funasr.utils import postprocess_utils |
|---|
| | | from funasr.utils.datadir_writer import DatadirWriter |
|---|
| | | from funasr.utils.register import register_class, registry_tables |
|---|
| | | from funasr.register import tables |
|---|
| | | |
|---|
| | | @register_class("model_classes", "Transformer") |
|---|
| | | @tables.register("model_classes", "Transformer") |
|---|
| | | class Transformer(nn.Module): |
|---|
| | | """CTC-attention hybrid Encoder-Decoder model""" |
|---|
| | | |
|---|
| | |
|---|
| | | super().__init__() |
|---|
| | | |
|---|
| | | if frontend is not None: |
|---|
| | | frontend_class = registry_tables.frontend_classes.get_class(frontend.lower()) |
|---|
| | | frontend_class = tables.frontend_classes.get_class(frontend.lower()) |
|---|
| | | frontend = frontend_class(**frontend_conf) |
|---|
| | | if specaug is not None: |
|---|
| | | specaug_class = registry_tables.specaug_classes.get_class(specaug.lower()) |
|---|
| | | specaug_class = tables.specaug_classes.get_class(specaug.lower()) |
|---|
| | | specaug = specaug_class(**specaug_conf) |
|---|
| | | if normalize is not None: |
|---|
| | | normalize_class = registry_tables.normalize_classes.get_class(normalize.lower()) |
|---|
| | | normalize_class = tables.normalize_classes.get_class(normalize.lower()) |
|---|
| | | normalize = normalize_class(**normalize_conf) |
|---|
| | | encoder_class = registry_tables.encoder_classes.get_class(encoder.lower()) |
|---|
| | | encoder_class = tables.encoder_classes.get_class(encoder.lower()) |
|---|
| | | encoder = encoder_class(input_size=input_size, **encoder_conf) |
|---|
| | | encoder_output_size = encoder.output_size() |
|---|
| | | if decoder is not None: |
|---|
| | | decoder_class = registry_tables.decoder_classes.get_class(decoder.lower()) |
|---|
| | | decoder_class = tables.decoder_classes.get_class(decoder.lower()) |
|---|
| | | decoder = decoder_class( |
|---|
| | | vocab_size=vocab_size, |
|---|
| | | encoder_output_size=encoder_output_size, |
|---|
| | |
|---|
| | | meta_data = {} |
|---|
| | | # extract fbank feats |
|---|
| | | time1 = time.perf_counter() |
|---|
| | | audio_sample_list = load_audio(data_in, fs=self.frontend.fs, audio_fs=kwargs.get("fs", 16000)) |
|---|
| | | audio_sample_list = load_audio_text_image_video(data_in, fs=self.frontend.fs, audio_fs=kwargs.get("fs", 16000)) |
|---|
| | | time2 = time.perf_counter() |
|---|
| | | meta_data["load_data"] = f"{time2 - time1:0.3f}" |
|---|
| | | speech, speech_lengths = extract_fbank(audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=self.frontend) |
|---|
