Improving FBank computation and implementing the dispose method (#1497)

manyeyes

2024-03-14 a2d6575d89c2ad6221d6ccc9ef148acad8c29b0c

Improving FBank computation and implementing the dispose method (#1497)

 runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs

@@ -1,19 +1,16 @@
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.ML;
using AliFsmnVadSharp.Model;
using AliFsmnVadSharp.Utils;
using Microsoft.Extensions.Logging;
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using Microsoft.Extensions.Logging;
using AliFsmnVadSharp.Model;
using AliFsmnVadSharp.Utils;

namespace AliFsmnVadSharp
{
    public class AliFsmnVad
    public class AliFsmnVad : IDisposable
    {
        private bool _disposed;
        private InferenceSession _onnxSession;
        private readonly ILogger<AliFsmnVad> _logger;
        private readonly ILogger _logger;
        private string _frontend;
        private WavFrontend _wavFrontend;
        private int _batchSize = 1;
@@ -23,7 +20,7 @@

        public AliFsmnVad(string modelFilePath, string configFilePath, string mvnFilePath, int batchSize = 1)
        {
            Microsoft.ML.OnnxRuntime.SessionOptions options = new Microsoft.ML.OnnxRuntime.SessionOptions();
            SessionOptions options = new SessionOptions();
            options.AppendExecutionProvider_CPU(0);
            options.InterOpNumThreads = 1;
            _onnxSession = new InferenceSession(modelFilePath, options);
@@ -371,17 +368,41 @@
            }
            return speech;
        }
        protected virtual void Dispose(bool disposing)
        {
            if (!_disposed)
            {
                if (disposing)
                {
                    if (_onnxSession != null)
                    {
                        _onnxSession.Dispose();
                    }
                    if (_wavFrontend != null)
                    {
                        _wavFrontend.Dispose();
                    }
                    if (_encoderConfEntity != null)
                    {
                        _encoderConfEntity = null;
                    }
                    if (_vad_post_conf != null)
                    {
                        _vad_post_conf = null;
                    }
                }
                _disposed = true;
            }
        }












        public void Dispose()
        {
            Dispose(disposing: true);
            GC.SuppressFinalize(this);
        }
        ~AliFsmnVad()
        {
            Dispose(_disposed);
        }
    }
}

 runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj

@@ -7,16 +7,13 @@
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="KaldiNativeFbankSharp" Version="1.1.2" />
    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
    <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.15.0" />
    <PackageReference Include="YamlDotNet" Version="13.1.0" />
  </ItemGroup>

  <ItemGroup>
    <None Update="Lib\kaldi-native-fbank-dll.dll">
      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
            <TargetPath>kaldi-native-fbank-dll.dll</TargetPath>
    </None>
    <None Update="speech_fsmn_vad_zh-cn-16k-common-pytorch\example\0.wav">
      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
    </None>

 runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs

File was deleted

 runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs

File was deleted

 runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs

@@ -503,6 +503,8 @@
            {
                return 0;
            }
            try
            {
            for (int i = _vad_opts.nn_eval_block_size - 1; i > -1; i += -1)
            {
                FrameState frame_state = FrameState.kFrameStateInvalid;
@@ -518,6 +520,11 @@


            }
            }
            catch (Exception e)
            {
                //
            }

            return 0;
        }

 runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll

Binary files differ

 runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs

File was deleted

 runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs

@@ -1,30 +1,23 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using AliFsmnVadSharp.Model;
using AliFsmnVadSharp.DLL;
using AliFsmnVadSharp.Struct;
using System.Runtime.InteropServices;
using AliFsmnVadSharp.Model;
using KaldiNativeFbankSharp;

namespace AliFsmnVadSharp
{
    internal class WavFrontend
    internal class WavFrontend : IDisposable
    {
        private string _mvnFilePath;
        private bool _disposed;

        private FrontendConfEntity _frontendConfEntity;
        IntPtr _opts = IntPtr.Zero;
        OnlineFbank _onlineFbank;
        private CmvnEntity _cmvnEntity;

        private static int _fbank_beg_idx = 0;

        public WavFrontend(string mvnFilePath, FrontendConfEntity frontendConfEntity)
        {
            _mvnFilePath = mvnFilePath;
            _frontendConfEntity = frontendConfEntity;
            _fbank_beg_idx = 0;
            _opts = KaldiNativeFbank.GetFbankOptions(
            _onlineFbank = new OnlineFbank(
                dither: _frontendConfEntity.dither,
                snip_edges: true,
                sample_rate: _frontendConfEntity.fs,
@@ -37,28 +30,7 @@
        {
            float sample_rate = _frontendConfEntity.fs;
            samples = samples.Select((float x) => x * 32768f).ToArray();
            // method1
            //FbankDatas fbankDatas = new FbankDatas();
            //KaldiNativeFbank.GetFbanks(_knfOnlineFbank, framesNum,ref fbankDatas);
            // method2
            KnfOnlineFbank _knfOnlineFbank = KaldiNativeFbank.GetOnlineFbank(_opts);
            KaldiNativeFbank.AcceptWaveform(_knfOnlineFbank, sample_rate, samples, samples.Length);
            KaldiNativeFbank.InputFinished(_knfOnlineFbank);
            int framesNum = KaldiNativeFbank.GetNumFramesReady(_knfOnlineFbank);
            float[] fbanks = new float[framesNum * 80];
            for (int i = 0; i < framesNum; i++)
            {
                FbankData fbankData = new FbankData();
                KaldiNativeFbank.GetFbank(_knfOnlineFbank, i, ref fbankData);
                float[] _fbankData = new float[fbankData.data_length];
                Marshal.Copy(fbankData.data, _fbankData, 0, fbankData.data_length);
                Array.Copy(_fbankData, 0, fbanks, i * 80, _fbankData.Length);
                fbankData.data = IntPtr.Zero;
                _fbankData = null;
            }

            samples = null;
            GC.Collect();
            float[] fbanks = _onlineFbank.GetFbank(samples);
            return fbanks;
        }

@@ -180,6 +152,29 @@
            cmvnEntity.Vars = vars_list;
            return cmvnEntity;
        }
        protected virtual void Dispose(bool disposing)
        {
            if (!_disposed)
            {
                if (disposing)
                {
                    if (_onlineFbank != null)
                    {
                        _onlineFbank.Dispose();
                    }
                }
                _disposed = true;
            }
        }

        public void Dispose()
        {
            Dispose(disposing: true);
            GC.SuppressFinalize(this);
        }
        ~WavFrontend()
        {
            Dispose(_disposed);
        }
    }
}