From a2d6575d89c2ad6221d6ccc9ef148acad8c29b0c Mon Sep 17 00:00:00 2001
From: manyeyes <32889020+manyeyes@users.noreply.github.com>
Date: 星期四, 14 三月 2024 17:11:25 +0800
Subject: [PATCH] Improving FBank computation and implementing the dispose method (#1497)
---
/dev/null | 6 --
runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs | 63 ++++++++++++++-------
runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs | 67 ++++++++++------------
runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj | 5 -
runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs | 27 +++++---
5 files changed, 91 insertions(+), 77 deletions(-)
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs
index f42bfb1..672eac2 100644
--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs
+++ b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs
@@ -1,19 +1,16 @@
-锘縰sing System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Microsoft.ML;
+锘縰sing AliFsmnVadSharp.Model;
+using AliFsmnVadSharp.Utils;
+using Microsoft.Extensions.Logging;
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
-using Microsoft.Extensions.Logging;
-using AliFsmnVadSharp.Model;
-using AliFsmnVadSharp.Utils;
namespace AliFsmnVadSharp
{
- public class AliFsmnVad
+ public class AliFsmnVad : IDisposable
{
+ private bool _disposed;
private InferenceSession _onnxSession;
- private readonly ILogger<AliFsmnVad> _logger;
+ private readonly ILogger _logger;
private string _frontend;
private WavFrontend _wavFrontend;
private int _batchSize = 1;
@@ -23,7 +20,7 @@
public AliFsmnVad(string modelFilePath, string configFilePath, string mvnFilePath, int batchSize = 1)
{
- Microsoft.ML.OnnxRuntime.SessionOptions options = new Microsoft.ML.OnnxRuntime.SessionOptions();
+ SessionOptions options = new SessionOptions();
options.AppendExecutionProvider_CPU(0);
options.InterOpNumThreads = 1;
_onnxSession = new InferenceSession(modelFilePath, options);
@@ -371,17 +368,41 @@
}
return speech;
}
+ protected virtual void Dispose(bool disposing)
+ {
+ if (!_disposed)
+ {
+ if (disposing)
+ {
+ if (_onnxSession != null)
+ {
+ _onnxSession.Dispose();
+ }
+ if (_wavFrontend != null)
+ {
+ _wavFrontend.Dispose();
+ }
+ if (_encoderConfEntity != null)
+ {
+ _encoderConfEntity = null;
+ }
+ if (_vad_post_conf != null)
+ {
+ _vad_post_conf = null;
+ }
+ }
+ _disposed = true;
+ }
+ }
-
-
-
-
-
-
-
-
-
-
-
+ public void Dispose()
+ {
+ Dispose(disposing: true);
+ GC.SuppressFinalize(this);
+ }
+ ~AliFsmnVad()
+ {
+ Dispose(_disposed);
+ }
}
}
\ No newline at end of file
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj
index 4991517..769bf0c 100644
--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj
+++ b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj
@@ -7,16 +7,13 @@
</PropertyGroup>
<ItemGroup>
+ <PackageReference Include="KaldiNativeFbankSharp" Version="1.1.2" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.15.0" />
<PackageReference Include="YamlDotNet" Version="13.1.0" />
</ItemGroup>
<ItemGroup>
- <None Update="Lib\kaldi-native-fbank-dll.dll">
- <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
- <TargetPath>kaldi-native-fbank-dll.dll</TargetPath>
- </None>
<None Update="speech_fsmn_vad_zh-cn-16k-common-pytorch\example\0.wav">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs
deleted file mode 100644
index af0ad36..0000000
--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs
+++ /dev/null
@@ -1,40 +0,0 @@
-锘縰sing System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using System.Runtime.InteropServices;
-using AliFsmnVadSharp.Struct;
-
-namespace AliFsmnVadSharp.DLL
-{
- public static class KaldiNativeFbank
- {
- private const string dllName = @"kaldi-native-fbank-dll";
-
- [DllImport(dllName, EntryPoint = "GetFbankOptions", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
- internal static extern IntPtr GetFbankOptions(float dither, bool snip_edges, float sample_rate, int num_bins, float frame_shift = 10.0f, float frame_length = 25.0f, float energy_floor = 0.0f, bool debug_mel = false, string window_type = "hamming");
-
- [DllImport(dllName, EntryPoint = "GetOnlineFbank", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
- internal static extern KnfOnlineFbank GetOnlineFbank(IntPtr opts);
-
- [DllImport(dllName, EntryPoint = "AcceptWaveform", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
- internal static extern void AcceptWaveform(KnfOnlineFbank knfOnlineFbank, float sample_rate, float[] samples, int samples_size);
-
- [DllImport(dllName, EntryPoint = "InputFinished", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
- internal static extern void InputFinished(KnfOnlineFbank knfOnlineFbank);
-
- [DllImport(dllName, EntryPoint = "GetNumFramesReady", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
- internal static extern int GetNumFramesReady(KnfOnlineFbank knfOnlineFbank);
-
- [DllImport(dllName, EntryPoint = "AcceptWaveformxxx", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
- internal static extern FbankDatas AcceptWaveformxxx(KnfOnlineFbank knfOnlineFbank, float sample_rate, float[] samples, int samples_size);
-
- [DllImport(dllName, EntryPoint = "GetFbank", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
- internal static extern void GetFbank(KnfOnlineFbank knfOnlineFbank,int frame, ref FbankData pData);
-
- [DllImport(dllName, EntryPoint = "GetFbanks", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
- internal static extern void GetFbanks(KnfOnlineFbank knfOnlineFbank, int framesNum, ref FbankDatas fbankDatas);
-
- }
-}
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs
deleted file mode 100644
index 45549b2..0000000
--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs
+++ /dev/null
@@ -1,26 +0,0 @@
-锘縰sing System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Runtime.InteropServices;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace AliFsmnVadSharp.DLL
-{
- internal struct FbankData
- {
- public IntPtr data;
- public int data_length;
- };
-
- internal struct FbankDatas
- {
- public IntPtr data;
- public int data_length;
- };
-
- internal struct KnfOnlineFbank
- {
- public IntPtr impl;
- };
-}
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs
index ce519b1..38d3c90 100644
--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs
+++ b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs
@@ -503,20 +503,27 @@
{
return 0;
}
- for (int i = _vad_opts.nn_eval_block_size - 1; i > -1; i += -1)
+ try
{
- FrameState frame_state = FrameState.kFrameStateInvalid;
- frame_state = GetFrameState(_frm_cnt - 1 - i);
- if (i != 0)
+ for (int i = _vad_opts.nn_eval_block_size - 1; i > -1; i += -1)
{
- DetectOneFrame(frame_state, _frm_cnt - 1 - i, false);
- }
- else
- {
- DetectOneFrame(frame_state, _frm_cnt - 1, true);
- }
+ FrameState frame_state = FrameState.kFrameStateInvalid;
+ frame_state = GetFrameState(_frm_cnt - 1 - i);
+ if (i != 0)
+ {
+ DetectOneFrame(frame_state, _frm_cnt - 1 - i, false);
+ }
+ else
+ {
+ DetectOneFrame(frame_state, _frm_cnt - 1, true);
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ //
}
return 0;
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll
deleted file mode 100644
index cddc940..0000000
--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll
+++ /dev/null
Binary files differ
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs
deleted file mode 100644
index bbad3dc..0000000
--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs
+++ /dev/null
@@ -1,6 +0,0 @@
-锘縰sing System.Runtime.InteropServices;
-
-namespace AliFsmnVadSharp.Struct
-{
-
-}
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs
index 2c5b50f..6ffb79f 100644
--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs
+++ b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs
@@ -1,30 +1,23 @@
-锘縰sing System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using AliFsmnVadSharp.Model;
-using AliFsmnVadSharp.DLL;
-using AliFsmnVadSharp.Struct;
-using System.Runtime.InteropServices;
+锘縰sing AliFsmnVadSharp.Model;
+using KaldiNativeFbankSharp;
namespace AliFsmnVadSharp
{
- internal class WavFrontend
+ internal class WavFrontend : IDisposable
{
- private string _mvnFilePath;
+ private bool _disposed;
+
private FrontendConfEntity _frontendConfEntity;
- IntPtr _opts = IntPtr.Zero;
+ OnlineFbank _onlineFbank;
private CmvnEntity _cmvnEntity;
private static int _fbank_beg_idx = 0;
public WavFrontend(string mvnFilePath, FrontendConfEntity frontendConfEntity)
{
- _mvnFilePath = mvnFilePath;
_frontendConfEntity = frontendConfEntity;
_fbank_beg_idx = 0;
- _opts = KaldiNativeFbank.GetFbankOptions(
+ _onlineFbank = new OnlineFbank(
dither: _frontendConfEntity.dither,
snip_edges: true,
sample_rate: _frontendConfEntity.fs,
@@ -37,28 +30,7 @@
{
float sample_rate = _frontendConfEntity.fs;
samples = samples.Select((float x) => x * 32768f).ToArray();
- // method1
- //FbankDatas fbankDatas = new FbankDatas();
- //KaldiNativeFbank.GetFbanks(_knfOnlineFbank, framesNum,ref fbankDatas);
- // method2
- KnfOnlineFbank _knfOnlineFbank = KaldiNativeFbank.GetOnlineFbank(_opts);
- KaldiNativeFbank.AcceptWaveform(_knfOnlineFbank, sample_rate, samples, samples.Length);
- KaldiNativeFbank.InputFinished(_knfOnlineFbank);
- int framesNum = KaldiNativeFbank.GetNumFramesReady(_knfOnlineFbank);
- float[] fbanks = new float[framesNum * 80];
- for (int i = 0; i < framesNum; i++)
- {
- FbankData fbankData = new FbankData();
- KaldiNativeFbank.GetFbank(_knfOnlineFbank, i, ref fbankData);
- float[] _fbankData = new float[fbankData.data_length];
- Marshal.Copy(fbankData.data, _fbankData, 0, fbankData.data_length);
- Array.Copy(_fbankData, 0, fbanks, i * 80, _fbankData.Length);
- fbankData.data = IntPtr.Zero;
- _fbankData = null;
- }
-
- samples = null;
- GC.Collect();
+ float[] fbanks = _onlineFbank.GetFbank(samples);
return fbanks;
}
@@ -180,6 +152,29 @@
cmvnEntity.Vars = vars_list;
return cmvnEntity;
}
+ protected virtual void Dispose(bool disposing)
+ {
+ if (!_disposed)
+ {
+ if (disposing)
+ {
+ if (_onlineFbank != null)
+ {
+ _onlineFbank.Dispose();
+ }
+ }
+ _disposed = true;
+ }
+ }
+ public void Dispose()
+ {
+ Dispose(disposing: true);
+ GC.SuppressFinalize(this);
+ }
+ ~WavFrontend()
+ {
+ Dispose(_disposed);
+ }
}
}
--
Gitblit v1.9.1