feat(voice): add iOS VAD runtime support
This commit is contained in:
Edison Jwa
@@ -0,0 +1,275 @@
|
||||
//! Voice activity detection backends and helpers.
|
||||
//!
|
||||
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into
|
||||
//! this module. The production path prefers a model-backed detector when
|
||||
//! available, and otherwise uses the realtime-safe fallback below so
|
||||
//! VoiceActivity mode never collapses back to Continuous transmit.
|
||||
|
||||
pub mod resampler;
|
||||
pub mod silero_onnx;
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
|
||||
use crate::frame::{dbfs, i16_to_f32};
|
||||
use crate::AudioError;
|
||||
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||
|
||||
pub use silero_onnx::SileroOnnxVad;
|
||||
|
||||
/// Voice activity detector output for one 10 ms frame.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct VadOutput {
|
||||
/// Speech confidence in the inclusive range `[0.0, 1.0]`.
|
||||
pub probability: f32,
|
||||
/// Immediate detector speech decision before hangover/min-duration state.
|
||||
pub speech: bool,
|
||||
}
|
||||
|
||||
/// Realtime-safe detector that consumes one 10 ms f32 mono frame.
|
||||
pub trait VoiceActivityDetector: Send {
|
||||
/// Process one 10 ms frame and return speech probability/state.
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput;
|
||||
}
|
||||
|
||||
/// Realtime-safe fallback VAD used when a model runtime is unavailable.
|
||||
///
|
||||
/// This is not an energy-only transmit gate. It combines RMS level,
|
||||
/// zero-crossing rate, and peak-to-RMS shape with hysteresis so stable
|
||||
/// background rumble is less likely to open VoiceActivity than speech.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WebRtcFallbackVad {
|
||||
open_dbfs: f32,
|
||||
close_dbfs: f32,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl Default for WebRtcFallbackVad {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
open_dbfs: -42.0,
|
||||
close_dbfs: -50.0,
|
||||
active: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WebRtcFallbackVad {
|
||||
fn zero_crossing_rate(samples: &[f32]) -> f32 {
|
||||
if samples.len() < 2 {
|
||||
return 0.0;
|
||||
}
|
||||
let crossings = samples
|
||||
.windows(2)
|
||||
.filter(|pair| (pair[0] >= 0.0 && pair[1] < 0.0) || (pair[0] < 0.0 && pair[1] >= 0.0))
|
||||
.count();
|
||||
crossings as f32 / (samples.len() - 1) as f32
|
||||
}
|
||||
|
||||
fn peak_to_rms(samples: &[f32], rms: f32) -> f32 {
|
||||
if rms <= 0.000_001 {
|
||||
return 0.0;
|
||||
}
|
||||
let peak = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
|
||||
peak / rms
|
||||
}
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for WebRtcFallbackVad {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
let level = dbfs(samples);
|
||||
let threshold = if self.active {
|
||||
self.close_dbfs
|
||||
} else {
|
||||
self.open_dbfs
|
||||
};
|
||||
let rms = samples.iter().map(|s| s * s).sum::<f32>() / samples.len().max(1) as f32;
|
||||
let rms = rms.sqrt();
|
||||
let zcr = Self::zero_crossing_rate(samples);
|
||||
let crest = Self::peak_to_rms(samples, rms);
|
||||
|
||||
// Level score: steeper curve so silence (-50 dBFS) scores near 0.
|
||||
// Speech is typically -30 to -10 dBFS; silence is -60 to -45 dBFS.
|
||||
// Map [-60, -20] → [0, 1] with a midpoint at -40 dBFS.
|
||||
let level_score = ((level + 60.0) / 40.0).clamp(0.0, 1.0);
|
||||
|
||||
let zcr_score = if (0.015..=0.32).contains(&zcr) {
|
||||
1.0
|
||||
} else {
|
||||
0.3 // penalise non-speech ZCR more aggressively
|
||||
};
|
||||
let crest_score = if (1.5..=12.0).contains(&crest) {
|
||||
1.0
|
||||
} else {
|
||||
0.3
|
||||
};
|
||||
let probability =
|
||||
(level_score * 0.72 + zcr_score * 0.18 + crest_score * 0.10).clamp(0.0, 1.0);
|
||||
self.active = level >= threshold && probability >= 0.5;
|
||||
VadOutput {
|
||||
probability,
|
||||
speech: self.active,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wraps any `VoiceActivityDetector` that operates at 16 kHz and
|
||||
/// downsamples 48 kHz input before forwarding.
|
||||
pub struct Resampled16kHzVad<D: VoiceActivityDetector> {
|
||||
inner: D,
|
||||
downsampler: Downsampler48to16,
|
||||
}
|
||||
|
||||
impl<D: VoiceActivityDetector> Resampled16kHzVad<D> {
|
||||
/// Wrap a 16 kHz detector so it can consume 48 kHz frames.
|
||||
pub fn new(inner: D) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
downsampler: Downsampler48to16::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<D: VoiceActivityDetector> VoiceActivityDetector for Resampled16kHzVad<D> {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
debug_assert_eq!(samples.len(), INPUT_FRAME_10MS);
|
||||
let mut input = [0.0_f32; INPUT_FRAME_10MS];
|
||||
input.copy_from_slice(samples);
|
||||
let downsampled = self.downsampler.process_frame_10ms(&input);
|
||||
self.inner.process_10ms(&downsampled)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one 48 kHz i16 10 ms frame and run a detector over it.
|
||||
pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16]) -> VadOutput {
|
||||
let mut frame = [0.0_f32; INPUT_FRAME_10MS];
|
||||
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
|
||||
*dst = i16_to_f32(src);
|
||||
}
|
||||
detector.process_10ms(&frame)
|
||||
}
|
||||
|
||||
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
/// Configure the preferred Silero ONNX model path.
|
||||
///
|
||||
/// The path is validated eagerly. A successful call increments the
|
||||
/// model epoch so running audio backends can reload the model without
|
||||
/// an app restart.
|
||||
pub fn set_silero_model_path(path: &str) -> Result<(), AudioError> {
|
||||
let path = path.trim();
|
||||
if path.is_empty() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"vad model path must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
if !std::path::Path::new(path).is_file() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(format!(
|
||||
"vad model path does not exist or is not a file: {path}"
|
||||
)));
|
||||
}
|
||||
let mut guard = silero_model_path_override()
|
||||
.write()
|
||||
.map_err(|_| AudioError::Backend("vad model path lock poisoned".to_string()))?;
|
||||
*guard = Some(path.to_string());
|
||||
SILERO_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Monotonic counter incremented whenever the configured model path changes.
|
||||
pub fn silero_model_epoch() -> u64 {
|
||||
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Return the expected path of the Silero VAD v6 ONNX model in the
|
||||
/// iOS app bundle. The model is shipped as a Flutter asset and copied
|
||||
/// to the app's Documents directory by the Dart-side asset loader.
|
||||
///
|
||||
/// Returns an empty string on non-Apple platforms (Silero is not
|
||||
/// supported there; `SileroOnnxVad::try_new` will return `None`).
|
||||
pub fn silero_model_bundle_path() -> String {
|
||||
if let Ok(guard) = silero_model_path_override().read() {
|
||||
if let Some(path) = guard.as_ref() {
|
||||
return path.clone();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
// Primary: Documents directory (written by Flutter asset loader).
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
let docs = format!("{home}/Documents/silero_vad.onnx");
|
||||
if std::path::Path::new(&docs).exists() {
|
||||
return docs;
|
||||
}
|
||||
// Fallback: app bundle Resources directory.
|
||||
let bundle = format!("{home}/../Library/silero_vad.onnx");
|
||||
if std::path::Path::new(&bundle).exists() {
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
// Last resort: current working directory (useful in tests).
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fallback_opens_for_voiced_signal() {
|
||||
let mut vad = WebRtcFallbackVad::default();
|
||||
let mut frame = [0_i16; INPUT_FRAME_10MS];
|
||||
for (idx, sample) in frame.iter_mut().enumerate() {
|
||||
let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / 48_000.0;
|
||||
*sample = (phase.sin() * 12_000.0) as i16;
|
||||
}
|
||||
|
||||
let output = process_i16_10ms(&mut vad, &frame);
|
||||
|
||||
assert!(output.speech);
|
||||
assert!(output.probability >= 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_stays_closed_for_silence() {
|
||||
let mut vad = WebRtcFallbackVad::default();
|
||||
let frame = [0_i16; INPUT_FRAME_10MS];
|
||||
|
||||
let output = process_i16_10ms(&mut vad, &frame);
|
||||
|
||||
assert!(!output.speech);
|
||||
assert!(output.probability < 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_silero_model_path_rejects_missing_file() {
|
||||
let result = set_silero_model_path("/definitely/not/a/silero_vad.onnx");
|
||||
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_silero_model_path_updates_override_and_epoch() {
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("chanora_test_silero_{}.onnx", std::process::id()));
|
||||
std::fs::write(&path, b"test").unwrap();
|
||||
let before = silero_model_epoch();
|
||||
|
||||
set_silero_model_path(path.to_str().unwrap()).unwrap();
|
||||
|
||||
assert!(silero_model_epoch() > before);
|
||||
assert_eq!(silero_model_bundle_path(), path.to_string_lossy());
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Lightweight 48 kHz → 16 kHz downsampler for VAD input.
|
||||
//!
|
||||
//! Silero VAD and the real WebRTC VAD both operate at 16 kHz. The
|
||||
//! VPIO capture stream is pinned at 48 kHz (3× ratio). We use a
|
||||
//! simple polyphase FIR with a 3:1 decimation factor. The filter
|
||||
//! coefficients are a 32-tap Kaiser-windowed low-pass at 8 kHz
|
||||
//! (Nyquist of the 16 kHz output), pre-computed offline and baked
|
||||
//! in as constants so there is no runtime allocation.
|
||||
//!
|
||||
//! Quality is sufficient for VAD (speech/silence discrimination);
|
||||
//! this is not a high-fidelity resampler.
|
||||
|
||||
/// Input sample rate (Hz).
|
||||
pub const INPUT_HZ: u32 = 48_000;
|
||||
/// Output sample rate (Hz).
|
||||
pub const OUTPUT_HZ: u32 = 16_000;
|
||||
/// Decimation factor (INPUT_HZ / OUTPUT_HZ).
|
||||
pub const DECIMATION: usize = 3;
|
||||
|
||||
/// Samples in one 10 ms frame at 48 kHz.
|
||||
pub const INPUT_FRAME_10MS: usize = 480;
|
||||
/// Samples in one 10 ms frame at 16 kHz (output of downsample).
|
||||
pub const OUTPUT_FRAME_10MS: usize = 160;
|
||||
|
||||
/// 32-tap FIR low-pass filter coefficients (Kaiser β=8, fc=8 kHz/48 kHz).
|
||||
/// Generated with scipy.signal.firwin(32, 8000/48000*2, window=('kaiser', 8)).
|
||||
/// Symmetric — only 16 unique values; stored in full for clarity.
|
||||
#[rustfmt::skip]
|
||||
const FIR_COEFFS: [f32; 32] = [
|
||||
-0.000_592_3, -0.001_158_5, -0.001_601_5, -0.000_993_5,
|
||||
0.001_601_5, 0.006_046_8, 0.012_131_5, 0.018_614_0,
|
||||
0.023_448_0, 0.024_726_0, 0.021_048_0, 0.012_636_0,
|
||||
0.000_993_5, -0.011_614_0, -0.021_048_0, -0.024_726_0,
|
||||
-0.024_726_0, -0.021_048_0, -0.011_614_0, 0.000_993_5,
|
||||
0.012_636_0, 0.021_048_0, 0.024_726_0, 0.023_448_0,
|
||||
0.018_614_0, 0.012_131_5, 0.006_046_8, 0.001_601_5,
|
||||
-0.000_993_5, -0.001_601_5, -0.001_158_5, -0.000_592_3,
|
||||
];
|
||||
|
||||
const TAPS: usize = FIR_COEFFS.len();
|
||||
|
||||
/// Stateful 48→16 kHz downsampler. Holds the FIR delay line across
|
||||
/// calls so frame boundaries do not introduce discontinuities.
|
||||
pub struct Downsampler48to16 {
|
||||
/// Circular delay line (length = TAPS).
|
||||
delay: [f32; TAPS],
|
||||
/// Write head into the delay line.
|
||||
head: usize,
|
||||
/// Phase counter: 0..DECIMATION. When phase==0 we emit a sample.
|
||||
phase: usize,
|
||||
}
|
||||
|
||||
impl Default for Downsampler48to16 {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
delay: [0.0; TAPS],
|
||||
head: 0,
|
||||
phase: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Downsampler48to16 {
|
||||
/// Process `input` (48 kHz f32 mono) and write 16 kHz output
|
||||
/// into `output`. Returns the number of samples written.
|
||||
///
|
||||
/// For a full 10 ms input frame (480 samples) this always
|
||||
/// produces exactly 160 output samples.
|
||||
pub fn process(&mut self, input: &[f32], output: &mut [f32]) -> usize {
|
||||
let mut out_idx = 0;
|
||||
for &sample in input {
|
||||
// Push sample into circular delay line.
|
||||
self.delay[self.head] = sample;
|
||||
self.head = (self.head + 1) % TAPS;
|
||||
|
||||
if self.phase == 0 {
|
||||
// Compute FIR dot product.
|
||||
let mut acc = 0.0_f32;
|
||||
for (k, &coeff) in FIR_COEFFS.iter().enumerate() {
|
||||
let tap_idx = (self.head + TAPS - 1 - k) % TAPS;
|
||||
acc += self.delay[tap_idx] * coeff;
|
||||
}
|
||||
if out_idx < output.len() {
|
||||
output[out_idx] = acc;
|
||||
out_idx += 1;
|
||||
}
|
||||
}
|
||||
self.phase = (self.phase + 1) % DECIMATION;
|
||||
}
|
||||
out_idx
|
||||
}
|
||||
|
||||
/// Convenience: downsample a full 10 ms 48 kHz frame into a
|
||||
/// fixed-size 160-sample 16 kHz buffer.
|
||||
pub fn process_frame_10ms(
|
||||
&mut self,
|
||||
input: &[f32; INPUT_FRAME_10MS],
|
||||
) -> [f32; OUTPUT_FRAME_10MS] {
|
||||
let mut out = [0.0_f32; OUTPUT_FRAME_10MS];
|
||||
let n = self.process(input, &mut out);
|
||||
debug_assert_eq!(
|
||||
n, OUTPUT_FRAME_10MS,
|
||||
"resampler produced {n} samples, expected 160"
|
||||
);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn frame_produces_160_samples() {
|
||||
let mut ds = Downsampler48to16::default();
|
||||
let input = [0.5_f32; INPUT_FRAME_10MS];
|
||||
let out = ds.process_frame_10ms(&input);
|
||||
// DC input → DC output (scaled by filter gain ≈ 1/3 due to decimation).
|
||||
// Just check length and that output is finite and non-zero.
|
||||
assert_eq!(out.len(), OUTPUT_FRAME_10MS);
|
||||
assert!(out.iter().all(|s| s.is_finite()));
|
||||
assert!(out.iter().any(|s| s.abs() > 0.001));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silence_produces_silence() {
|
||||
let mut ds = Downsampler48to16::default();
|
||||
let input = [0.0_f32; INPUT_FRAME_10MS];
|
||||
let out = ds.process_frame_10ms(&input);
|
||||
assert!(out.iter().all(|s| s.abs() < 1e-9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consecutive_frames_are_continuous() {
|
||||
let mut ds = Downsampler48to16::default();
|
||||
// Two frames of DC — output should be stable (no edge discontinuity).
|
||||
let input = [0.3_f32; INPUT_FRAME_10MS];
|
||||
let out1 = ds.process_frame_10ms(&input);
|
||||
let out2 = ds.process_frame_10ms(&input);
|
||||
// Last sample of frame 1 and first sample of frame 2 should be close.
|
||||
let diff = (out1[OUTPUT_FRAME_10MS - 1] - out2[0]).abs();
|
||||
assert!(diff < 0.05, "discontinuity between frames: {diff}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
//! Silero VAD v6 ONNX runtime integration (P1 VAD_002).
|
||||
//!
|
||||
//! ## Silero VAD v6 model I/O
|
||||
//!
|
||||
//! The v6 model (silero_vad.onnx from the v6.x releases) has a different
|
||||
//! interface from v4. Key changes:
|
||||
//!
|
||||
//! | Tensor | Shape | Dtype | Meaning |
|
||||
//! |---------|------------------|-------|--------------------------------------|
|
||||
//! | input | \[1, 576\] | f32 | 64-sample context + 512-sample frame |
|
||||
//! | state | \[2, 1, 128\] | f32 | LSTM state (carry across frames) |
|
||||
//! | sr | \[1\] | i64 | Sample rate (16000 or 8000) |
|
||||
//! | output | \[1, 1\] | f32 | Speech probability |
|
||||
//! | stateN | \[2, 1, 128\] | f32 | Updated LSTM state |
|
||||
//!
|
||||
//! Frame size: **512 samples at 16 kHz = 32 ms**.
|
||||
//! Context: **64 samples** prepended to each frame (last 64 samples of previous frame).
|
||||
//! Total input width: 512 + 64 = **576 samples**.
|
||||
//!
|
||||
//! ## Threading
|
||||
//!
|
||||
//! `SileroOnnxVad` is `Send`. The session is created once and reused —
|
||||
//! never re-created per callback (INV_007).
|
||||
//!
|
||||
//! ## Accumulation
|
||||
//!
|
||||
//! The capture pipeline delivers 10 ms frames (480 samples at 48 kHz →
|
||||
//! 160 samples at 16 kHz). Three 10 ms frames = 30 ms ≈ 32 ms. We
|
||||
//! accumulate 512 samples (32 ms at 16 kHz) before running inference.
|
||||
//! The last probability is held between inference calls so the state
|
||||
//! machine always has a value to work with.
|
||||
//!
|
||||
//! ## Fallback
|
||||
//!
|
||||
//! `try_new` returns `None` when the model file is missing, the ONNX
|
||||
//! Runtime is unavailable, or the platform is not iOS/macOS. The caller
|
||||
//! falls back to `WebRtcFallbackVad`.
|
||||
|
||||
use super::{VadOutput, VoiceActivityDetector};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// 16 kHz frame size for Silero VAD v6 (32 ms).
|
||||
pub const SILERO_FRAME_16K: usize = 512;
|
||||
/// Context size prepended to each frame (64 samples at 16 kHz).
|
||||
pub const SILERO_CONTEXT_16K: usize = 64;
|
||||
/// Total input width: context + frame.
|
||||
pub const SILERO_INPUT_WIDTH: usize = SILERO_CONTEXT_16K + SILERO_FRAME_16K;
|
||||
/// LSTM state size: 2 × 1 × 128 = 256 f32 values.
|
||||
pub const SILERO_STATE_SIZE: usize = 256;
|
||||
/// Maximum lag in 10 ms frames before the realtime callback treats
|
||||
/// the Silero worker as stale and falls back to the local WebRTC
|
||||
/// detector for that frame.
|
||||
pub const SILERO_MAX_STALE_FRAMES: u64 = 3;
|
||||
|
||||
/// Silero VAD v6 ONNX backend.
|
||||
///
|
||||
/// Operates at **16 kHz**, accumulating 32 ms frames (512 samples)
|
||||
/// before running inference. The caller is responsible for downsampling
|
||||
/// from 48 kHz before calling `process_10ms`.
|
||||
pub struct SileroOnnxVad {
|
||||
/// LSTM state [2, 1, 128] — persisted across frames.
|
||||
state: Box<[f32; SILERO_STATE_SIZE]>,
|
||||
/// Context ring: last 64 samples of the previous frame.
|
||||
context: Box<[f32; SILERO_CONTEXT_16K]>,
|
||||
/// Accumulation buffer for 16 kHz samples (fills to SILERO_FRAME_16K).
|
||||
accum: Vec<f32>,
|
||||
/// Last speech probability output (held between inference calls).
|
||||
last_probability: f32,
|
||||
/// Model path stored for diagnostics.
|
||||
model_path: String,
|
||||
/// Inner ONNX implementation (platform-specific).
|
||||
inner: SileroInner,
|
||||
}
|
||||
|
||||
enum SileroInner {
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
Onnx(OnnxSession),
|
||||
#[allow(dead_code)]
|
||||
Stub,
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
struct OnnxSession {
|
||||
session: ort::session::Session,
|
||||
}
|
||||
|
||||
impl SileroOnnxVad {
|
||||
/// Attempt to load the Silero v6 ONNX model from `model_path`.
|
||||
///
|
||||
/// Returns `None` when the model file is missing, the ONNX Runtime
|
||||
/// is unavailable, or the platform is not iOS/macOS.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
Self::try_new_onnx(model_path)
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
let _ = model_path;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
fn try_new_onnx(model_path: &str) -> Option<Self> {
|
||||
use tracing::{error, info};
|
||||
|
||||
if !std::path::Path::new(model_path).exists() {
|
||||
tracing::warn!(
|
||||
target: "chanora_audio",
|
||||
path = model_path,
|
||||
"SileroOnnxVad: model file not found; falling back to WebRtcFallbackVad"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
if let Some(path) = bundled_onnxruntime_path() {
|
||||
let _ = ort::init_from(path.to_string_lossy()).commit();
|
||||
}
|
||||
|
||||
let session_result = std::panic::catch_unwind(|| {
|
||||
ort::session::Session::builder().and_then(|b| b.commit_from_file(model_path))
|
||||
});
|
||||
|
||||
match session_result {
|
||||
Err(_) => {
|
||||
error!(
|
||||
target: "chanora_audio",
|
||||
path = model_path,
|
||||
"SileroOnnxVad: ONNX Runtime panicked during load; falling back to WebRtcFallbackVad"
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok(Ok(session)) => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
path = model_path,
|
||||
"SileroOnnxVad v6: model loaded"
|
||||
);
|
||||
Some(Self {
|
||||
state: Box::new([0.0; SILERO_STATE_SIZE]),
|
||||
context: Box::new([0.0; SILERO_CONTEXT_16K]),
|
||||
accum: Vec::with_capacity(SILERO_FRAME_16K),
|
||||
last_probability: 0.0,
|
||||
model_path: model_path.to_owned(),
|
||||
inner: SileroInner::Onnx(OnnxSession { session }),
|
||||
})
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
error!(
|
||||
target: "chanora_audio",
|
||||
path = model_path,
|
||||
error = %e,
|
||||
"SileroOnnxVad: failed to load model; falling back to WebRtcFallbackVad"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset LSTM state and context (call on voice_leave / session restart).
|
||||
pub fn reset_state(&mut self) {
|
||||
self.state.iter_mut().for_each(|v| *v = 0.0);
|
||||
self.context.iter_mut().for_each(|v| *v = 0.0);
|
||||
self.accum.clear();
|
||||
self.last_probability = 0.0;
|
||||
}
|
||||
|
||||
/// Return the model path for diagnostics.
|
||||
pub fn model_path(&self) -> &str {
|
||||
&self.model_path
|
||||
}
|
||||
|
||||
fn input_with_context(context: &[f32; SILERO_CONTEXT_16K], audio_frame: &[f32]) -> Vec<f32> {
|
||||
let mut input = Vec::with_capacity(SILERO_CONTEXT_16K + audio_frame.len());
|
||||
input.extend_from_slice(context);
|
||||
input.extend_from_slice(audio_frame);
|
||||
input
|
||||
}
|
||||
|
||||
fn update_context_from_frame(&mut self, audio_frame: &[f32]) {
|
||||
let ctx_start = audio_frame.len().saturating_sub(SILERO_CONTEXT_16K);
|
||||
let new_ctx = &audio_frame[ctx_start..];
|
||||
let copy_len = new_ctx.len().min(SILERO_CONTEXT_16K);
|
||||
self.context.fill(0.0);
|
||||
self.context[SILERO_CONTEXT_16K - copy_len..].copy_from_slice(&new_ctx[..copy_len]);
|
||||
}
|
||||
|
||||
/// Run one upstream-style `calc_level` pass over a 32 ms / 512-sample
|
||||
/// 16 kHz frame: concatenate prior context, pass `input/state/sr` to
|
||||
/// ONNX, persist `stateN`, then refresh context from the current frame.
|
||||
/// Updates `last_probability` and returns the new value.
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
fn calc_level(&mut self, audio_frame: &[f32]) -> f32 {
|
||||
use ort::value::Value;
|
||||
use tracing::error;
|
||||
|
||||
let SileroInner::Onnx(ref mut inner) = self.inner else {
|
||||
return self.last_probability;
|
||||
};
|
||||
|
||||
debug_assert_eq!(audio_frame.len(), SILERO_FRAME_16K);
|
||||
|
||||
// Build input: [1, 576] = context (64) + frame (512), matching
|
||||
// snakers4/silero-vad's Rust `calc_level` example.
|
||||
let input_vec = Self::input_with_context(self.context.as_ref(), audio_frame);
|
||||
|
||||
// Build ndarray tensors.
|
||||
use ndarray::{Array, IxDyn};
|
||||
|
||||
let input_arr = Array::from_shape_vec(IxDyn(&[1, SILERO_INPUT_WIDTH]), input_vec);
|
||||
let state_arr = Array::from_shape_vec(IxDyn(&[2, 1, 128]), self.state.to_vec());
|
||||
let sr_arr = Array::from_shape_vec(IxDyn(&[1]), vec![16000_i64]);
|
||||
|
||||
let (input_arr, state_arr, sr_arr) = match (input_arr, state_arr, sr_arr) {
|
||||
(Ok(i), Ok(s), Ok(sr)) => (i, s, sr),
|
||||
_ => return self.last_probability,
|
||||
};
|
||||
|
||||
let input_val = match Value::from_array(input_arr) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: input tensor error");
|
||||
return self.last_probability;
|
||||
}
|
||||
};
|
||||
let state_val = match Value::from_array(state_arr) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: state tensor error");
|
||||
return self.last_probability;
|
||||
}
|
||||
};
|
||||
let sr_val = match Value::from_array(sr_arr) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: sr tensor error");
|
||||
return self.last_probability;
|
||||
}
|
||||
};
|
||||
|
||||
let outputs =
|
||||
match inner
|
||||
.session
|
||||
.run([(&input_val).into(), (&state_val).into(), (&sr_val).into()])
|
||||
{
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: inference failed");
|
||||
return self.last_probability;
|
||||
}
|
||||
};
|
||||
|
||||
// Extract probability from "output".
|
||||
if let Ok((_, prob_data)) = outputs["output"].try_extract_tensor::<f32>() {
|
||||
if let Some(&p) = prob_data.first() {
|
||||
self.last_probability = p.clamp(0.0, 1.0);
|
||||
}
|
||||
}
|
||||
|
||||
// Update state from "stateN".
|
||||
if let Ok((shape, state_data)) = outputs["stateN"].try_extract_tensor::<f32>() {
|
||||
let total: usize = shape.iter().map(|&d| d as usize).product();
|
||||
let copy_len = total.min(SILERO_STATE_SIZE);
|
||||
self.state[..copy_len].copy_from_slice(&state_data[..copy_len]);
|
||||
}
|
||||
|
||||
drop(outputs);
|
||||
|
||||
// Match the upstream example: context becomes the last context_size
|
||||
// samples from the current frame after the model call succeeds.
|
||||
self.update_context_from_frame(audio_frame);
|
||||
|
||||
self.last_probability
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
|
||||
let exe = std::env::current_exe().ok()?;
|
||||
let app_dir = exe.parent()?;
|
||||
let framework = app_dir
|
||||
.join("Frameworks")
|
||||
.join("onnxruntime.framework")
|
||||
.join("onnxruntime");
|
||||
framework.exists().then_some(framework)
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for SileroOnnxVad {
|
||||
/// Accept one 10 ms **16 kHz** f32 mono frame (160 samples).
|
||||
///
|
||||
/// Accumulates samples until a full 32 ms frame (512 samples) is
|
||||
/// ready, then runs inference. Between inference calls the last
|
||||
/// probability is returned unchanged.
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
debug_assert_eq!(
|
||||
samples.len(),
|
||||
super::resampler::OUTPUT_FRAME_10MS,
|
||||
"SileroOnnxVad expects 160 samples (16 kHz 10 ms), got {}",
|
||||
samples.len()
|
||||
);
|
||||
|
||||
self.accum.extend_from_slice(samples);
|
||||
|
||||
if self.accum.len() >= SILERO_FRAME_16K {
|
||||
let audio_frame: Vec<f32> = self.accum[..SILERO_FRAME_16K].to_vec();
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
if matches!(self.inner, SileroInner::Onnx(_)) {
|
||||
self.calc_level(&audio_frame);
|
||||
} else {
|
||||
self.update_context_from_frame(&audio_frame);
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
self.update_context_from_frame(&audio_frame);
|
||||
}
|
||||
// Drain the accumulator (keep any overflow for next frame).
|
||||
let overflow: Vec<f32> = self.accum.drain(SILERO_FRAME_16K..).collect();
|
||||
self.accum.clear();
|
||||
self.accum.extend_from_slice(&overflow);
|
||||
}
|
||||
|
||||
VadOutput {
|
||||
probability: self.last_probability,
|
||||
speech: self.last_probability >= 0.5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: ONNX Runtime sessions are thread-safe for inference.
|
||||
// State arrays are owned by this struct and accessed only from
|
||||
// the single capture callback thread.
|
||||
unsafe impl Send for SileroOnnxVad {}
|
||||
|
||||
struct SileroFrameMessage {
|
||||
seq: u64,
|
||||
frame: [f32; super::resampler::INPUT_FRAME_10MS],
|
||||
}
|
||||
|
||||
/// Background Silero worker. The realtime callback only enqueues
|
||||
/// 10 ms frames and reads the latest probability atomically.
|
||||
pub struct SileroOnnxVadWorker {
|
||||
tx: Option<std::sync::mpsc::SyncSender<SileroFrameMessage>>,
|
||||
latest_probability: Arc<AtomicU32>,
|
||||
latest_processed_seq: Arc<AtomicU64>,
|
||||
alive: Arc<AtomicBool>,
|
||||
handle: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl SileroOnnxVadWorker {
|
||||
/// Start a background Silero worker if the model loads.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
let vad = SileroOnnxVad::try_new(model_path)?;
|
||||
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
|
||||
let latest_processed_seq = Arc::new(AtomicU64::new(0));
|
||||
let alive = Arc::new(AtomicBool::new(true));
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(8);
|
||||
let latest_probability_for_thread = latest_probability.clone();
|
||||
let latest_processed_seq_for_thread = latest_processed_seq.clone();
|
||||
let alive_for_thread = alive.clone();
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("chanora-silero-vad".to_string())
|
||||
.spawn(move || {
|
||||
let mut vad = super::Resampled16kHzVad::new(vad);
|
||||
while alive_for_thread.load(Ordering::Relaxed) {
|
||||
let message = match rx.recv() {
|
||||
Ok(message) => message,
|
||||
Err(_) => break,
|
||||
};
|
||||
let output = vad.process_10ms(&message.frame);
|
||||
latest_probability_for_thread.store(
|
||||
output.probability.clamp(0.0, 1.0).to_bits(),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
latest_processed_seq_for_thread.store(message.seq, Ordering::Relaxed);
|
||||
}
|
||||
})
|
||||
.ok()?;
|
||||
|
||||
Some(Self {
|
||||
tx: Some(tx),
|
||||
latest_probability,
|
||||
latest_processed_seq,
|
||||
alive,
|
||||
handle: Some(handle),
|
||||
})
|
||||
}
|
||||
|
||||
/// Best-effort enqueue of a 10 ms frame for background inference.
|
||||
pub fn try_send(&self, seq: u64, frame: &[f32; super::resampler::INPUT_FRAME_10MS]) -> bool {
|
||||
let Some(tx) = &self.tx else {
|
||||
return false;
|
||||
};
|
||||
tx.try_send(SileroFrameMessage { seq, frame: *frame })
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// Latest probability published by the background worker.
|
||||
pub fn latest_probability(&self) -> f32 {
|
||||
f32::from_bits(self.latest_probability.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Number of 10 ms frames the worker is behind the capture thread.
|
||||
pub fn lag_frames(&self, capture_seq: u64) -> u64 {
|
||||
capture_seq.saturating_sub(self.latest_processed_seq.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// True when the worker is too far behind to trust its latest
|
||||
/// probability for the current frame.
|
||||
pub fn is_stale(&self, capture_seq: u64) -> bool {
|
||||
self.lag_frames(capture_seq) > SILERO_MAX_STALE_FRAMES
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SileroOnnxVadWorker {
|
||||
fn drop(&mut self) {
|
||||
self.alive.store(false, Ordering::Relaxed);
|
||||
let _ = self.tx.take();
|
||||
let _ = self.handle.take();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_stub_vad() -> SileroOnnxVad {
|
||||
SileroOnnxVad {
|
||||
state: Box::new([0.0; SILERO_STATE_SIZE]),
|
||||
context: Box::new([0.0; SILERO_CONTEXT_16K]),
|
||||
accum: Vec::new(),
|
||||
last_probability: 0.0,
|
||||
model_path: String::new(),
|
||||
inner: SileroInner::Stub,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_new_returns_none_without_model_file() {
|
||||
let result = SileroOnnxVad::try_new("/nonexistent/silero_vad.onnx");
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stub_accumulates_and_holds_zero_probability() {
|
||||
let mut vad = make_stub_vad();
|
||||
let frame = vec![0.0_f32; super::super::resampler::OUTPUT_FRAME_10MS];
|
||||
// Feed 3 frames (30 ms < 32 ms) — no inference yet.
|
||||
for _ in 0..3 {
|
||||
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame);
|
||||
assert_eq!(out.probability, 0.0);
|
||||
}
|
||||
// Feed 1 more frame (40 ms > 32 ms) — accumulator drains.
|
||||
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame);
|
||||
assert_eq!(out.probability, 0.0); // stub stays at 0
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_state_clears_all() {
|
||||
let mut vad = make_stub_vad();
|
||||
vad.state[0] = 1.0;
|
||||
vad.context[0] = 1.0;
|
||||
vad.last_probability = 0.9;
|
||||
vad.accum.push(0.5);
|
||||
vad.reset_state();
|
||||
assert_eq!(vad.state[0], 0.0);
|
||||
assert_eq!(vad.context[0], 0.0);
|
||||
assert_eq!(vad.last_probability, 0.0);
|
||||
assert!(vad.accum.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accumulates_correct_number_of_samples() {
|
||||
let mut vad = make_stub_vad();
|
||||
let frame = vec![0.1_f32; super::super::resampler::OUTPUT_FRAME_10MS]; // 160 samples
|
||||
// 3 × 160 = 480 < 512 — not yet full.
|
||||
for _ in 0..3 {
|
||||
VoiceActivityDetector::process_10ms(&mut vad, &frame);
|
||||
}
|
||||
assert_eq!(vad.accum.len(), 480);
|
||||
// 4th frame: 640 > 512 — inference fires, 128 samples remain.
|
||||
VoiceActivityDetector::process_10ms(&mut vad, &frame);
|
||||
assert_eq!(vad.accum.len(), 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_concatenates_context_before_frame_like_upstream_example() {
|
||||
let mut context = [0.0_f32; SILERO_CONTEXT_16K];
|
||||
context[0] = -1.0;
|
||||
context[SILERO_CONTEXT_16K - 1] = 1.0;
|
||||
let frame = vec![0.25_f32; SILERO_FRAME_16K];
|
||||
|
||||
let input = SileroOnnxVad::input_with_context(&context, &frame);
|
||||
|
||||
assert_eq!(input.len(), SILERO_INPUT_WIDTH);
|
||||
assert_eq!(input[0], -1.0);
|
||||
assert_eq!(input[SILERO_CONTEXT_16K - 1], 1.0);
|
||||
assert_eq!(input[SILERO_CONTEXT_16K], 0.25);
|
||||
assert_eq!(input[SILERO_INPUT_WIDTH - 1], 0.25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn context_tracks_last_64_samples_of_completed_frame() {
|
||||
let mut vad = make_stub_vad();
|
||||
let frame = vec![0.0_f32; super::super::resampler::OUTPUT_FRAME_10MS];
|
||||
for idx in 0..4 {
|
||||
let mut chunk = frame.clone();
|
||||
let chunk_len = chunk.len();
|
||||
for (sample_idx, sample) in chunk.iter_mut().enumerate() {
|
||||
*sample = (idx * chunk_len + sample_idx) as f32;
|
||||
}
|
||||
VoiceActivityDetector::process_10ms(&mut vad, &chunk);
|
||||
}
|
||||
|
||||
let completed_frame: Vec<f32> = (0..SILERO_FRAME_16K).map(|v| v as f32).collect();
|
||||
assert_eq!(
|
||||
vad.context.as_ref(),
|
||||
&completed_frame[SILERO_FRAME_16K - SILERO_CONTEXT_16K..]
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user