feat(voice): add iOS VAD runtime support
This commit is contained in:
Edison Jwa
@@ -0,0 +1,275 @@
|
||||
//! Voice activity detection backends and helpers.
|
||||
//!
|
||||
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into
|
||||
//! this module. The production path prefers a model-backed detector when
|
||||
//! available, and otherwise uses the realtime-safe fallback below so
|
||||
//! VoiceActivity mode never collapses back to Continuous transmit.
|
||||
|
||||
pub mod resampler;
|
||||
pub mod silero_onnx;
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
|
||||
use crate::frame::{dbfs, i16_to_f32};
|
||||
use crate::AudioError;
|
||||
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||
|
||||
pub use silero_onnx::SileroOnnxVad;
|
||||
|
||||
/// Voice activity detector output for one 10 ms frame.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct VadOutput {
|
||||
/// Speech confidence in the inclusive range `[0.0, 1.0]`.
|
||||
pub probability: f32,
|
||||
/// Immediate detector speech decision before hangover/min-duration state.
|
||||
pub speech: bool,
|
||||
}
|
||||
|
||||
/// Realtime-safe detector that consumes one 10 ms f32 mono frame.
|
||||
pub trait VoiceActivityDetector: Send {
|
||||
/// Process one 10 ms frame and return speech probability/state.
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput;
|
||||
}
|
||||
|
||||
/// Realtime-safe fallback VAD used when a model runtime is unavailable.
|
||||
///
|
||||
/// This is not an energy-only transmit gate. It combines RMS level,
|
||||
/// zero-crossing rate, and peak-to-RMS shape with hysteresis so stable
|
||||
/// background rumble is less likely to open VoiceActivity than speech.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WebRtcFallbackVad {
|
||||
open_dbfs: f32,
|
||||
close_dbfs: f32,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl Default for WebRtcFallbackVad {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
open_dbfs: -42.0,
|
||||
close_dbfs: -50.0,
|
||||
active: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WebRtcFallbackVad {
|
||||
fn zero_crossing_rate(samples: &[f32]) -> f32 {
|
||||
if samples.len() < 2 {
|
||||
return 0.0;
|
||||
}
|
||||
let crossings = samples
|
||||
.windows(2)
|
||||
.filter(|pair| (pair[0] >= 0.0 && pair[1] < 0.0) || (pair[0] < 0.0 && pair[1] >= 0.0))
|
||||
.count();
|
||||
crossings as f32 / (samples.len() - 1) as f32
|
||||
}
|
||||
|
||||
fn peak_to_rms(samples: &[f32], rms: f32) -> f32 {
|
||||
if rms <= 0.000_001 {
|
||||
return 0.0;
|
||||
}
|
||||
let peak = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
|
||||
peak / rms
|
||||
}
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for WebRtcFallbackVad {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
let level = dbfs(samples);
|
||||
let threshold = if self.active {
|
||||
self.close_dbfs
|
||||
} else {
|
||||
self.open_dbfs
|
||||
};
|
||||
let rms = samples.iter().map(|s| s * s).sum::<f32>() / samples.len().max(1) as f32;
|
||||
let rms = rms.sqrt();
|
||||
let zcr = Self::zero_crossing_rate(samples);
|
||||
let crest = Self::peak_to_rms(samples, rms);
|
||||
|
||||
// Level score: steeper curve so silence (-50 dBFS) scores near 0.
|
||||
// Speech is typically -30 to -10 dBFS; silence is -60 to -45 dBFS.
|
||||
// Map [-60, -20] → [0, 1] with a midpoint at -40 dBFS.
|
||||
let level_score = ((level + 60.0) / 40.0).clamp(0.0, 1.0);
|
||||
|
||||
let zcr_score = if (0.015..=0.32).contains(&zcr) {
|
||||
1.0
|
||||
} else {
|
||||
0.3 // penalise non-speech ZCR more aggressively
|
||||
};
|
||||
let crest_score = if (1.5..=12.0).contains(&crest) {
|
||||
1.0
|
||||
} else {
|
||||
0.3
|
||||
};
|
||||
let probability =
|
||||
(level_score * 0.72 + zcr_score * 0.18 + crest_score * 0.10).clamp(0.0, 1.0);
|
||||
self.active = level >= threshold && probability >= 0.5;
|
||||
VadOutput {
|
||||
probability,
|
||||
speech: self.active,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wraps any `VoiceActivityDetector` that operates at 16 kHz and
|
||||
/// downsamples 48 kHz input before forwarding.
|
||||
pub struct Resampled16kHzVad<D: VoiceActivityDetector> {
|
||||
inner: D,
|
||||
downsampler: Downsampler48to16,
|
||||
}
|
||||
|
||||
impl<D: VoiceActivityDetector> Resampled16kHzVad<D> {
|
||||
/// Wrap a 16 kHz detector so it can consume 48 kHz frames.
|
||||
pub fn new(inner: D) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
downsampler: Downsampler48to16::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<D: VoiceActivityDetector> VoiceActivityDetector for Resampled16kHzVad<D> {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
debug_assert_eq!(samples.len(), INPUT_FRAME_10MS);
|
||||
let mut input = [0.0_f32; INPUT_FRAME_10MS];
|
||||
input.copy_from_slice(samples);
|
||||
let downsampled = self.downsampler.process_frame_10ms(&input);
|
||||
self.inner.process_10ms(&downsampled)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one 48 kHz i16 10 ms frame and run a detector over it.
|
||||
pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16]) -> VadOutput {
|
||||
let mut frame = [0.0_f32; INPUT_FRAME_10MS];
|
||||
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
|
||||
*dst = i16_to_f32(src);
|
||||
}
|
||||
detector.process_10ms(&frame)
|
||||
}
|
||||
|
||||
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
/// Configure the preferred Silero ONNX model path.
|
||||
///
|
||||
/// The path is validated eagerly. A successful call increments the
|
||||
/// model epoch so running audio backends can reload the model without
|
||||
/// an app restart.
|
||||
pub fn set_silero_model_path(path: &str) -> Result<(), AudioError> {
|
||||
let path = path.trim();
|
||||
if path.is_empty() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"vad model path must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
if !std::path::Path::new(path).is_file() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(format!(
|
||||
"vad model path does not exist or is not a file: {path}"
|
||||
)));
|
||||
}
|
||||
let mut guard = silero_model_path_override()
|
||||
.write()
|
||||
.map_err(|_| AudioError::Backend("vad model path lock poisoned".to_string()))?;
|
||||
*guard = Some(path.to_string());
|
||||
SILERO_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Monotonic counter incremented whenever the configured model path changes.
|
||||
pub fn silero_model_epoch() -> u64 {
|
||||
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Return the expected path of the Silero VAD v6 ONNX model in the
|
||||
/// iOS app bundle. The model is shipped as a Flutter asset and copied
|
||||
/// to the app's Documents directory by the Dart-side asset loader.
|
||||
///
|
||||
/// Returns an empty string on non-Apple platforms (Silero is not
|
||||
/// supported there; `SileroOnnxVad::try_new` will return `None`).
|
||||
pub fn silero_model_bundle_path() -> String {
|
||||
if let Ok(guard) = silero_model_path_override().read() {
|
||||
if let Some(path) = guard.as_ref() {
|
||||
return path.clone();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
// Primary: Documents directory (written by Flutter asset loader).
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
let docs = format!("{home}/Documents/silero_vad.onnx");
|
||||
if std::path::Path::new(&docs).exists() {
|
||||
return docs;
|
||||
}
|
||||
// Fallback: app bundle Resources directory.
|
||||
let bundle = format!("{home}/../Library/silero_vad.onnx");
|
||||
if std::path::Path::new(&bundle).exists() {
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
// Last resort: current working directory (useful in tests).
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fallback_opens_for_voiced_signal() {
|
||||
let mut vad = WebRtcFallbackVad::default();
|
||||
let mut frame = [0_i16; INPUT_FRAME_10MS];
|
||||
for (idx, sample) in frame.iter_mut().enumerate() {
|
||||
let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / 48_000.0;
|
||||
*sample = (phase.sin() * 12_000.0) as i16;
|
||||
}
|
||||
|
||||
let output = process_i16_10ms(&mut vad, &frame);
|
||||
|
||||
assert!(output.speech);
|
||||
assert!(output.probability >= 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_stays_closed_for_silence() {
|
||||
let mut vad = WebRtcFallbackVad::default();
|
||||
let frame = [0_i16; INPUT_FRAME_10MS];
|
||||
|
||||
let output = process_i16_10ms(&mut vad, &frame);
|
||||
|
||||
assert!(!output.speech);
|
||||
assert!(output.probability < 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_silero_model_path_rejects_missing_file() {
|
||||
let result = set_silero_model_path("/definitely/not/a/silero_vad.onnx");
|
||||
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_silero_model_path_updates_override_and_epoch() {
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("chanora_test_silero_{}.onnx", std::process::id()));
|
||||
std::fs::write(&path, b"test").unwrap();
|
||||
let before = silero_model_epoch();
|
||||
|
||||
set_silero_model_path(path.to_str().unwrap()).unwrap();
|
||||
|
||||
assert!(silero_model_epoch() > before);
|
||||
assert_eq!(silero_model_bundle_path(), path.to_string_lossy());
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user