edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(audio): add Apple CoreML Silero VAD

Browse Source
This commit is contained in:
Edison Jwa
2026-06-02 01:52:08 +09:00
parent 71b7502ab7
commit 813a38e92b
6 changed files with 411 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/audio_processing.rs
+1 -4
View File
@@ -102,7 +102,7 @@ pub enum VadBackend {
#[cfg(target_os = "ios")]
fn default_vad_backend() -> VadBackend {
VadBackend::WebrtcVad
VadBackend::SileroOnnx
}
#[cfg(not(target_os = "ios"))]
@@ -252,9 +252,6 @@ mod tests {
assert_eq!(config.aec, EffectOwner::Platform);
assert_eq!(config.ns, EffectOwner::Platform);
assert_eq!(config.agc, EffectOwner::Platform);
#[cfg(target_os = "ios")]
assert_eq!(config.vad_backend, VadBackend::WebrtcVad);
#[cfg(not(target_os = "ios"))]
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
}
crates/chanora_audio/src/ios_raw_unit.rs
+42 -5
View File
@@ -116,6 +116,7 @@ mod inner {
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_coreml_worker: Option<crate::vad::apple_coreml::AppleCoreMlVadWorker>,
current_vad_backend: crate::VadBackend,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
@@ -152,6 +153,7 @@ mod inner {
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_coreml_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
@@ -272,6 +274,7 @@ mod inner {
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_coreml_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
@@ -297,9 +300,16 @@ mod inner {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
if vad_backend == crate::VadBackend::SileroOnnx {
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
self.silero_coreml_worker =
crate::vad::apple_coreml::AppleCoreMlVadWorker::try_new();
if self.silero_coreml_worker.is_none() {
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
} else {
self.audio_processing_stats.set_vad_fallback_active(false);
}
} else {
self.silero_coreml_worker = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
self.vad_state.reset();
@@ -307,6 +317,7 @@ mod inner {
let (vad_probability, active) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
@@ -314,9 +325,35 @@ mod inner {
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
if let Some(worker) = self.silero_coreml_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
crates/chanora_audio/src/ios_voice_unit.rs
+39 -5
View File
@@ -138,6 +138,7 @@ struct IosCaptureState {
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_coreml_worker: Option<crate::vad::apple_coreml::AppleCoreMlVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
fallback_warned_backend: Option<crate::VadBackend>,
@@ -177,6 +178,7 @@ impl IosCaptureState {
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_coreml_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
@@ -350,6 +352,7 @@ impl IosCaptureState {
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_coreml_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
@@ -373,9 +376,16 @@ impl IosCaptureState {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
if vad_backend == crate::VadBackend::SileroOnnx {
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
self.silero_coreml_worker =
crate::vad::apple_coreml::AppleCoreMlVadWorker::try_new();
if self.silero_coreml_worker.is_none() {
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
} else {
self.audio_processing_stats.set_vad_fallback_active(false);
}
} else {
self.silero_coreml_worker = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
// Reset VAD state machine timers on backend switch.
@@ -418,6 +428,7 @@ impl IosCaptureState {
// VAD: only evaluate while VoiceActivity mode is active.
let (vad_probability, gate_open) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
@@ -425,9 +436,32 @@ impl IosCaptureState {
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
if let Some(worker) = self.silero_coreml_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
crates/chanora_audio/src/route_policy.rs
+9 -9
View File
@@ -30,7 +30,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::WebrtcVad,
vad_backend: VadBackend::SileroOnnx,
aec: EffectOwner::Platform,
// VPIO owns NS and AGC on the shipping default path (IOSP_002/003).
ns: EffectOwner::Platform,
@@ -43,7 +43,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::Noop,
vad_backend: VadBackend::WebrtcVad,
vad_backend: VadBackend::SileroOnnx,
// No AEC needed for wired headset (no acoustic echo path).
aec: EffectOwner::Off,
// Conservative NS/AGC: optional, not forced.
@@ -57,7 +57,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::WebrtcVad,
vad_backend: VadBackend::SileroOnnx,
// BT HFP manages its own AEC in the headset firmware.
aec: EffectOwner::Off,
ns: EffectOwner::Conservative,
@@ -87,7 +87,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::Noop,
vad_backend: VadBackend::WebrtcVad,
vad_backend: VadBackend::SileroOnnx,
// Safe fallback: AEC off until route is classified.
aec: EffectOwner::Off,
ns: EffectOwner::Off,
@@ -146,7 +146,7 @@ mod tests {
cfg.ios_mode,
IosVoiceProcessingMode::PlatformVoiceProcessing
);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
assert_eq!(cfg.vad_backend, VadBackend::SileroOnnx);
}
#[test]
@@ -157,7 +157,7 @@ mod tests {
AudioBackend::PlatformVoiceProcessing
);
assert_eq!(cfg.aec, EffectOwner::Platform);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
assert_eq!(cfg.vad_backend, VadBackend::SileroOnnx);
}
#[test]
@@ -165,14 +165,14 @@ mod tests {
let cfg = ios_route_policy(AudioRoute::WiredHeadset);
assert_eq!(cfg.aec, EffectOwner::Off);
assert_eq!(cfg.processing_backend, AudioBackend::Noop);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
assert_eq!(cfg.vad_backend, VadBackend::SileroOnnx);
}
#[test]
fn bluetooth_hfp_disables_app_aec() {
let cfg = ios_route_policy(AudioRoute::BluetoothHfp);
assert_eq!(cfg.aec, EffectOwner::Off);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
assert_eq!(cfg.vad_backend, VadBackend::SileroOnnx);
}
#[test]
@@ -187,7 +187,7 @@ mod tests {
fn unknown_route_safe_fallback_no_aec() {
let cfg = ios_route_policy(AudioRoute::Unknown);
assert_eq!(cfg.aec, EffectOwner::Off);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
assert_eq!(cfg.vad_backend, VadBackend::SileroOnnx);
}
#[test]
crates/chanora_audio/src/vad/apple_coreml.rs
+318
View File
@@ -0,0 +1,318 @@
//! Apple/CoreML Silero VAD bridge.
//!
//! The Swift Runner target exports a tiny C ABI around
//! `SileroCoreML.SileroVAD`. This Rust side resolves those symbols at
//! runtime, then runs inference on a background worker so realtime CoreAudio
//! callbacks only enqueue frames and read atomics.
use super::{VadOutput, VoiceActivityDetector};
use std::ffi::{c_char, c_void, CStr};
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::sync::Arc;
use std::thread::JoinHandle;
/// 16 kHz frame size required by `SileroCoreML.SileroVAD.process(_:)`.
pub const SILERO_COREML_FRAME_16K: usize = 512;
/// Maximum lag in 10 ms frames before the realtime callback falls back.
pub const SILERO_COREML_MAX_STALE_FRAMES: u64 = 3;
const SILERO_COREML_THRESHOLD: f32 = 0.5;
const RTLD_DEFAULT: *mut c_void = -2_isize as *mut c_void;
type CreateFn = unsafe extern "C" fn() -> *mut c_void;
type DestroyFn = unsafe extern "C" fn(*mut c_void);
type ResetFn = unsafe extern "C" fn(*mut c_void) -> i32;
type ProcessFn = unsafe extern "C" fn(*mut c_void, *const f32, usize, *mut f32) -> i32;
type LastErrorFn = unsafe extern "C" fn() -> *mut c_char;
type FreeStringFn = unsafe extern "C" fn(*mut c_char);
#[allow(improper_ctypes)]
extern "C" {
fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;
}
#[derive(Clone, Copy)]
struct AppleSileroSymbols {
create: CreateFn,
destroy: DestroyFn,
reset: ResetFn,
process: ProcessFn,
last_error: LastErrorFn,
free_string: FreeStringFn,
}
impl AppleSileroSymbols {
fn resolve() -> Option<Self> {
unsafe {
Some(Self {
create: std::mem::transmute::<*mut c_void, CreateFn>(resolve_symbol(
b"chanora_silero_vad_create\0",
)?),
destroy: std::mem::transmute::<*mut c_void, DestroyFn>(resolve_symbol(
b"chanora_silero_vad_destroy\0",
)?),
reset: std::mem::transmute::<*mut c_void, ResetFn>(resolve_symbol(
b"chanora_silero_vad_reset\0",
)?),
process: std::mem::transmute::<*mut c_void, ProcessFn>(resolve_symbol(
b"chanora_silero_vad_process\0",
)?),
last_error: std::mem::transmute::<*mut c_void, LastErrorFn>(resolve_symbol(
b"chanora_silero_vad_last_error\0",
)?),
free_string: std::mem::transmute::<*mut c_void, FreeStringFn>(resolve_symbol(
b"chanora_silero_vad_free_string\0",
)?),
})
}
}
fn last_error_message(&self) -> String {
unsafe {
let ptr = (self.last_error)();
if ptr.is_null() {
return "unknown SileroCoreML bridge error".to_string();
}
let message = CStr::from_ptr(ptr).to_string_lossy().into_owned();
(self.free_string)(ptr);
message
}
}
}
unsafe fn resolve_symbol(name: &'static [u8]) -> Option<*mut c_void> {
let ptr = dlsym(RTLD_DEFAULT, name.as_ptr().cast());
if ptr.is_null() {
None
} else {
Some(ptr)
}
}
/// 16 kHz detector backed by Swift `SileroCoreML.SileroVAD`.
pub struct AppleCoreMlVad {
handle: *mut c_void,
symbols: AppleSileroSymbols,
accum: Vec<f32>,
last_probability: f32,
}
impl AppleCoreMlVad {
/// Create a detector when the Swift Runner bridge symbols are linked.
pub fn try_new() -> Option<Self> {
let symbols = AppleSileroSymbols::resolve()?;
Self::try_new_with_symbols(symbols)
}
fn try_new_with_symbols(symbols: AppleSileroSymbols) -> Option<Self> {
let handle = unsafe { (symbols.create)() };
if handle.is_null() {
tracing::warn!(
target: "chanora_audio",
error = %symbols.last_error_message(),
"AppleCoreMlVad: Swift SileroCoreML bridge unavailable; falling back to WebRtcFallbackVad"
);
return None;
}
tracing::info!(
target: "chanora_audio",
backend = "apple_coreml",
model = "silero_vad",
model_version = "6.2.1",
model_resource = "SileroVADModel",
model_artifact = "mlmodelc_or_mlpackage",
sample_rate_hz = 16_000,
chunk_size = SILERO_COREML_FRAME_16K,
threshold = SILERO_COREML_THRESHOLD,
"AppleCoreMlVad: using Apple CoreML Silero VAD"
);
Some(Self {
handle,
symbols,
accum: Vec::with_capacity(SILERO_COREML_FRAME_16K),
last_probability: 0.0,
})
}
/// Reset accumulated samples, last probability, and the Swift VAD stream.
pub fn reset_state(&mut self) {
self.accum.clear();
self.last_probability = 0.0;
let rc = unsafe { (self.symbols.reset)(self.handle) };
if rc != 0 {
tracing::warn!(
target: "chanora_audio",
error = %self.symbols.last_error_message(),
"AppleCoreMlVad: reset failed"
);
}
}
fn calc_level(&mut self, audio_frame: &[f32]) -> f32 {
debug_assert_eq!(audio_frame.len(), SILERO_COREML_FRAME_16K);
let mut probability = self.last_probability;
let rc = unsafe {
(self.symbols.process)(
self.handle,
audio_frame.as_ptr(),
audio_frame.len(),
&mut probability,
)
};
if rc == 0 {
self.last_probability = probability.clamp(0.0, 1.0);
} else {
tracing::warn!(
target: "chanora_audio",
error = %self.symbols.last_error_message(),
"AppleCoreMlVad: inference failed; holding last probability"
);
}
self.last_probability
}
}
impl VoiceActivityDetector for AppleCoreMlVad {
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
debug_assert_eq!(
samples.len(),
super::resampler::OUTPUT_FRAME_10MS,
"AppleCoreMlVad expects 160 samples (16 kHz 10 ms), got {}",
samples.len()
);
self.accum.extend_from_slice(samples);
if self.accum.len() >= SILERO_COREML_FRAME_16K {
let audio_frame: Vec<f32> = self.accum[..SILERO_COREML_FRAME_16K].to_vec();
self.calc_level(&audio_frame);
let overflow: Vec<f32> = self.accum.drain(SILERO_COREML_FRAME_16K..).collect();
self.accum.clear();
self.accum.extend_from_slice(&overflow);
}
VadOutput {
probability: self.last_probability,
speech: self.last_probability >= SILERO_COREML_THRESHOLD,
}
}
}
impl Drop for AppleCoreMlVad {
fn drop(&mut self) {
unsafe { (self.symbols.destroy)(self.handle) };
}
}
// SAFETY: the opaque Swift object is owned by this detector and only used by
// the worker thread after construction. It is never shared concurrently.
unsafe impl Send for AppleCoreMlVad {}
struct SileroFrameMessage {
seq: u64,
frame: [f32; super::resampler::INPUT_FRAME_10MS],
}
/// Background Apple/CoreML Silero worker.
pub struct AppleCoreMlVadWorker {
tx: Option<std::sync::mpsc::SyncSender<SileroFrameMessage>>,
latest_probability: Arc<AtomicU32>,
latest_processed_seq: Arc<AtomicU64>,
alive: Arc<AtomicBool>,
handle: Option<JoinHandle<()>>,
}
impl AppleCoreMlVadWorker {
/// Start the background CoreML worker when the Swift bridge is available.
pub fn try_new() -> Option<Self> {
let symbols = AppleSileroSymbols::resolve()?;
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(64);
let latest_probability_for_thread = latest_probability.clone();
let latest_processed_seq_for_thread = latest_processed_seq.clone();
let alive_for_thread = alive.clone();
let handle = std::thread::Builder::new()
.name("chanora-apple-silero-vad".to_string())
.spawn(move || {
let Some(vad) = AppleCoreMlVad::try_new_with_symbols(symbols) else {
return;
};
let mut vad = super::Resampled16kHzVad::new(vad);
while alive_for_thread.load(Ordering::Relaxed) {
let message = match rx.recv() {
Ok(message) => message,
Err(_) => break,
};
let output = vad.process_10ms(&message.frame);
latest_probability_for_thread.store(
output.probability.clamp(0.0, 1.0).to_bits(),
Ordering::Relaxed,
);
latest_processed_seq_for_thread.store(message.seq, Ordering::Relaxed);
}
})
.ok()?;
Some(Self {
tx: Some(tx),
latest_probability,
latest_processed_seq,
alive,
handle: Some(handle),
})
}
/// Enqueue one 48 kHz 10 ms frame without blocking the caller.
pub fn try_send(&self, seq: u64, frame: &[f32; super::resampler::INPUT_FRAME_10MS]) -> bool {
let Some(tx) = &self.tx else {
return false;
};
tx.try_send(SileroFrameMessage { seq, frame: *frame })
.is_ok()
}
/// Return the latest probability published by the worker thread.
pub fn latest_probability(&self) -> f32 {
f32::from_bits(self.latest_probability.load(Ordering::Relaxed))
}
/// Return true until the worker has produced a recent probability.
pub fn is_stale(&self, capture_seq: u64) -> bool {
let latest = self.latest_processed_seq.load(Ordering::Relaxed);
latest == u64::MAX || capture_seq.saturating_sub(latest) > SILERO_COREML_MAX_STALE_FRAMES
}
}
impl Drop for AppleCoreMlVadWorker {
fn drop(&mut self) {
self.alive.store(false, Ordering::Relaxed);
let _ = self.tx.take();
// Drop can run from the realtime audio callback during backend changes;
// never join here. Closing tx lets the worker exit and dropping the
// handle detaches the thread without blocking the callback.
let _ = self.handle.take();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn coreml_constants_match_silero_package_contract() {
assert_eq!(SILERO_COREML_FRAME_16K, 512);
assert_eq!(SILERO_COREML_MAX_STALE_FRAMES, 3);
}
#[test]
fn worker_is_unavailable_without_swift_bridge_symbols_on_host_tests() {
assert!(AppleCoreMlVadWorker::try_new().is_none());
}
#[test]
fn vad_is_unavailable_without_swift_bridge_symbols_on_host_tests() {
assert!(AppleCoreMlVad::try_new().is_none());
}
}
crates/chanora_audio/src/vad/mod.rs
+2
View File
@@ -5,6 +5,8 @@
//! platforms may use a model-backed detector when available so
//! VoiceActivity mode never collapses back to Continuous transmit.
#[cfg(any(target_os = "ios", target_os = "macos"))]
pub mod apple_coreml;
pub mod resampler;
#[cfg(not(target_os = "ios"))]
pub mod silero_onnx;