feat(audio): add Apple CoreML Silero VAD

crates/chanora_audio/src/vad/apple_coreml.rs

@@ -0,0 +1,318 @@
+//! Apple/CoreML Silero VAD bridge.
+//!
+//! The Swift Runner target exports a tiny C ABI around
+//! `SileroCoreML.SileroVAD`. This Rust side resolves those symbols at
+//! runtime, then runs inference on a background worker so realtime CoreAudio
+//! callbacks only enqueue frames and read atomics.
+
+use super::{VadOutput, VoiceActivityDetector};
+use std::ffi::{c_char, c_void, CStr};
+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
+use std::sync::Arc;
+use std::thread::JoinHandle;
+
+/// 16 kHz frame size required by `SileroCoreML.SileroVAD.process(_:)`.
+pub const SILERO_COREML_FRAME_16K: usize = 512;
+/// Maximum lag in 10 ms frames before the realtime callback falls back.
+pub const SILERO_COREML_MAX_STALE_FRAMES: u64 = 3;
+
+const SILERO_COREML_THRESHOLD: f32 = 0.5;
+const RTLD_DEFAULT: *mut c_void = -2_isize as *mut c_void;
+
+type CreateFn = unsafe extern "C" fn() -> *mut c_void;
+type DestroyFn = unsafe extern "C" fn(*mut c_void);
+type ResetFn = unsafe extern "C" fn(*mut c_void) -> i32;
+type ProcessFn = unsafe extern "C" fn(*mut c_void, *const f32, usize, *mut f32) -> i32;
+type LastErrorFn = unsafe extern "C" fn() -> *mut c_char;
+type FreeStringFn = unsafe extern "C" fn(*mut c_char);
+
+#[allow(improper_ctypes)]
+extern "C" {
+    fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;
+}
+
+#[derive(Clone, Copy)]
+struct AppleSileroSymbols {
+    create: CreateFn,
+    destroy: DestroyFn,
+    reset: ResetFn,
+    process: ProcessFn,
+    last_error: LastErrorFn,
+    free_string: FreeStringFn,
+}
+
+impl AppleSileroSymbols {
+    fn resolve() -> Option<Self> {
+        unsafe {
+            Some(Self {
+                create: std::mem::transmute::<*mut c_void, CreateFn>(resolve_symbol(
+                    b"chanora_silero_vad_create\0",
+                )?),
+                destroy: std::mem::transmute::<*mut c_void, DestroyFn>(resolve_symbol(
+                    b"chanora_silero_vad_destroy\0",
+                )?),
+                reset: std::mem::transmute::<*mut c_void, ResetFn>(resolve_symbol(
+                    b"chanora_silero_vad_reset\0",
+                )?),
+                process: std::mem::transmute::<*mut c_void, ProcessFn>(resolve_symbol(
+                    b"chanora_silero_vad_process\0",
+                )?),
+                last_error: std::mem::transmute::<*mut c_void, LastErrorFn>(resolve_symbol(
+                    b"chanora_silero_vad_last_error\0",
+                )?),
+                free_string: std::mem::transmute::<*mut c_void, FreeStringFn>(resolve_symbol(
+                    b"chanora_silero_vad_free_string\0",
+                )?),
+            })
+        }
+    }
+
+    fn last_error_message(&self) -> String {
+        unsafe {
+            let ptr = (self.last_error)();
+            if ptr.is_null() {
+                return "unknown SileroCoreML bridge error".to_string();
+            }
+            let message = CStr::from_ptr(ptr).to_string_lossy().into_owned();
+            (self.free_string)(ptr);
+            message
+        }
+    }
+}
+
+unsafe fn resolve_symbol(name: &'static [u8]) -> Option<*mut c_void> {
+    let ptr = dlsym(RTLD_DEFAULT, name.as_ptr().cast());
+    if ptr.is_null() {
+        None
+    } else {
+        Some(ptr)
+    }
+}
+
+/// 16 kHz detector backed by Swift `SileroCoreML.SileroVAD`.
+pub struct AppleCoreMlVad {
+    handle: *mut c_void,
+    symbols: AppleSileroSymbols,
+    accum: Vec<f32>,
+    last_probability: f32,
+}
+
+impl AppleCoreMlVad {
+    /// Create a detector when the Swift Runner bridge symbols are linked.
+    pub fn try_new() -> Option<Self> {
+        let symbols = AppleSileroSymbols::resolve()?;
+        Self::try_new_with_symbols(symbols)
+    }
+
+    fn try_new_with_symbols(symbols: AppleSileroSymbols) -> Option<Self> {
+        let handle = unsafe { (symbols.create)() };
+        if handle.is_null() {
+            tracing::warn!(
+                target: "chanora_audio",
+                error = %symbols.last_error_message(),
+                "AppleCoreMlVad: Swift SileroCoreML bridge unavailable; falling back to WebRtcFallbackVad"
+            );
+            return None;
+        }
+        tracing::info!(
+            target: "chanora_audio",
+            backend = "apple_coreml",
+            model = "silero_vad",
+            model_version = "6.2.1",
+            model_resource = "SileroVADModel",
+            model_artifact = "mlmodelc_or_mlpackage",
+            sample_rate_hz = 16_000,
+            chunk_size = SILERO_COREML_FRAME_16K,
+            threshold = SILERO_COREML_THRESHOLD,
+            "AppleCoreMlVad: using Apple CoreML Silero VAD"
+        );
+        Some(Self {
+            handle,
+            symbols,
+            accum: Vec::with_capacity(SILERO_COREML_FRAME_16K),
+            last_probability: 0.0,
+        })
+    }
+
+    /// Reset accumulated samples, last probability, and the Swift VAD stream.
+    pub fn reset_state(&mut self) {
+        self.accum.clear();
+        self.last_probability = 0.0;
+        let rc = unsafe { (self.symbols.reset)(self.handle) };
+        if rc != 0 {
+            tracing::warn!(
+                target: "chanora_audio",
+                error = %self.symbols.last_error_message(),
+                "AppleCoreMlVad: reset failed"
+            );
+        }
+    }
+
+    fn calc_level(&mut self, audio_frame: &[f32]) -> f32 {
+        debug_assert_eq!(audio_frame.len(), SILERO_COREML_FRAME_16K);
+        let mut probability = self.last_probability;
+        let rc = unsafe {
+            (self.symbols.process)(
+                self.handle,
+                audio_frame.as_ptr(),
+                audio_frame.len(),
+                &mut probability,
+            )
+        };
+        if rc == 0 {
+            self.last_probability = probability.clamp(0.0, 1.0);
+        } else {
+            tracing::warn!(
+                target: "chanora_audio",
+                error = %self.symbols.last_error_message(),
+                "AppleCoreMlVad: inference failed; holding last probability"
+            );
+        }
+        self.last_probability
+    }
+}
+
+impl VoiceActivityDetector for AppleCoreMlVad {
+    fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
+        debug_assert_eq!(
+            samples.len(),
+            super::resampler::OUTPUT_FRAME_10MS,
+            "AppleCoreMlVad expects 160 samples (16 kHz 10 ms), got {}",
+            samples.len()
+        );
+
+        self.accum.extend_from_slice(samples);
+        if self.accum.len() >= SILERO_COREML_FRAME_16K {
+            let audio_frame: Vec<f32> = self.accum[..SILERO_COREML_FRAME_16K].to_vec();
+            self.calc_level(&audio_frame);
+            let overflow: Vec<f32> = self.accum.drain(SILERO_COREML_FRAME_16K..).collect();
+            self.accum.clear();
+            self.accum.extend_from_slice(&overflow);
+        }
+
+        VadOutput {
+            probability: self.last_probability,
+            speech: self.last_probability >= SILERO_COREML_THRESHOLD,
+        }
+    }
+}
+
+impl Drop for AppleCoreMlVad {
+    fn drop(&mut self) {
+        unsafe { (self.symbols.destroy)(self.handle) };
+    }
+}
+
+// SAFETY: the opaque Swift object is owned by this detector and only used by
+// the worker thread after construction. It is never shared concurrently.
+unsafe impl Send for AppleCoreMlVad {}
+
+struct SileroFrameMessage {
+    seq: u64,
+    frame: [f32; super::resampler::INPUT_FRAME_10MS],
+}
+
+/// Background Apple/CoreML Silero worker.
+pub struct AppleCoreMlVadWorker {
+    tx: Option<std::sync::mpsc::SyncSender<SileroFrameMessage>>,
+    latest_probability: Arc<AtomicU32>,
+    latest_processed_seq: Arc<AtomicU64>,
+    alive: Arc<AtomicBool>,
+    handle: Option<JoinHandle<()>>,
+}
+
+impl AppleCoreMlVadWorker {
+    /// Start the background CoreML worker when the Swift bridge is available.
+    pub fn try_new() -> Option<Self> {
+        let symbols = AppleSileroSymbols::resolve()?;
+        let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
+        let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
+        let alive = Arc::new(AtomicBool::new(true));
+        let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(64);
+        let latest_probability_for_thread = latest_probability.clone();
+        let latest_processed_seq_for_thread = latest_processed_seq.clone();
+        let alive_for_thread = alive.clone();
+
+        let handle = std::thread::Builder::new()
+            .name("chanora-apple-silero-vad".to_string())
+            .spawn(move || {
+                let Some(vad) = AppleCoreMlVad::try_new_with_symbols(symbols) else {
+                    return;
+                };
+                let mut vad = super::Resampled16kHzVad::new(vad);
+                while alive_for_thread.load(Ordering::Relaxed) {
+                    let message = match rx.recv() {
+                        Ok(message) => message,
+                        Err(_) => break,
+                    };
+                    let output = vad.process_10ms(&message.frame);
+                    latest_probability_for_thread.store(
+                        output.probability.clamp(0.0, 1.0).to_bits(),
+                        Ordering::Relaxed,
+                    );
+                    latest_processed_seq_for_thread.store(message.seq, Ordering::Relaxed);
+                }
+            })
+            .ok()?;
+
+        Some(Self {
+            tx: Some(tx),
+            latest_probability,
+            latest_processed_seq,
+            alive,
+            handle: Some(handle),
+        })
+    }
+
+    /// Enqueue one 48 kHz 10 ms frame without blocking the caller.
+    pub fn try_send(&self, seq: u64, frame: &[f32; super::resampler::INPUT_FRAME_10MS]) -> bool {
+        let Some(tx) = &self.tx else {
+            return false;
+        };
+        tx.try_send(SileroFrameMessage { seq, frame: *frame })
+            .is_ok()
+    }
+
+    /// Return the latest probability published by the worker thread.
+    pub fn latest_probability(&self) -> f32 {
+        f32::from_bits(self.latest_probability.load(Ordering::Relaxed))
+    }
+
+    /// Return true until the worker has produced a recent probability.
+    pub fn is_stale(&self, capture_seq: u64) -> bool {
+        let latest = self.latest_processed_seq.load(Ordering::Relaxed);
+        latest == u64::MAX || capture_seq.saturating_sub(latest) > SILERO_COREML_MAX_STALE_FRAMES
+    }
+}
+
+impl Drop for AppleCoreMlVadWorker {
+    fn drop(&mut self) {
+        self.alive.store(false, Ordering::Relaxed);
+        let _ = self.tx.take();
+        // Drop can run from the realtime audio callback during backend changes;
+        // never join here. Closing tx lets the worker exit and dropping the
+        // handle detaches the thread without blocking the callback.
+        let _ = self.handle.take();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn coreml_constants_match_silero_package_contract() {
+        assert_eq!(SILERO_COREML_FRAME_16K, 512);
+        assert_eq!(SILERO_COREML_MAX_STALE_FRAMES, 3);
+    }
+
+    #[test]
+    fn worker_is_unavailable_without_swift_bridge_symbols_on_host_tests() {
+        assert!(AppleCoreMlVadWorker::try_new().is_none());
+    }
+
+    #[test]
+    fn vad_is_unavailable_without_swift_bridge_symbols_on_host_tests() {
+        assert!(AppleCoreMlVad::try_new().is_none());
+    }
+}

crates/chanora_audio/src/vad/mod.rs

@@ -5,6 +5,8 @@
 //! platforms may use a model-backed detector when available so
 //! VoiceActivity mode never collapses back to Continuous transmit.
 
+#[cfg(any(target_os = "ios", target_os = "macos"))]
+pub mod apple_coreml;
 pub mod resampler;
 #[cfg(not(target_os = "ios"))]
 pub mod silero_onnx;