edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(audio): use WebRTC VAD on iOS

Browse Source
This commit is contained in:
Edison Jwa
2026-05-31 22:30:28 +09:00
parent c02d4e6df3
commit 6f7063971d
6 changed files with 119 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/ios_voice_unit.rs
+72 -109
View File
@@ -48,20 +48,15 @@
//!
//! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the
//! AudioUnit internally holds the C `AudioUnit` opaque pointer and
//! the wrapper's destructor calls `AudioComponentInstanceDispose`,
//! which (per Apple's threading rules) must be called from the
//! thread that owns the unit. We open the unit on the same thread
//! that calls `Self::start` (the tokio worker that runs
//! `chanora_core::ChanoraSession::start_audio`, the same pattern
//! cpal + SDL use) and never move it. The outer `AudioEngine`
//! already carries an `unsafe impl Send` to satisfy the same
//! constraint for cpal's `!Send` Stream type; that impl covers
//! VPIO too.
//!
//! Dropping `IosVoiceUnit` calls `audio_unit.stop()` via the
//! wrapper's `Drop`, which detaches the render + input callbacks
//! and stops the unit. The AudioHandler + CaptureState `Arc`s the
//! callbacks held are then released.
//! the wrapper's destructor calls `AudioComponentInstanceDispose`.
//! `IosVoiceUnit` stores the unit as `Option<AudioUnit>` so initial
//! setup and lifecycle operations can dispatch CoreAudio
//! initialize/start/stop calls to `DispatchQueue::main()` while the
//! wrapper keeps ownership and preserves the render/input callback
//! state in the surrounding `Arc`s. `restart`, `pause`, and
//! `resume` may temporarily move the unit through that helper, and
//! `Drop` stops it if still present before those callback `Arc`s are
//! released.
//!
//! ## What this file does NOT do
//!
@@ -143,13 +138,8 @@ struct IosCaptureState {
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
/// Background Silero worker — enqueues frames off the realtime
/// callback and publishes the latest probability atomically.
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
fallback_warned_backend: Option<crate::VadBackend>,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
@@ -187,9 +177,7 @@ impl IosCaptureState {
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config: params.audio_processing_config.clone(),
@@ -349,7 +337,7 @@ impl IosCaptureState {
false,
false,
true,
crate::VadBackend::SileroOnnx,
crate::VadBackend::WebrtcVad,
crate::voice_activity::VAD_HANGOVER_MS,
false,
crate::AudioRoute::Unknown,
@@ -362,18 +350,12 @@ impl IosCaptureState {
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_vad_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
// Switch VAD backend only while VoiceActivity mode is active.
let silero_model_epoch = crate::vad::silero_model_epoch();
let silero_model_changed = voice_activity_mode
&& vad_backend == crate::VadBackend::SileroOnnx
&& silero_model_epoch != self.silero_model_epoch;
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
if debug_wav_dump_enabled {
if recorder_guard.is_none() {
@@ -387,35 +369,14 @@ impl IosCaptureState {
}
}
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_model_changed)
{
if voice_activity_mode && vad_backend != self.current_vad_backend {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
self.silero_model_epoch = silero_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
// Attempt to load Silero model from the well-known
// bundle path. The actual inference runs on a
// background worker; the callback only enqueues
// 10 ms frames and falls back to WebRTC if the
// worker is missing or stale.
let model_path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path);
if self.silero_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"Silero VAD model not found at {model_path}; falling back to WebRTC VAD"
);
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
}
self.audio_processing_stats
.set_vad_fallback_active(self.silero_vad_worker.is_none());
}
_ => {
self.silero_vad_worker = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
if vad_backend == crate::VadBackend::SileroOnnx {
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
} else {
self.audio_processing_stats.set_vad_fallback_active(false);
}
// Reset VAD state machine timers on backend switch.
self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new(
@@ -457,7 +418,6 @@ impl IosCaptureState {
// VAD: only evaluate while VoiceActivity mode is active.
let (vad_probability, gate_open) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
@@ -465,32 +425,9 @@ impl IosCaptureState {
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
@@ -578,10 +515,39 @@ pub struct IosVoiceUnit {
// Drop = stop the audio unit (severs render callback). The
// wrapper's own Drop calls AudioComponentInstanceDispose
// after stop returns.
unit: AudioUnit,
unit: Option<AudioUnit>,
}
impl IosVoiceUnit {
#[cfg(target_os = "ios")]
fn exec_main_queue_lifecycle(
&mut self,
op: impl FnOnce(&mut AudioUnit) -> Result<(), String> + Send + 'static,
) -> Result<(), AudioError> {
let unit = self
.unit
.take()
.ok_or_else(|| AudioError::Backend("vpio lifecycle: audio unit missing".to_string()))?;
let (tx, rx) = std::sync::mpsc::sync_channel::<Result<(), String>>(1);
let unit_arc = Arc::new(Mutex::new(Some(unit)));
let unit_arc2 = Arc::clone(&unit_arc);
dispatch2::DispatchQueue::main().exec_async(move || {
let mut guard = unit_arc2.lock().unwrap();
let unit = guard.as_mut().unwrap();
let _ = tx.send(op(unit));
});
let result = match rx.recv() {
Ok(result) => result,
Err(_) => Err("vpio lifecycle: main thread channel closed unexpectedly".to_string()),
};
self.unit = unit_arc.lock().unwrap().take();
result.map_err(AudioError::Backend)
}
/// Open a VoiceProcessingIO AudioUnit, pin its stream format
/// to 48 kHz Int16 mono on both buses, install render + input
/// callbacks, and start it. The unit begins pumping audio
@@ -994,7 +960,7 @@ impl IosVoiceUnit {
),
}
Ok(Self { unit })
Ok(Self { unit: Some(unit) })
}
/// Restart the audio unit after route change handling.
@@ -1003,23 +969,18 @@ impl IosVoiceUnit {
/// VoiceProcessingIO unit through an uninitialize/reinitialize
/// cycle, then start again.
///
/// Called from the Flutter method channel handler which runs on
/// the main isolate — that runs on the main thread — so the
/// CoreAudio RPC is already on the correct thread here.
#[cfg(target_os = "ios")]
pub fn restart(&mut self) -> Result<(), AudioError> {
self.unit
.stop()
.map_err(|e| AudioError::Backend(format!("vpio restart stop: {e}")))?;
self.unit
.uninitialize()
.map_err(|e| AudioError::Backend(format!("vpio restart uninit: {e}")))?;
self.unit
.initialize()
.map_err(|e| AudioError::Backend(format!("vpio restart init: {e}")))?;
self.unit
.start()
.map_err(|e| AudioError::Backend(format!("vpio restart start: {e}")))?;
self.exec_main_queue_lifecycle(|unit| {
unit.stop().map_err(|e| format!("vpio restart stop: {e}"))?;
unit.uninitialize()
.map_err(|e| format!("vpio restart uninit: {e}"))?;
unit.initialize()
.map_err(|e| format!("vpio restart init: {e}"))?;
unit.start()
.map_err(|e| format!("vpio restart start: {e}"))?;
Ok(())
})?;
info!(target: "chanora_audio", "ios VPIO audio unit restarted");
Ok(())
}
@@ -1027,17 +988,17 @@ impl IosVoiceUnit {
/// Pause the audio unit during an interruption.
#[cfg(target_os = "ios")]
pub fn pause(&mut self) -> Result<(), AudioError> {
self.unit
.stop()
.map_err(|e| AudioError::Backend(format!("vpio pause stop: {e}")))
self.exec_main_queue_lifecycle(|unit| {
unit.stop().map_err(|e| format!("vpio pause stop: {e}"))
})
}
/// Resume the audio unit after an interruption.
#[cfg(target_os = "ios")]
pub fn resume(&mut self) -> Result<(), AudioError> {
self.unit
.start()
.map_err(|e| AudioError::Backend(format!("vpio resume start: {e}")))
self.exec_main_queue_lifecycle(|unit| {
unit.start().map_err(|e| format!("vpio resume start: {e}"))
})
}
}
@@ -1046,10 +1007,12 @@ impl Drop for IosVoiceUnit {
// Stop the audio unit so the render callback no longer
// fires. The coreaudio-rs wrapper's own Drop calls
// AudioComponentInstanceDispose afterwards.
if let Err(e) = self.unit.stop() {
warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed");
} else {
info!(target: "chanora_audio", "ios audio unit stopped");
if let Some(unit) = self.unit.as_mut() {
if let Err(e) = unit.stop() {
warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed");
} else {
info!(target: "chanora_audio", "ios audio unit stopped");
}
}
}
}