From d39d6c78d26b116db4f3209b7fed685e36e9a3e4 Mon Sep 17 00:00:00 2001 From: Edison Jwa Date: Sun, 31 May 2026 22:29:30 +0900 Subject: [PATCH 1/5] fix(core): unblock iOS connect audio startup --- core/chanora_core/src/lib.rs | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/core/chanora_core/src/lib.rs b/core/chanora_core/src/lib.rs index ee742aa..670517d 100644 --- a/core/chanora_core/src/lib.rs +++ b/core/chanora_core/src/lib.rs @@ -891,15 +891,18 @@ impl ChanoraSession { if let Some(projection) = projection { self.emit_voice_state(projection).await; } - if let Err(audio_err) = self.ensure_audio_running().await { - warn!( - target: "chanora_core", - error = %audio_err, - "auto-join default channel: server placed us in a channel but audio engine \ - failed to start; continuing with no-audio in-channel state" - ); - let _ = self.events_tx.send(SessionEvent::AudioStopped); - } + let session = self.clone(); + tokio::spawn(async move { + if let Err(audio_err) = session.ensure_audio_running().await { + warn!( + target: "chanora_core", + error = %audio_err, + "auto-join default channel: server placed us in a channel but audio engine \ + failed to start; continuing with no-audio in-channel state" + ); + let _ = session.events_tx.send(SessionEvent::AudioStopped); + } + }); } Ok(snap) @@ -1388,11 +1391,12 @@ impl ChanoraSession { Some(audio.audio_processing_stats()) } - /// Configure the preferred Silero ONNX VAD model path. + /// Configure the preferred Silero ONNX VAD model path on platforms + /// that ship the ONNX detector. /// - /// This does not require an active connection. Running iOS audio - /// backends observe the model-path epoch and reload on the next - /// capture frame when Silero is selected. + /// This does not require an active connection. Running non-iOS + /// audio backends can observe the model-path epoch and reload on + /// the next capture frame when Silero is selected. pub async fn set_vad_model_path(&self, path: String) -> Result<(), CoreError> { chanora_audio::vad::set_silero_model_path(&path)?; Ok(()) @@ -1404,8 +1408,10 @@ impl ChanoraSession { let guard = self.inner.lock().await; if let Some(state) = guard.as_ref() { if let Some(audio) = state.audio.as_ref() { - let mut config = audio.audio_processing_config_snapshot(); - config.route = route; + let config = chanora_audio::route_policy::apply_route_change( + &audio.audio_processing_config_snapshot(), + route, + ); audio.set_audio_processing_config(config)?; #[cfg(any(target_os = "ios", target_os = "macos"))] { From c02d4e6df3ea26c4fe49021a17c81ccf3a820d0f Mon Sep 17 00:00:00 2001 From: Edison Jwa Date: Sun, 31 May 2026 22:29:38 +0900 Subject: [PATCH 2/5] fix(bridge): serialize iOS audio lifecycle events --- crates/chanora_bridge/Cargo.toml | 2 +- crates/chanora_bridge/src/api.rs | 100 +++++++++++++++++++++++-------- 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/crates/chanora_bridge/Cargo.toml b/crates/chanora_bridge/Cargo.toml index 1ee3b6b..2c3e72d 100644 --- a/crates/chanora_bridge/Cargo.toml +++ b/crates/chanora_bridge/Cargo.toml @@ -23,7 +23,7 @@ thiserror.workspace = true serde.workspace = true tracing.workspace = true tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] } # Android: route `tracing` output to logcat so a user can see protocol # and audio diagnostics via `adb logcat -s chanora`. Also brings in the diff --git a/crates/chanora_bridge/src/api.rs b/crates/chanora_bridge/src/api.rs index f42ca38..196d0b0 100644 --- a/crates/chanora_bridge/src/api.rs +++ b/crates/chanora_bridge/src/api.rs @@ -11,7 +11,7 @@ use std::time::Duration; use flutter_rust_bridge::frb; use tokio::runtime::Runtime; -use tokio::sync::broadcast; +use tokio::sync::{broadcast, mpsc}; use tracing::{info, warn}; use crate::frb_generated::StreamSink; @@ -42,6 +42,73 @@ fn task_join_error(task: &'static str, error: tokio::task::JoinError) -> BridgeE BridgeError::Unmapped(format!("join: {error}")) } +enum PlatformAudioEvent { + RouteChanged(chanora_audio::AudioRoute), + MediaServicesResetWithRoute(chanora_audio::AudioRoute), + InterruptionBegan, + InterruptionEnded { should_resume: bool }, + AudioOutputRoute(chanora_audio::AudioRoute), + Lifecycle { state: String }, +} + +impl PlatformAudioEvent { + async fn process(self) { + match self { + Self::RouteChanged(route) => { + if let Err(e) = session().ios_handle_route_change(route).await { + warn!(target: "chanora_bridge", error = %e, "iOS route-change handling failed"); + } + } + Self::MediaServicesResetWithRoute(route) => { + if let Err(e) = session().ios_handle_media_services_reset(route).await { + warn!(target: "chanora_bridge", error = %e, "iOS media-services reset (with route) handling failed"); + } + } + Self::InterruptionBegan => { + if let Err(e) = session().ios_handle_interruption_began().await { + warn!(target: "chanora_bridge", error = %e, "iOS interruption-began handling failed"); + } + } + Self::InterruptionEnded { should_resume } => { + if let Err(e) = session().ios_handle_interruption_ended(should_resume).await { + warn!(target: "chanora_bridge", error = %e, "iOS interruption-ended handling failed"); + } + } + Self::AudioOutputRoute(route) => { + if let Err(e) = session().ios_handle_route_change(route).await { + warn!(target: "chanora_bridge", error = %e, "audio output route handling failed"); + } + } + Self::Lifecycle { state } => { + session().record_lifecycle_event(&state).await; + } + } + } +} + +fn platform_audio_events() -> &'static mpsc::UnboundedSender { + static TX: OnceLock> = OnceLock::new(); + TX.get_or_init(|| { + let (tx, mut rx) = mpsc::unbounded_channel::(); + runtime().spawn(async move { + while let Some(event) = rx.recv().await { + event.process().await; + } + }); + tx + }) +} + +fn dispatch_platform_audio_event(event: PlatformAudioEvent) { + if let Err(e) = platform_audio_events().send(event) { + warn!( + target: "chanora_bridge", + error = %e, + "ordered platform audio event dispatch failed" + ); + } +} + fn install_panic_diagnostic_hook() { static INSTALLED: OnceLock<()> = OnceLock::new(); let _ = INSTALLED.get_or_init(|| { @@ -615,11 +682,7 @@ pub async fn is_connected() -> bool { /// Handle iOS AVAudioSession route changes (SDD-100). #[frb(sync)] pub fn handle_route_change(route: BridgeAudioRoute) { - let result = - runtime().block_on(async { session().ios_handle_route_change(route.into()).await }); - if let Err(e) = result { - warn!(target: "chanora_bridge", error = %e, "iOS route-change handling failed"); - } + dispatch_platform_audio_event(PlatformAudioEvent::RouteChanged(route.into())); } /// Handle iOS AVAudioSession media-services reset with the current @@ -629,30 +692,19 @@ pub fn handle_route_change(route: BridgeAudioRoute) { #[frb(sync)] pub fn handle_media_services_reset_with_route(route_class: String) { let route = chanora_audio::AudioRoute::from_route_class(&route_class); - let result = - runtime().block_on(async { session().ios_handle_media_services_reset(route).await }); - if let Err(e) = result { - warn!(target: "chanora_bridge", error = %e, "iOS media-services reset (with route) handling failed"); - } + dispatch_platform_audio_event(PlatformAudioEvent::MediaServicesResetWithRoute(route)); } /// Handle iOS AVAudioSession interruption begin (SDD-101). #[frb(sync)] pub fn handle_interruption_began() { - let result = runtime().block_on(async { session().ios_handle_interruption_began().await }); - if let Err(e) = result { - warn!(target: "chanora_bridge", error = %e, "iOS interruption-began handling failed"); - } + dispatch_platform_audio_event(PlatformAudioEvent::InterruptionBegan); } /// Handle iOS AVAudioSession interruption end (SDD-101). #[frb(sync)] pub fn handle_interruption_ended(should_resume: bool) { - let result = - runtime().block_on(async { session().ios_handle_interruption_ended(should_resume).await }); - if let Err(e) = result { - warn!(target: "chanora_bridge", error = %e, "iOS interruption-ended handling failed"); - } + dispatch_platform_audio_event(PlatformAudioEvent::InterruptionEnded { should_resume }); } /// Set the focused/on-screen push-to-talk hold state. @@ -2139,14 +2191,10 @@ pub async fn set_ios_voice_processing_mode( /// Set the preferred audio output route (Android/iOS). #[frb(sync)] pub fn set_audio_output_route(route: BridgeAudioRoute) { - runtime().block_on(async { - let _ = session().ios_handle_route_change(route.into()).await; - }); + dispatch_platform_audio_event(PlatformAudioEvent::AudioOutputRoute(route.into())); } /// Called from Flutter when the app enters background/foreground. #[frb(sync)] pub fn record_lifecycle_event(state: String) { - runtime().block_on(async { - session().record_lifecycle_event(&state).await; - }); + dispatch_platform_audio_event(PlatformAudioEvent::Lifecycle { state }); } From 6f7063971d8c359dbabc61d7c1db103b49814727 Mon Sep 17 00:00:00 2001 From: Edison Jwa Date: Sun, 31 May 2026 22:30:28 +0900 Subject: [PATCH 3/5] fix(audio): use WebRTC VAD on iOS --- crates/chanora_audio/Cargo.toml | 8 +- crates/chanora_audio/src/audio_processing.rs | 15 +- crates/chanora_audio/src/ios_raw_unit.rs | 66 +------ crates/chanora_audio/src/ios_voice_unit.rs | 181 ++++++++----------- crates/chanora_audio/src/route_policy.rs | 13 +- crates/chanora_audio/src/vad/mod.rs | 25 +-- 6 files changed, 119 insertions(+), 189 deletions(-) diff --git a/crates/chanora_audio/Cargo.toml b/crates/chanora_audio/Cargo.toml index a66567c..d94642a 100644 --- a/crates/chanora_audio/Cargo.toml +++ b/crates/chanora_audio/Cargo.toml @@ -17,7 +17,7 @@ sonora = "0.1" webrtc-vad = "0.4" # ndarray is required by ort's tensor construction API and by -# Silero VAD ONNX inference across all platforms. +# Silero VAD ONNX inference on non-iOS targets. ndarray = "0.17" # Opus encoder. tsclientlib already pulls this; we depend explicitly so @@ -50,12 +50,6 @@ coreaudio-rs = "0.14" # on the main queue to avoid the VPIO RPC timeout on iOS simulator. dispatch2 = "0.3" -[target.'cfg(target_os = "ios")'.dependencies] -# ONNX Runtime Rust binding for Silero VAD v6 (P1 VAD_002). The official -# iOS CocoaPod ships ONNX Runtime as a static framework, so iOS links it -# into chanora_bridge at build time instead of loading a dylib at runtime. -ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray"] } - [target.'cfg(not(target_os = "ios"))'.dependencies] ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "api-24"] } diff --git a/crates/chanora_audio/src/audio_processing.rs b/crates/chanora_audio/src/audio_processing.rs index f7ec530..a40cfca 100644 --- a/crates/chanora_audio/src/audio_processing.rs +++ b/crates/chanora_audio/src/audio_processing.rs @@ -100,6 +100,16 @@ pub enum VadBackend { Disabled, } +#[cfg(target_os = "ios")] +fn default_vad_backend() -> VadBackend { + VadBackend::WebrtcVad +} + +#[cfg(not(target_os = "ios"))] +fn default_vad_backend() -> VadBackend { + VadBackend::SileroOnnx +} + impl VadBackend { /// Stable bridge/debug string. pub fn as_str(self) -> &'static str { @@ -164,7 +174,7 @@ impl Default for AudioProcessingConfig { route: AudioRoute::Speaker, ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing, processing_backend: AudioBackend::PlatformVoiceProcessing, - vad_backend: VadBackend::SileroOnnx, + vad_backend: default_vad_backend(), aec: EffectOwner::Platform, // iOS VPIO owns NS/AGC on the default shipping path. Software // effects are opt-in through the experimental raw route only. @@ -242,6 +252,9 @@ mod tests { assert_eq!(config.aec, EffectOwner::Platform); assert_eq!(config.ns, EffectOwner::Platform); assert_eq!(config.agc, EffectOwner::Platform); + #[cfg(target_os = "ios")] + assert_eq!(config.vad_backend, VadBackend::WebrtcVad); + #[cfg(not(target_os = "ios"))] assert_eq!(config.vad_backend, VadBackend::SileroOnnx); } diff --git a/crates/chanora_audio/src/ios_raw_unit.rs b/crates/chanora_audio/src/ios_raw_unit.rs index 46275e3..2f899d8 100644 --- a/crates/chanora_audio/src/ios_raw_unit.rs +++ b/crates/chanora_audio/src/ios_raw_unit.rs @@ -116,9 +116,7 @@ mod inner { mic_gain: f32, voice_activity_selector: Option>, vad_detector: crate::vad::WebRtcFallbackVad, - silero_vad_worker: Option, current_vad_backend: crate::VadBackend, - silero_model_epoch: u64, capture_frame_seq: u64, vad_state: crate::voice_activity::VoiceActivityStateMachine, /// Processing config — retained for route-change reloads. @@ -154,9 +152,7 @@ mod inner { mic_gain: params.mic_gain, voice_activity_selector: params.voice_activity_selector.clone(), vad_detector: crate::vad::WebRtcFallbackVad::default(), - silero_vad_worker: None, current_vad_backend: crate::VadBackend::WebrtcVad, - silero_model_epoch: crate::vad::silero_model_epoch(), capture_frame_seq: 0, vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), audio_processing_config: params.audio_processing_config.clone(), @@ -276,7 +272,6 @@ mod inner { .map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity) .unwrap_or(false); if !voice_activity_mode { - self.silero_vad_worker = None; self.current_vad_backend = crate::VadBackend::Disabled; self.fallback_warned_backend = None; self.audio_processing_stats.set_vad_fallback_active(false); @@ -298,37 +293,20 @@ mod inner { ); } - // Switch VAD backend only while VoiceActivity mode is active. - let silero_epoch = crate::vad::silero_model_epoch(); - let silero_changed = voice_activity_mode - && vad_backend == crate::VadBackend::SileroOnnx - && silero_epoch != self.silero_model_epoch; - if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_changed) { + if voice_activity_mode && vad_backend != self.current_vad_backend { self.current_vad_backend = vad_backend; - self.silero_model_epoch = silero_epoch; self.fallback_warned_backend = None; - match vad_backend { - crate::VadBackend::SileroOnnx => { - let path = crate::vad::silero_model_bundle_path(); - self.silero_vad_worker = - crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path); - if self.silero_vad_worker.is_none() { - tracing::warn!( - target: "chanora_audio", - "Silero VAD model not found at {path}; falling back to WebRTC VAD" - ); - } - } - _ => { - self.silero_vad_worker = None; - } + if vad_backend == crate::VadBackend::SileroOnnx { + self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); + self.audio_processing_stats.set_vad_fallback_active(true); + } else { + self.audio_processing_stats.set_vad_fallback_active(false); } self.vad_state.reset(); } let (vad_probability, active) = if voice_activity_mode { self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1); - let capture_seq = self.capture_frame_seq; let mut used_fallback_vad = false; let vad = if vad_backend == crate::VadBackend::Disabled { crate::vad::VadOutput { @@ -336,35 +314,9 @@ mod inner { speech: true, } } else if vad_backend == crate::VadBackend::SileroOnnx { - if let Some(worker) = self.silero_vad_worker.as_ref() { - let enqueued = worker.try_send(capture_seq, &frame); - if !worker.is_stale(capture_seq) { - let p = worker.latest_probability(); - crate::vad::VadOutput { - probability: p, - speech: p >= 0.5, - } - } else if enqueued { - crate::vad::VadOutput { - probability: 0.0, - speech: false, - } - } else { - used_fallback_vad = true; - self.mark_vad_fallback_active(vad_backend); - crate::vad::VoiceActivityDetector::process_10ms( - &mut self.vad_detector, - &frame, - ) - } - } else { - used_fallback_vad = true; - self.mark_vad_fallback_active(vad_backend); - crate::vad::VoiceActivityDetector::process_10ms( - &mut self.vad_detector, - &frame, - ) - } + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) } else { crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) }; diff --git a/crates/chanora_audio/src/ios_voice_unit.rs b/crates/chanora_audio/src/ios_voice_unit.rs index b36769b..48a8df1 100644 --- a/crates/chanora_audio/src/ios_voice_unit.rs +++ b/crates/chanora_audio/src/ios_voice_unit.rs @@ -48,20 +48,15 @@ //! //! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the //! AudioUnit internally holds the C `AudioUnit` opaque pointer and -//! the wrapper's destructor calls `AudioComponentInstanceDispose`, -//! which (per Apple's threading rules) must be called from the -//! thread that owns the unit. We open the unit on the same thread -//! that calls `Self::start` (the tokio worker that runs -//! `chanora_core::ChanoraSession::start_audio`, the same pattern -//! cpal + SDL use) and never move it. The outer `AudioEngine` -//! already carries an `unsafe impl Send` to satisfy the same -//! constraint for cpal's `!Send` Stream type; that impl covers -//! VPIO too. -//! -//! Dropping `IosVoiceUnit` calls `audio_unit.stop()` via the -//! wrapper's `Drop`, which detaches the render + input callbacks -//! and stops the unit. The AudioHandler + CaptureState `Arc`s the -//! callbacks held are then released. +//! the wrapper's destructor calls `AudioComponentInstanceDispose`. +//! `IosVoiceUnit` stores the unit as `Option` so initial +//! setup and lifecycle operations can dispatch CoreAudio +//! initialize/start/stop calls to `DispatchQueue::main()` while the +//! wrapper keeps ownership and preserves the render/input callback +//! state in the surrounding `Arc`s. `restart`, `pause`, and +//! `resume` may temporarily move the unit through that helper, and +//! `Drop` stops it if still present before those callback `Arc`s are +//! released. //! //! ## What this file does NOT do //! @@ -143,13 +138,8 @@ struct IosCaptureState { mic_gain: f32, voice_activity_selector: Option>, vad_detector: crate::vad::WebRtcFallbackVad, - /// Background Silero worker — enqueues frames off the realtime - /// callback and publishes the latest probability atomically. - silero_vad_worker: Option, /// Last VAD backend we configured — used to detect backend changes. current_vad_backend: crate::VadBackend, - /// Last observed configured Silero model epoch. - silero_model_epoch: u64, fallback_warned_backend: Option, vad_state: crate::voice_activity::VoiceActivityStateMachine, audio_processing_config: Arc>, @@ -187,9 +177,7 @@ impl IosCaptureState { mic_gain: params.mic_gain, voice_activity_selector: params.voice_activity_selector.clone(), vad_detector: crate::vad::WebRtcFallbackVad::default(), - silero_vad_worker: None, current_vad_backend: crate::VadBackend::WebrtcVad, - silero_model_epoch: crate::vad::silero_model_epoch(), fallback_warned_backend: None, vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), audio_processing_config: params.audio_processing_config.clone(), @@ -349,7 +337,7 @@ impl IosCaptureState { false, false, true, - crate::VadBackend::SileroOnnx, + crate::VadBackend::WebrtcVad, crate::voice_activity::VAD_HANGOVER_MS, false, crate::AudioRoute::Unknown, @@ -362,18 +350,12 @@ impl IosCaptureState { .map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity) .unwrap_or(false); if !voice_activity_mode { - self.silero_vad_worker = None; self.current_vad_backend = crate::VadBackend::Disabled; self.fallback_warned_backend = None; self.audio_processing_stats.set_vad_fallback_active(false); } // Switch VAD backend only while VoiceActivity mode is active. - let silero_model_epoch = crate::vad::silero_model_epoch(); - let silero_model_changed = voice_activity_mode - && vad_backend == crate::VadBackend::SileroOnnx - && silero_model_epoch != self.silero_model_epoch; - if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() { if debug_wav_dump_enabled { if recorder_guard.is_none() { @@ -387,35 +369,14 @@ impl IosCaptureState { } } - if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_model_changed) - { + if voice_activity_mode && vad_backend != self.current_vad_backend { self.current_vad_backend = vad_backend; self.fallback_warned_backend = None; - self.silero_model_epoch = silero_model_epoch; - match vad_backend { - crate::VadBackend::SileroOnnx => { - // Attempt to load Silero model from the well-known - // bundle path. The actual inference runs on a - // background worker; the callback only enqueues - // 10 ms frames and falls back to WebRTC if the - // worker is missing or stale. - let model_path = crate::vad::silero_model_bundle_path(); - self.silero_vad_worker = - crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path); - if self.silero_vad_worker.is_none() { - warn!( - target: "chanora_audio", - "Silero VAD model not found at {model_path}; falling back to WebRTC VAD" - ); - self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); - } - self.audio_processing_stats - .set_vad_fallback_active(self.silero_vad_worker.is_none()); - } - _ => { - self.silero_vad_worker = None; - self.audio_processing_stats.set_vad_fallback_active(false); - } + if vad_backend == crate::VadBackend::SileroOnnx { + self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); + self.audio_processing_stats.set_vad_fallback_active(true); + } else { + self.audio_processing_stats.set_vad_fallback_active(false); } // Reset VAD state machine timers on backend switch. self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new( @@ -457,7 +418,6 @@ impl IosCaptureState { // VAD: only evaluate while VoiceActivity mode is active. let (vad_probability, gate_open) = if voice_activity_mode { self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1); - let capture_seq = self.capture_frame_seq; let mut used_fallback_vad = false; let vad = if vad_backend == crate::VadBackend::Disabled { crate::vad::VadOutput { @@ -465,32 +425,9 @@ impl IosCaptureState { speech: true, } } else if vad_backend == crate::VadBackend::SileroOnnx { - if let Some(worker) = self.silero_vad_worker.as_ref() { - let enqueued = worker.try_send(capture_seq, &frame); - if !worker.is_stale(capture_seq) { - let probability = worker.latest_probability(); - crate::vad::VadOutput { - probability, - speech: probability >= 0.5, - } - } else if enqueued { - crate::vad::VadOutput { - probability: 0.0, - speech: false, - } - } else { - used_fallback_vad = true; - self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); - crate::vad::VoiceActivityDetector::process_10ms( - &mut self.vad_detector, - &frame, - ) - } - } else { - used_fallback_vad = true; - self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); - crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) - } + used_fallback_vad = true; + self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); + crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) } else { crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) }; @@ -578,10 +515,39 @@ pub struct IosVoiceUnit { // Drop = stop the audio unit (severs render callback). The // wrapper's own Drop calls AudioComponentInstanceDispose // after stop returns. - unit: AudioUnit, + unit: Option, } impl IosVoiceUnit { + #[cfg(target_os = "ios")] + fn exec_main_queue_lifecycle( + &mut self, + op: impl FnOnce(&mut AudioUnit) -> Result<(), String> + Send + 'static, + ) -> Result<(), AudioError> { + let unit = self + .unit + .take() + .ok_or_else(|| AudioError::Backend("vpio lifecycle: audio unit missing".to_string()))?; + + let (tx, rx) = std::sync::mpsc::sync_channel::>(1); + let unit_arc = Arc::new(Mutex::new(Some(unit))); + let unit_arc2 = Arc::clone(&unit_arc); + + dispatch2::DispatchQueue::main().exec_async(move || { + let mut guard = unit_arc2.lock().unwrap(); + let unit = guard.as_mut().unwrap(); + let _ = tx.send(op(unit)); + }); + + let result = match rx.recv() { + Ok(result) => result, + Err(_) => Err("vpio lifecycle: main thread channel closed unexpectedly".to_string()), + }; + + self.unit = unit_arc.lock().unwrap().take(); + result.map_err(AudioError::Backend) + } + /// Open a VoiceProcessingIO AudioUnit, pin its stream format /// to 48 kHz Int16 mono on both buses, install render + input /// callbacks, and start it. The unit begins pumping audio @@ -994,7 +960,7 @@ impl IosVoiceUnit { ), } - Ok(Self { unit }) + Ok(Self { unit: Some(unit) }) } /// Restart the audio unit after route change handling. @@ -1003,23 +969,18 @@ impl IosVoiceUnit { /// VoiceProcessingIO unit through an uninitialize/reinitialize /// cycle, then start again. /// - /// Called from the Flutter method channel handler which runs on - /// the main isolate — that runs on the main thread — so the - /// CoreAudio RPC is already on the correct thread here. #[cfg(target_os = "ios")] pub fn restart(&mut self) -> Result<(), AudioError> { - self.unit - .stop() - .map_err(|e| AudioError::Backend(format!("vpio restart stop: {e}")))?; - self.unit - .uninitialize() - .map_err(|e| AudioError::Backend(format!("vpio restart uninit: {e}")))?; - self.unit - .initialize() - .map_err(|e| AudioError::Backend(format!("vpio restart init: {e}")))?; - self.unit - .start() - .map_err(|e| AudioError::Backend(format!("vpio restart start: {e}")))?; + self.exec_main_queue_lifecycle(|unit| { + unit.stop().map_err(|e| format!("vpio restart stop: {e}"))?; + unit.uninitialize() + .map_err(|e| format!("vpio restart uninit: {e}"))?; + unit.initialize() + .map_err(|e| format!("vpio restart init: {e}"))?; + unit.start() + .map_err(|e| format!("vpio restart start: {e}"))?; + Ok(()) + })?; info!(target: "chanora_audio", "ios VPIO audio unit restarted"); Ok(()) } @@ -1027,17 +988,17 @@ impl IosVoiceUnit { /// Pause the audio unit during an interruption. #[cfg(target_os = "ios")] pub fn pause(&mut self) -> Result<(), AudioError> { - self.unit - .stop() - .map_err(|e| AudioError::Backend(format!("vpio pause stop: {e}"))) + self.exec_main_queue_lifecycle(|unit| { + unit.stop().map_err(|e| format!("vpio pause stop: {e}")) + }) } /// Resume the audio unit after an interruption. #[cfg(target_os = "ios")] pub fn resume(&mut self) -> Result<(), AudioError> { - self.unit - .start() - .map_err(|e| AudioError::Backend(format!("vpio resume start: {e}"))) + self.exec_main_queue_lifecycle(|unit| { + unit.start().map_err(|e| format!("vpio resume start: {e}")) + }) } } @@ -1046,10 +1007,12 @@ impl Drop for IosVoiceUnit { // Stop the audio unit so the render callback no longer // fires. The coreaudio-rs wrapper's own Drop calls // AudioComponentInstanceDispose afterwards. - if let Err(e) = self.unit.stop() { - warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed"); - } else { - info!(target: "chanora_audio", "ios audio unit stopped"); + if let Some(unit) = self.unit.as_mut() { + if let Err(e) = unit.stop() { + warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed"); + } else { + info!(target: "chanora_audio", "ios audio unit stopped"); + } } } } diff --git a/crates/chanora_audio/src/route_policy.rs b/crates/chanora_audio/src/route_policy.rs index d83ea14..1f7148d 100644 --- a/crates/chanora_audio/src/route_policy.rs +++ b/crates/chanora_audio/src/route_policy.rs @@ -30,7 +30,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig { route, ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing, processing_backend: AudioBackend::PlatformVoiceProcessing, - vad_backend: VadBackend::SileroOnnx, + vad_backend: VadBackend::WebrtcVad, aec: EffectOwner::Platform, // VPIO owns NS and AGC on the shipping default path (IOSP_002/003). ns: EffectOwner::Platform, @@ -43,7 +43,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig { route, ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing, processing_backend: AudioBackend::Noop, - vad_backend: VadBackend::SileroOnnx, + vad_backend: VadBackend::WebrtcVad, // No AEC needed for wired headset (no acoustic echo path). aec: EffectOwner::Off, // Conservative NS/AGC: optional, not forced. @@ -57,7 +57,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig { route, ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing, processing_backend: AudioBackend::PlatformVoiceProcessing, - vad_backend: VadBackend::SileroOnnx, + vad_backend: VadBackend::WebrtcVad, // BT HFP manages its own AEC in the headset firmware. aec: EffectOwner::Off, ns: EffectOwner::Conservative, @@ -87,7 +87,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig { route, ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing, processing_backend: AudioBackend::Noop, - vad_backend: VadBackend::SileroOnnx, + vad_backend: VadBackend::WebrtcVad, // Safe fallback: AEC off until route is classified. aec: EffectOwner::Off, ns: EffectOwner::Off, @@ -146,6 +146,7 @@ mod tests { cfg.ios_mode, IosVoiceProcessingMode::PlatformVoiceProcessing ); + assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad); } #[test] @@ -156,6 +157,7 @@ mod tests { AudioBackend::PlatformVoiceProcessing ); assert_eq!(cfg.aec, EffectOwner::Platform); + assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad); } #[test] @@ -163,12 +165,14 @@ mod tests { let cfg = ios_route_policy(AudioRoute::WiredHeadset); assert_eq!(cfg.aec, EffectOwner::Off); assert_eq!(cfg.processing_backend, AudioBackend::Noop); + assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad); } #[test] fn bluetooth_hfp_disables_app_aec() { let cfg = ios_route_policy(AudioRoute::BluetoothHfp); assert_eq!(cfg.aec, EffectOwner::Off); + assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad); } #[test] @@ -183,6 +187,7 @@ mod tests { fn unknown_route_safe_fallback_no_aec() { let cfg = ios_route_policy(AudioRoute::Unknown); assert_eq!(cfg.aec, EffectOwner::Off); + assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad); } #[test] diff --git a/crates/chanora_audio/src/vad/mod.rs b/crates/chanora_audio/src/vad/mod.rs index db878d9..74102bb 100644 --- a/crates/chanora_audio/src/vad/mod.rs +++ b/crates/chanora_audio/src/vad/mod.rs @@ -1,11 +1,12 @@ //! Voice activity detection backends and helpers. //! //! iOS capture feeds VoiceProcessingIO-processed microphone frames into -//! this module. The production path prefers a model-backed detector when -//! available, and otherwise uses the realtime-safe fallback below so +//! this module and uses the realtime-safe WebRTC fallback. Other +//! platforms may use a model-backed detector when available so //! VoiceActivity mode never collapses back to Continuous transmit. pub mod resampler; +#[cfg(not(target_os = "ios"))] pub mod silero_onnx; use std::sync::atomic::{AtomicU64, Ordering}; @@ -15,6 +16,7 @@ use crate::frame::{f32_to_i16, i16_to_f32}; use crate::AudioError; use resampler::{Downsampler48to16, INPUT_FRAME_10MS}; +#[cfg(not(target_os = "ios"))] pub use silero_onnx::SileroOnnxVad; /// Voice activity detector output for one 10 ms frame. @@ -111,11 +113,11 @@ fn silero_model_path_override() -> &'static RwLock> { SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None)) } -/// Configure the preferred Silero ONNX model path. +/// Configure the preferred Silero ONNX model path on supported platforms. /// /// The path is validated eagerly. A successful call increments the -/// model epoch so running audio backends can reload the model without -/// an app restart. +/// model epoch so running non-iOS audio backends can reload the model +/// without an app restart. pub fn set_silero_model_path(path: &str) -> Result<(), AudioError> { let path = path.trim(); if path.is_empty() { @@ -141,13 +143,13 @@ pub fn silero_model_epoch() -> u64 { SILERO_MODEL_EPOCH.load(Ordering::Relaxed) } -/// Return the expected path of the Silero VAD v6 ONNX model. +/// Return the expected path of the Silero VAD v6 ONNX model on +/// supported platforms. /// The model is shipped as a Flutter asset and copied to the app's /// data directory by the Dart-side asset loader. /// -/// On iOS/Android the model lives in the app's Documents/files -/// directory. On desktop, the caller should set the path explicitly -/// via `set_silero_model_path`. +/// Android and macOS may use app data/Documents locations. Desktop +/// callers can set the path explicitly via `set_silero_model_path`. pub fn silero_model_bundle_path() -> String { if let Ok(guard) = silero_model_path_override().read() { if let Some(path) = guard.as_ref() { @@ -155,8 +157,9 @@ pub fn silero_model_bundle_path() -> String { } } - // iOS: Documents directory (written by Flutter asset loader). - // macOS: same Documents pattern. + // macOS: Documents directory (written by Flutter asset loader). + // iOS keeps this fallback only for API compatibility; the ONNX + // detector is not compiled into iOS builds. #[cfg(any(target_os = "ios", target_os = "macos"))] { if let Ok(home) = std::env::var("HOME") { From e8e9fa8ccf90532a0efa1d9be6d830e41a767839 Mon Sep 17 00:00:00 2001 From: Edison Jwa Date: Sun, 31 May 2026 22:30:46 +0900 Subject: [PATCH 4/5] build(ios): remove onnxruntime pod wiring --- apps/chanora_flutter/ios/Podfile | 1 - apps/chanora_flutter/ios/Podfile.lock | 11 ++-------- .../ios/chanora_bridge.podspec | 20 ------------------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/apps/chanora_flutter/ios/Podfile b/apps/chanora_flutter/ios/Podfile index ef65559..2085ef7 100644 --- a/apps/chanora_flutter/ios/Podfile +++ b/apps/chanora_flutter/ios/Podfile @@ -39,7 +39,6 @@ target 'Runner' do # flutter_rust_bridge can dlopen() it at runtime via FRB's # default `chanora_bridge.framework/chanora_bridge` lookup path. pod 'chanora_bridge', :path => '.' - pod 'onnxruntime-c', '1.22.0' flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) target 'RunnerTests' do diff --git a/apps/chanora_flutter/ios/Podfile.lock b/apps/chanora_flutter/ios/Podfile.lock index d251b11..982fd00 100644 --- a/apps/chanora_flutter/ios/Podfile.lock +++ b/apps/chanora_flutter/ios/Podfile.lock @@ -9,7 +9,6 @@ PODS: - Flutter - haptic_kit (1.0.0): - Flutter - - onnxruntime-c (1.22.0) - package_info_plus (0.4.5): - Flutter - share_plus (0.0.1): @@ -27,16 +26,11 @@ DEPENDENCIES: - Flutter (from `Flutter`) - flutter_foreground_task (from `.symlinks/plugins/flutter_foreground_task/ios`) - haptic_kit (from `.symlinks/plugins/haptic_kit/ios`) - - onnxruntime-c (= 1.22.0) - package_info_plus (from `.symlinks/plugins/package_info_plus/ios`) - share_plus (from `.symlinks/plugins/share_plus/ios`) - shared_preferences_foundation (from `.symlinks/plugins/shared_preferences_foundation/darwin`) - url_launcher_ios (from `.symlinks/plugins/url_launcher_ios/ios`) -SPEC REPOS: - trunk: - - onnxruntime-c - EXTERNAL SOURCES: audio_session: :path: ".symlinks/plugins/audio_session/ios" @@ -61,17 +55,16 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: audio_session: 9bb7f6c970f21241b19f5a3658097ae459681ba0 - chanora_bridge: 0289413733edf8b7c937c50c3c3424b3319b94b5 + chanora_bridge: e1c7a6f9135400efec036d9df706b2b4b29b2cd5 connectivity_plus: cb623214f4e1f6ef8fe7403d580fdad517d2f7dd Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 flutter_foreground_task: a159d2c2173b33699ddb3e6c2a067045d7cebb89 haptic_kit: b22c4fbb2aa7b0d66f2891f81a9e950ad2de5758 - onnxruntime-c: 7f778680e96145956c0a31945f260321eed2611a package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499 share_plus: 50da8cb520a8f0f65671c6c6a99b3617ed10a58a shared_preferences_foundation: 7036424c3d8ec98dfe75ff1667cb0cd531ec82bb url_launcher_ios: 7a95fa5b60cc718a708b8f2966718e93db0cef1b -PODFILE CHECKSUM: a3abe93db2fc91b90387b399576e9c42a54226e0 +PODFILE CHECKSUM: 50c5b575d74b9daff4bf33213abfd2c15d7f2e41 COCOAPODS: 1.16.2 diff --git a/apps/chanora_flutter/ios/chanora_bridge.podspec b/apps/chanora_flutter/ios/chanora_bridge.podspec index b7ae0ea..d204783 100644 --- a/apps/chanora_flutter/ios/chanora_bridge.podspec +++ b/apps/chanora_flutter/ios/chanora_bridge.podspec @@ -85,13 +85,6 @@ Pod::Spec.new do |s| } CARGO_BIN="$(find_cargo)" RUSTC_BIN="$(find_rustc)" - ORT_FRAMEWORK="$REPO_ROOT/apps/chanora_flutter/ios/Pods/onnxruntime-c/onnxruntime.xcframework/ios-arm64/onnxruntime.framework" - ORT_LINK_DIR="$REPO_ROOT/target/onnxruntime-ios-device" - if [ -f "$ORT_FRAMEWORK/onnxruntime" ]; then - mkdir -p "$ORT_LINK_DIR" - lipo "$ORT_FRAMEWORK/onnxruntime" -thin arm64 -output "$ORT_LINK_DIR/libonnxruntime.a" - fi - echo "[chanora_bridge.podspec] cargo build aarch64-apple-ios" cd "$REPO_ROOT" HOME="$USER_HOME" \\ @@ -99,7 +92,6 @@ Pod::Spec.new do |s| RUSTUP_HOME="$USER_HOME/.rustup" \\ RUSTUP_TOOLCHAIN="stable-aarch64-apple-darwin" \\ RUSTC="$RUSTC_BIN" \\ - ORT_LIB_LOCATION="$ORT_LINK_DIR" \\ IPHONEOS_DEPLOYMENT_TARGET=15.1 \\ CMAKE_POLICY_VERSION_MINIMUM=3.5 \\ CMAKE_OSX_DEPLOYMENT_TARGET=15.1 \\ @@ -196,22 +188,11 @@ PLIST if [ "${PLATFORM_NAME:-iphoneos}" = "iphonesimulator" ]; then RUST_TARGET="aarch64-apple-ios-sim" SUPPORTED_PLATFORM="iPhoneSimulator" - ORT_SLICE="ios-arm64_x86_64-simulator" else RUST_TARGET="aarch64-apple-ios" SUPPORTED_PLATFORM="iPhoneOS" - ORT_SLICE="ios-arm64" fi BRIDGE="$REPO_ROOT/target/$RUST_TARGET/release/libchanora_bridge.dylib" - ORT_FRAMEWORK="$REPO_ROOT/apps/chanora_flutter/ios/Pods/onnxruntime-c/onnxruntime.xcframework/$ORT_SLICE/onnxruntime.framework" - ORT_LINK_DIR="$REPO_ROOT/target/onnxruntime-$RUST_TARGET" - if [ ! -f "$ORT_FRAMEWORK/onnxruntime" ]; then - echo "ERROR: ONNX Runtime framework not found at $ORT_FRAMEWORK" >&2 - exit 1 - fi - mkdir -p "$ORT_LINK_DIR" - lipo "$ORT_FRAMEWORK/onnxruntime" -thin arm64 -output "$ORT_LINK_DIR/libonnxruntime.a" - echo "[chanora_bridge script_phase] cargo build $RUST_TARGET" cd "$REPO_ROOT" HOME="$USER_HOME" \\ @@ -219,7 +200,6 @@ PLIST RUSTUP_HOME="$USER_HOME/.rustup" \\ RUSTUP_TOOLCHAIN="stable-aarch64-apple-darwin" \\ RUSTC="$RUSTC_BIN" \\ - ORT_LIB_LOCATION="$ORT_LINK_DIR" \\ IPHONEOS_DEPLOYMENT_TARGET=15.1 \\ CMAKE_POLICY_VERSION_MINIMUM=3.5 \\ CMAKE_OSX_DEPLOYMENT_TARGET=15.1 \\ From ecb9ae96369b224a6bac1ae441f764f92ccb6866 Mon Sep 17 00:00:00 2001 From: Edison Jwa Date: Sun, 31 May 2026 22:30:56 +0900 Subject: [PATCH 5/5] fix(audio): restart iOS voice unit in place --- crates/chanora_audio/src/engine.rs | 34 ++++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/crates/chanora_audio/src/engine.rs b/crates/chanora_audio/src/engine.rs index 376ee23..4da1443 100644 --- a/crates/chanora_audio/src/engine.rs +++ b/crates/chanora_audio/src/engine.rs @@ -395,6 +395,22 @@ enum IosVoiceBackend { #[cfg(any(target_os = "ios", target_os = "macos"))] impl IosVoiceBackend { + fn restart(&mut self) -> Result<(), AudioError> { + #[cfg(target_os = "ios")] + { + match self { + Self::Vpio(unit) => unit.restart(), + Self::Raw(unit) => unit.restart(), + } + } + #[cfg(target_os = "macos")] + { + match self { + Self::Vpio(_unit) => Ok(()), + } + } + } + fn pause(&mut self) -> Result<(), AudioError> { #[cfg(target_os = "ios")] { @@ -1404,21 +1420,11 @@ impl AudioEngine { pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> { #[cfg(any(target_os = "ios", target_os = "macos"))] { - let backend = open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams { - handler: self.audio_handler.clone(), - output_gain: self.output_gain.clone(), - output_muted: self.output_muted.clone(), - voice_out_tx: self.voice_out_tx.clone(), - transmit_active: self.transmit_gate.flag_arc(), - frames_sent: self.frames_sent.clone(), - mic_gain: self.mic_gain, - voice_activity_selector: self.voice_activity_selector.clone(), - audio_processing_config: self.audio_processing_config.clone(), - audio_processing_stats: self.audio_processing_stats.clone(), - })?; let mut guard = self._ios_voice_backend.lock().unwrap(); - *guard = Some(backend); - Ok(()) + let backend = guard + .as_mut() + .ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?; + backend.restart() } #[cfg(not(any(target_os = "ios", target_os = "macos")))] {