edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #10 from EdisonJwa/fix/ios-audio-lifecycle-lag

Browse Source 
fix(ios): stabilize audio lifecycle startup
This commit is contained in:
Edison Jwa
2026-05-31 23:43:32 +09:00
committed by GitHub
parent 7fb89c3dc7 ecb9ae9636
commit 25e6a16eb1
13 changed files with 237 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/chanora_flutter/ios/Podfile
-1
View File
@@ -39,7 +39,6 @@ target 'Runner' do
# flutter_rust_bridge can dlopen() it at runtime via FRB's
# default `chanora_bridge.framework/chanora_bridge` lookup path.
pod 'chanora_bridge', :path => '.'
pod 'onnxruntime-c', '1.22.0'
flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__))
target 'RunnerTests' do
apps/chanora_flutter/ios/Podfile.lock
+2 -9
View File
@@ -9,7 +9,6 @@ PODS:
- Flutter
- haptic_kit (1.0.0):
- Flutter
- onnxruntime-c (1.22.0)
- package_info_plus (0.4.5):
- Flutter
- share_plus (0.0.1):
@@ -27,16 +26,11 @@ DEPENDENCIES:
- Flutter (from `Flutter`)
- flutter_foreground_task (from `.symlinks/plugins/flutter_foreground_task/ios`)
- haptic_kit (from `.symlinks/plugins/haptic_kit/ios`)
- onnxruntime-c (= 1.22.0)
- package_info_plus (from `.symlinks/plugins/package_info_plus/ios`)
- share_plus (from `.symlinks/plugins/share_plus/ios`)
- shared_preferences_foundation (from `.symlinks/plugins/shared_preferences_foundation/darwin`)
- url_launcher_ios (from `.symlinks/plugins/url_launcher_ios/ios`)
SPEC REPOS:
trunk:
- onnxruntime-c
EXTERNAL SOURCES:
audio_session:
:path: ".symlinks/plugins/audio_session/ios"
@@ -61,17 +55,16 @@ EXTERNAL SOURCES:
SPEC CHECKSUMS:
audio_session: 9bb7f6c970f21241b19f5a3658097ae459681ba0
chanora_bridge: 0289413733edf8b7c937c50c3c3424b3319b94b5
chanora_bridge: e1c7a6f9135400efec036d9df706b2b4b29b2cd5
connectivity_plus: cb623214f4e1f6ef8fe7403d580fdad517d2f7dd
Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467
flutter_foreground_task: a159d2c2173b33699ddb3e6c2a067045d7cebb89
haptic_kit: b22c4fbb2aa7b0d66f2891f81a9e950ad2de5758
onnxruntime-c: 7f778680e96145956c0a31945f260321eed2611a
package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499
share_plus: 50da8cb520a8f0f65671c6c6a99b3617ed10a58a
shared_preferences_foundation: 7036424c3d8ec98dfe75ff1667cb0cd531ec82bb
url_launcher_ios: 7a95fa5b60cc718a708b8f2966718e93db0cef1b
PODFILE CHECKSUM: a3abe93db2fc91b90387b399576e9c42a54226e0
PODFILE CHECKSUM: 50c5b575d74b9daff4bf33213abfd2c15d7f2e41
COCOAPODS: 1.16.2
apps/chanora_flutter/ios/chanora_bridge.podspec
-20
View File
@@ -85,13 +85,6 @@ Pod::Spec.new do |s|
}
CARGO_BIN="$(find_cargo)"
RUSTC_BIN="$(find_rustc)"
ORT_FRAMEWORK="$REPO_ROOT/apps/chanora_flutter/ios/Pods/onnxruntime-c/onnxruntime.xcframework/ios-arm64/onnxruntime.framework"
ORT_LINK_DIR="$REPO_ROOT/target/onnxruntime-ios-device"
if [ -f "$ORT_FRAMEWORK/onnxruntime" ]; then
mkdir -p "$ORT_LINK_DIR"
lipo "$ORT_FRAMEWORK/onnxruntime" -thin arm64 -output "$ORT_LINK_DIR/libonnxruntime.a"
fi
echo "[chanora_bridge.podspec] cargo build aarch64-apple-ios"
cd "$REPO_ROOT"
HOME="$USER_HOME" \\
@@ -99,7 +92,6 @@ Pod::Spec.new do |s|
RUSTUP_HOME="$USER_HOME/.rustup" \\
RUSTUP_TOOLCHAIN="stable-aarch64-apple-darwin" \\
RUSTC="$RUSTC_BIN" \\
ORT_LIB_LOCATION="$ORT_LINK_DIR" \\
IPHONEOS_DEPLOYMENT_TARGET=15.1 \\
CMAKE_POLICY_VERSION_MINIMUM=3.5 \\
CMAKE_OSX_DEPLOYMENT_TARGET=15.1 \\
@@ -196,22 +188,11 @@ PLIST
if [ "${PLATFORM_NAME:-iphoneos}" = "iphonesimulator" ]; then
RUST_TARGET="aarch64-apple-ios-sim"
SUPPORTED_PLATFORM="iPhoneSimulator"
ORT_SLICE="ios-arm64_x86_64-simulator"
else
RUST_TARGET="aarch64-apple-ios"
SUPPORTED_PLATFORM="iPhoneOS"
ORT_SLICE="ios-arm64"
fi
BRIDGE="$REPO_ROOT/target/$RUST_TARGET/release/libchanora_bridge.dylib"
ORT_FRAMEWORK="$REPO_ROOT/apps/chanora_flutter/ios/Pods/onnxruntime-c/onnxruntime.xcframework/$ORT_SLICE/onnxruntime.framework"
ORT_LINK_DIR="$REPO_ROOT/target/onnxruntime-$RUST_TARGET"
if [ ! -f "$ORT_FRAMEWORK/onnxruntime" ]; then
echo "ERROR: ONNX Runtime framework not found at $ORT_FRAMEWORK" >&2
exit 1
fi
mkdir -p "$ORT_LINK_DIR"
lipo "$ORT_FRAMEWORK/onnxruntime" -thin arm64 -output "$ORT_LINK_DIR/libonnxruntime.a"
echo "[chanora_bridge script_phase] cargo build $RUST_TARGET"
cd "$REPO_ROOT"
HOME="$USER_HOME" \\
@@ -219,7 +200,6 @@ PLIST
RUSTUP_HOME="$USER_HOME/.rustup" \\
RUSTUP_TOOLCHAIN="stable-aarch64-apple-darwin" \\
RUSTC="$RUSTC_BIN" \\
ORT_LIB_LOCATION="$ORT_LINK_DIR" \\
IPHONEOS_DEPLOYMENT_TARGET=15.1 \\
CMAKE_POLICY_VERSION_MINIMUM=3.5 \\
CMAKE_OSX_DEPLOYMENT_TARGET=15.1 \\
core/chanora_core/src/lib.rs
+21 -15
View File
@@ -891,15 +891,18 @@ impl ChanoraSession {
if let Some(projection) = projection {
self.emit_voice_state(projection).await;
}
if let Err(audio_err) = self.ensure_audio_running().await {
warn!(
target: "chanora_core",
error = %audio_err,
"auto-join default channel: server placed us in a channel but audio engine \
failed to start; continuing with no-audio in-channel state"
);
let _ = self.events_tx.send(SessionEvent::AudioStopped);
}
let session = self.clone();
tokio::spawn(async move {
if let Err(audio_err) = session.ensure_audio_running().await {
warn!(
target: "chanora_core",
error = %audio_err,
"auto-join default channel: server placed us in a channel but audio engine \
failed to start; continuing with no-audio in-channel state"
);
let _ = session.events_tx.send(SessionEvent::AudioStopped);
}
});
}
Ok(snap)
@@ -1388,11 +1391,12 @@ impl ChanoraSession {
Some(audio.audio_processing_stats())
}
/// Configure the preferred Silero ONNX VAD model path.
/// Configure the preferred Silero ONNX VAD model path on platforms
/// that ship the ONNX detector.
///
/// This does not require an active connection. Running iOS audio
/// backends observe the model-path epoch and reload on the next
/// capture frame when Silero is selected.
/// This does not require an active connection. Running non-iOS
/// audio backends can observe the model-path epoch and reload on
/// the next capture frame when Silero is selected.
pub async fn set_vad_model_path(&self, path: String) -> Result<(), CoreError> {
chanora_audio::vad::set_silero_model_path(&path)?;
Ok(())
@@ -1404,8 +1408,10 @@ impl ChanoraSession {
let guard = self.inner.lock().await;
if let Some(state) = guard.as_ref() {
if let Some(audio) = state.audio.as_ref() {
let mut config = audio.audio_processing_config_snapshot();
config.route = route;
let config = chanora_audio::route_policy::apply_route_change(
&audio.audio_processing_config_snapshot(),
route,
);
audio.set_audio_processing_config(config)?;
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
crates/chanora_audio/Cargo.toml
+1 -7
View File
@@ -17,7 +17,7 @@ sonora = "0.1"
webrtc-vad = "0.4"
# ndarray is required by ort's tensor construction API and by
# Silero VAD ONNX inference across all platforms.
# Silero VAD ONNX inference on non-iOS targets.
ndarray = "0.17"
# Opus encoder. tsclientlib already pulls this; we depend explicitly so
@@ -50,12 +50,6 @@ coreaudio-rs = "0.14"
# on the main queue to avoid the VPIO RPC timeout on iOS simulator.
dispatch2 = "0.3"
[target.'cfg(target_os = "ios")'.dependencies]
# ONNX Runtime Rust binding for Silero VAD v6 (P1 VAD_002). The official
# iOS CocoaPod ships ONNX Runtime as a static framework, so iOS links it
# into chanora_bridge at build time instead of loading a dylib at runtime.
ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray"] }
[target.'cfg(not(target_os = "ios"))'.dependencies]
ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "api-24"] }
crates/chanora_audio/src/audio_processing.rs
+14 -1
View File
@@ -100,6 +100,16 @@ pub enum VadBackend {
Disabled,
}
#[cfg(target_os = "ios")]
fn default_vad_backend() -> VadBackend {
VadBackend::WebrtcVad
}
#[cfg(not(target_os = "ios"))]
fn default_vad_backend() -> VadBackend {
VadBackend::SileroOnnx
}
impl VadBackend {
/// Stable bridge/debug string.
pub fn as_str(self) -> &'static str {
@@ -164,7 +174,7 @@ impl Default for AudioProcessingConfig {
route: AudioRoute::Speaker,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::SileroOnnx,
vad_backend: default_vad_backend(),
aec: EffectOwner::Platform,
// iOS VPIO owns NS/AGC on the default shipping path. Software
// effects are opt-in through the experimental raw route only.
@@ -242,6 +252,9 @@ mod tests {
assert_eq!(config.aec, EffectOwner::Platform);
assert_eq!(config.ns, EffectOwner::Platform);
assert_eq!(config.agc, EffectOwner::Platform);
#[cfg(target_os = "ios")]
assert_eq!(config.vad_backend, VadBackend::WebrtcVad);
#[cfg(not(target_os = "ios"))]
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
}
crates/chanora_audio/src/engine.rs
+20 -14
View File
@@ -395,6 +395,22 @@ enum IosVoiceBackend {
#[cfg(any(target_os = "ios", target_os = "macos"))]
impl IosVoiceBackend {
fn restart(&mut self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
{
match self {
Self::Vpio(unit) => unit.restart(),
Self::Raw(unit) => unit.restart(),
}
}
#[cfg(target_os = "macos")]
{
match self {
Self::Vpio(_unit) => Ok(()),
}
}
}
fn pause(&mut self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
{
@@ -1404,21 +1420,11 @@ impl AudioEngine {
pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let backend = open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams {
handler: self.audio_handler.clone(),
output_gain: self.output_gain.clone(),
output_muted: self.output_muted.clone(),
voice_out_tx: self.voice_out_tx.clone(),
transmit_active: self.transmit_gate.flag_arc(),
frames_sent: self.frames_sent.clone(),
mic_gain: self.mic_gain,
voice_activity_selector: self.voice_activity_selector.clone(),
audio_processing_config: self.audio_processing_config.clone(),
audio_processing_stats: self.audio_processing_stats.clone(),
})?;
let mut guard = self._ios_voice_backend.lock().unwrap();
*guard = Some(backend);
Ok(())
let backend = guard
.as_mut()
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
backend.restart()
}
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
crates/chanora_audio/src/ios_raw_unit.rs
+9 -57
View File
@@ -116,9 +116,7 @@ mod inner {
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
/// Processing config — retained for route-change reloads.
@@ -154,9 +152,7 @@ mod inner {
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config: params.audio_processing_config.clone(),
@@ -276,7 +272,6 @@ mod inner {
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_vad_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
@@ -298,37 +293,20 @@ mod inner {
);
}
// Switch VAD backend only while VoiceActivity mode is active.
let silero_epoch = crate::vad::silero_model_epoch();
let silero_changed = voice_activity_mode
&& vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_changed) {
if voice_activity_mode && vad_backend != self.current_vad_backend {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
let path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
if self.silero_vad_worker.is_none() {
tracing::warn!(
target: "chanora_audio",
"Silero VAD model not found at {path}; falling back to WebRTC VAD"
);
}
}
_ => {
self.silero_vad_worker = None;
}
if vad_backend == crate::VadBackend::SileroOnnx {
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
} else {
self.audio_processing_stats.set_vad_fallback_active(false);
}
self.vad_state.reset();
}
let (vad_probability, active) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
@@ -336,35 +314,9 @@ mod inner {
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
crates/chanora_audio/src/ios_voice_unit.rs
+72 -109
View File
@@ -48,20 +48,15 @@
//!
//! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the
//! AudioUnit internally holds the C `AudioUnit` opaque pointer and
//! the wrapper's destructor calls `AudioComponentInstanceDispose`,
//! which (per Apple's threading rules) must be called from the
//! thread that owns the unit. We open the unit on the same thread
//! that calls `Self::start` (the tokio worker that runs
//! `chanora_core::ChanoraSession::start_audio`, the same pattern
//! cpal + SDL use) and never move it. The outer `AudioEngine`
//! already carries an `unsafe impl Send` to satisfy the same
//! constraint for cpal's `!Send` Stream type; that impl covers
//! VPIO too.
//!
//! Dropping `IosVoiceUnit` calls `audio_unit.stop()` via the
//! wrapper's `Drop`, which detaches the render + input callbacks
//! and stops the unit. The AudioHandler + CaptureState `Arc`s the
//! callbacks held are then released.
//! the wrapper's destructor calls `AudioComponentInstanceDispose`.
//! `IosVoiceUnit` stores the unit as `Option<AudioUnit>` so initial
//! setup and lifecycle operations can dispatch CoreAudio
//! initialize/start/stop calls to `DispatchQueue::main()` while the
//! wrapper keeps ownership and preserves the render/input callback
//! state in the surrounding `Arc`s. `restart`, `pause`, and
//! `resume` may temporarily move the unit through that helper, and
//! `Drop` stops it if still present before those callback `Arc`s are
//! released.
//!
//! ## What this file does NOT do
//!
@@ -143,13 +138,8 @@ struct IosCaptureState {
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
/// Background Silero worker — enqueues frames off the realtime
/// callback and publishes the latest probability atomically.
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
fallback_warned_backend: Option<crate::VadBackend>,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
@@ -187,9 +177,7 @@ impl IosCaptureState {
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config: params.audio_processing_config.clone(),
@@ -349,7 +337,7 @@ impl IosCaptureState {
false,
false,
true,
crate::VadBackend::SileroOnnx,
crate::VadBackend::WebrtcVad,
crate::voice_activity::VAD_HANGOVER_MS,
false,
crate::AudioRoute::Unknown,
@@ -362,18 +350,12 @@ impl IosCaptureState {
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_vad_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
// Switch VAD backend only while VoiceActivity mode is active.
let silero_model_epoch = crate::vad::silero_model_epoch();
let silero_model_changed = voice_activity_mode
&& vad_backend == crate::VadBackend::SileroOnnx
&& silero_model_epoch != self.silero_model_epoch;
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
if debug_wav_dump_enabled {
if recorder_guard.is_none() {
@@ -387,35 +369,14 @@ impl IosCaptureState {
}
}
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_model_changed)
{
if voice_activity_mode && vad_backend != self.current_vad_backend {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
self.silero_model_epoch = silero_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
// Attempt to load Silero model from the well-known
// bundle path. The actual inference runs on a
// background worker; the callback only enqueues
// 10 ms frames and falls back to WebRTC if the
// worker is missing or stale.
let model_path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path);
if self.silero_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"Silero VAD model not found at {model_path}; falling back to WebRTC VAD"
);
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
}
self.audio_processing_stats
.set_vad_fallback_active(self.silero_vad_worker.is_none());
}
_ => {
self.silero_vad_worker = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
if vad_backend == crate::VadBackend::SileroOnnx {
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
} else {
self.audio_processing_stats.set_vad_fallback_active(false);
}
// Reset VAD state machine timers on backend switch.
self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new(
@@ -457,7 +418,6 @@ impl IosCaptureState {
// VAD: only evaluate while VoiceActivity mode is active.
let (vad_probability, gate_open) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
@@ -465,32 +425,9 @@ impl IosCaptureState {
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
@@ -578,10 +515,39 @@ pub struct IosVoiceUnit {
// Drop = stop the audio unit (severs render callback). The
// wrapper's own Drop calls AudioComponentInstanceDispose
// after stop returns.
unit: AudioUnit,
unit: Option<AudioUnit>,
}
impl IosVoiceUnit {
#[cfg(target_os = "ios")]
fn exec_main_queue_lifecycle(
&mut self,
op: impl FnOnce(&mut AudioUnit) -> Result<(), String> + Send + 'static,
) -> Result<(), AudioError> {
let unit = self
.unit
.take()
.ok_or_else(|| AudioError::Backend("vpio lifecycle: audio unit missing".to_string()))?;
let (tx, rx) = std::sync::mpsc::sync_channel::<Result<(), String>>(1);
let unit_arc = Arc::new(Mutex::new(Some(unit)));
let unit_arc2 = Arc::clone(&unit_arc);
dispatch2::DispatchQueue::main().exec_async(move || {
let mut guard = unit_arc2.lock().unwrap();
let unit = guard.as_mut().unwrap();
let _ = tx.send(op(unit));
});
let result = match rx.recv() {
Ok(result) => result,
Err(_) => Err("vpio lifecycle: main thread channel closed unexpectedly".to_string()),
};
self.unit = unit_arc.lock().unwrap().take();
result.map_err(AudioError::Backend)
}
/// Open a VoiceProcessingIO AudioUnit, pin its stream format
/// to 48 kHz Int16 mono on both buses, install render + input
/// callbacks, and start it. The unit begins pumping audio
@@ -994,7 +960,7 @@ impl IosVoiceUnit {
),
}
Ok(Self { unit })
Ok(Self { unit: Some(unit) })
}
/// Restart the audio unit after route change handling.
@@ -1003,23 +969,18 @@ impl IosVoiceUnit {
/// VoiceProcessingIO unit through an uninitialize/reinitialize
/// cycle, then start again.
///
/// Called from the Flutter method channel handler which runs on
/// the main isolate — that runs on the main thread — so the
/// CoreAudio RPC is already on the correct thread here.
#[cfg(target_os = "ios")]
pub fn restart(&mut self) -> Result<(), AudioError> {
self.unit
.stop()
.map_err(|e| AudioError::Backend(format!("vpio restart stop: {e}")))?;
self.unit
.uninitialize()
.map_err(|e| AudioError::Backend(format!("vpio restart uninit: {e}")))?;
self.unit
.initialize()
.map_err(|e| AudioError::Backend(format!("vpio restart init: {e}")))?;
self.unit
.start()
.map_err(|e| AudioError::Backend(format!("vpio restart start: {e}")))?;
self.exec_main_queue_lifecycle(|unit| {
unit.stop().map_err(|e| format!("vpio restart stop: {e}"))?;
unit.uninitialize()
.map_err(|e| format!("vpio restart uninit: {e}"))?;
unit.initialize()
.map_err(|e| format!("vpio restart init: {e}"))?;
unit.start()
.map_err(|e| format!("vpio restart start: {e}"))?;
Ok(())
})?;
info!(target: "chanora_audio", "ios VPIO audio unit restarted");
Ok(())
}
@@ -1027,17 +988,17 @@ impl IosVoiceUnit {
/// Pause the audio unit during an interruption.
#[cfg(target_os = "ios")]
pub fn pause(&mut self) -> Result<(), AudioError> {
self.unit
.stop()
.map_err(|e| AudioError::Backend(format!("vpio pause stop: {e}")))
self.exec_main_queue_lifecycle(|unit| {
unit.stop().map_err(|e| format!("vpio pause stop: {e}"))
})
}
/// Resume the audio unit after an interruption.
#[cfg(target_os = "ios")]
pub fn resume(&mut self) -> Result<(), AudioError> {
self.unit
.start()
.map_err(|e| AudioError::Backend(format!("vpio resume start: {e}")))
self.exec_main_queue_lifecycle(|unit| {
unit.start().map_err(|e| format!("vpio resume start: {e}"))
})
}
}
@@ -1046,10 +1007,12 @@ impl Drop for IosVoiceUnit {
// Stop the audio unit so the render callback no longer
// fires. The coreaudio-rs wrapper's own Drop calls
// AudioComponentInstanceDispose afterwards.
if let Err(e) = self.unit.stop() {
warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed");
} else {
info!(target: "chanora_audio", "ios audio unit stopped");
if let Some(unit) = self.unit.as_mut() {
if let Err(e) = unit.stop() {
warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed");
} else {
info!(target: "chanora_audio", "ios audio unit stopped");
}
}
}
}
crates/chanora_audio/src/route_policy.rs
+9 -4
View File
@@ -30,7 +30,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::SileroOnnx,
vad_backend: VadBackend::WebrtcVad,
aec: EffectOwner::Platform,
// VPIO owns NS and AGC on the shipping default path (IOSP_002/003).
ns: EffectOwner::Platform,
@@ -43,7 +43,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::Noop,
vad_backend: VadBackend::SileroOnnx,
vad_backend: VadBackend::WebrtcVad,
// No AEC needed for wired headset (no acoustic echo path).
aec: EffectOwner::Off,
// Conservative NS/AGC: optional, not forced.
@@ -57,7 +57,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::SileroOnnx,
vad_backend: VadBackend::WebrtcVad,
// BT HFP manages its own AEC in the headset firmware.
aec: EffectOwner::Off,
ns: EffectOwner::Conservative,
@@ -87,7 +87,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::Noop,
vad_backend: VadBackend::SileroOnnx,
vad_backend: VadBackend::WebrtcVad,
// Safe fallback: AEC off until route is classified.
aec: EffectOwner::Off,
ns: EffectOwner::Off,
@@ -146,6 +146,7 @@ mod tests {
cfg.ios_mode,
IosVoiceProcessingMode::PlatformVoiceProcessing
);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
}
#[test]
@@ -156,6 +157,7 @@ mod tests {
AudioBackend::PlatformVoiceProcessing
);
assert_eq!(cfg.aec, EffectOwner::Platform);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
}
#[test]
@@ -163,12 +165,14 @@ mod tests {
let cfg = ios_route_policy(AudioRoute::WiredHeadset);
assert_eq!(cfg.aec, EffectOwner::Off);
assert_eq!(cfg.processing_backend, AudioBackend::Noop);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
}
#[test]
fn bluetooth_hfp_disables_app_aec() {
let cfg = ios_route_policy(AudioRoute::BluetoothHfp);
assert_eq!(cfg.aec, EffectOwner::Off);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
}
#[test]
@@ -183,6 +187,7 @@ mod tests {
fn unknown_route_safe_fallback_no_aec() {
let cfg = ios_route_policy(AudioRoute::Unknown);
assert_eq!(cfg.aec, EffectOwner::Off);
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
}
#[test]
crates/chanora_audio/src/vad/mod.rs
+14 -11
View File
@@ -1,11 +1,12 @@
//! Voice activity detection backends and helpers.
//!
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into
//! this module. The production path prefers a model-backed detector when
//! available, and otherwise uses the realtime-safe fallback below so
//! this module and uses the realtime-safe WebRTC fallback. Other
//! platforms may use a model-backed detector when available so
//! VoiceActivity mode never collapses back to Continuous transmit.
pub mod resampler;
#[cfg(not(target_os = "ios"))]
pub mod silero_onnx;
use std::sync::atomic::{AtomicU64, Ordering};
@@ -15,6 +16,7 @@ use crate::frame::{f32_to_i16, i16_to_f32};
use crate::AudioError;
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
#[cfg(not(target_os = "ios"))]
pub use silero_onnx::SileroOnnxVad;
/// Voice activity detector output for one 10 ms frame.
@@ -111,11 +113,11 @@ fn silero_model_path_override() -> &'static RwLock<Option<String>> {
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
}
/// Configure the preferred Silero ONNX model path.
/// Configure the preferred Silero ONNX model path on supported platforms.
///
/// The path is validated eagerly. A successful call increments the
/// model epoch so running audio backends can reload the model without
/// an app restart.
/// model epoch so running non-iOS audio backends can reload the model
/// without an app restart.
pub fn set_silero_model_path(path: &str) -> Result<(), AudioError> {
let path = path.trim();
if path.is_empty() {
@@ -141,13 +143,13 @@ pub fn silero_model_epoch() -> u64 {
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
}
/// Return the expected path of the Silero VAD v6 ONNX model.
/// Return the expected path of the Silero VAD v6 ONNX model on
/// supported platforms.
/// The model is shipped as a Flutter asset and copied to the app's
/// data directory by the Dart-side asset loader.
///
/// On iOS/Android the model lives in the app's Documents/files
/// directory. On desktop, the caller should set the path explicitly
/// via `set_silero_model_path`.
/// Android and macOS may use app data/Documents locations. Desktop
/// callers can set the path explicitly via `set_silero_model_path`.
pub fn silero_model_bundle_path() -> String {
if let Ok(guard) = silero_model_path_override().read() {
if let Some(path) = guard.as_ref() {
@@ -155,8 +157,9 @@ pub fn silero_model_bundle_path() -> String {
}
}
// iOS: Documents directory (written by Flutter asset loader).
// macOS: same Documents pattern.
// macOS: Documents directory (written by Flutter asset loader).
// iOS keeps this fallback only for API compatibility; the ONNX
// detector is not compiled into iOS builds.
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
if let Ok(home) = std::env::var("HOME") {
crates/chanora_bridge/Cargo.toml
+1 -1
View File
@@ -23,7 +23,7 @@ thiserror.workspace = true
serde.workspace = true
tracing.workspace = true
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] }
# Android: route `tracing` output to logcat so a user can see protocol
# and audio diagnostics via `adb logcat -s chanora`. Also brings in the
crates/chanora_bridge/src/api.rs
+74 -26
View File
@@ -11,7 +11,7 @@ use std::time::Duration;
use flutter_rust_bridge::frb;
use tokio::runtime::Runtime;
use tokio::sync::broadcast;
use tokio::sync::{broadcast, mpsc};
use tracing::{info, warn};
use crate::frb_generated::StreamSink;
@@ -42,6 +42,73 @@ fn task_join_error(task: &'static str, error: tokio::task::JoinError) -> BridgeE
BridgeError::Unmapped(format!("join: {error}"))
}
enum PlatformAudioEvent {
RouteChanged(chanora_audio::AudioRoute),
MediaServicesResetWithRoute(chanora_audio::AudioRoute),
InterruptionBegan,
InterruptionEnded { should_resume: bool },
AudioOutputRoute(chanora_audio::AudioRoute),
Lifecycle { state: String },
}
impl PlatformAudioEvent {
async fn process(self) {
match self {
Self::RouteChanged(route) => {
if let Err(e) = session().ios_handle_route_change(route).await {
warn!(target: "chanora_bridge", error = %e, "iOS route-change handling failed");
}
}
Self::MediaServicesResetWithRoute(route) => {
if let Err(e) = session().ios_handle_media_services_reset(route).await {
warn!(target: "chanora_bridge", error = %e, "iOS media-services reset (with route) handling failed");
}
}
Self::InterruptionBegan => {
if let Err(e) = session().ios_handle_interruption_began().await {
warn!(target: "chanora_bridge", error = %e, "iOS interruption-began handling failed");
}
}
Self::InterruptionEnded { should_resume } => {
if let Err(e) = session().ios_handle_interruption_ended(should_resume).await {
warn!(target: "chanora_bridge", error = %e, "iOS interruption-ended handling failed");
}
}
Self::AudioOutputRoute(route) => {
if let Err(e) = session().ios_handle_route_change(route).await {
warn!(target: "chanora_bridge", error = %e, "audio output route handling failed");
}
}
Self::Lifecycle { state } => {
session().record_lifecycle_event(&state).await;
}
}
}
}
fn platform_audio_events() -> &'static mpsc::UnboundedSender<PlatformAudioEvent> {
static TX: OnceLock<mpsc::UnboundedSender<PlatformAudioEvent>> = OnceLock::new();
TX.get_or_init(|| {
let (tx, mut rx) = mpsc::unbounded_channel::<PlatformAudioEvent>();
runtime().spawn(async move {
while let Some(event) = rx.recv().await {
event.process().await;
}
});
tx
})
}
fn dispatch_platform_audio_event(event: PlatformAudioEvent) {
if let Err(e) = platform_audio_events().send(event) {
warn!(
target: "chanora_bridge",
error = %e,
"ordered platform audio event dispatch failed"
);
}
}
fn install_panic_diagnostic_hook() {
static INSTALLED: OnceLock<()> = OnceLock::new();
let _ = INSTALLED.get_or_init(|| {
@@ -615,11 +682,7 @@ pub async fn is_connected() -> bool {
/// Handle iOS AVAudioSession route changes (SDD-100).
#[frb(sync)]
pub fn handle_route_change(route: BridgeAudioRoute) {
let result =
runtime().block_on(async { session().ios_handle_route_change(route.into()).await });
if let Err(e) = result {
warn!(target: "chanora_bridge", error = %e, "iOS route-change handling failed");
}
dispatch_platform_audio_event(PlatformAudioEvent::RouteChanged(route.into()));
}
/// Handle iOS AVAudioSession media-services reset with the current
@@ -629,30 +692,19 @@ pub fn handle_route_change(route: BridgeAudioRoute) {
#[frb(sync)]
pub fn handle_media_services_reset_with_route(route_class: String) {
let route = chanora_audio::AudioRoute::from_route_class(&route_class);
let result =
runtime().block_on(async { session().ios_handle_media_services_reset(route).await });
if let Err(e) = result {
warn!(target: "chanora_bridge", error = %e, "iOS media-services reset (with route) handling failed");
}
dispatch_platform_audio_event(PlatformAudioEvent::MediaServicesResetWithRoute(route));
}
/// Handle iOS AVAudioSession interruption begin (SDD-101).
#[frb(sync)]
pub fn handle_interruption_began() {
let result = runtime().block_on(async { session().ios_handle_interruption_began().await });
if let Err(e) = result {
warn!(target: "chanora_bridge", error = %e, "iOS interruption-began handling failed");
}
dispatch_platform_audio_event(PlatformAudioEvent::InterruptionBegan);
}
/// Handle iOS AVAudioSession interruption end (SDD-101).
#[frb(sync)]
pub fn handle_interruption_ended(should_resume: bool) {
let result =
runtime().block_on(async { session().ios_handle_interruption_ended(should_resume).await });
if let Err(e) = result {
warn!(target: "chanora_bridge", error = %e, "iOS interruption-ended handling failed");
}
dispatch_platform_audio_event(PlatformAudioEvent::InterruptionEnded { should_resume });
}
/// Set the focused/on-screen push-to-talk hold state.
@@ -2139,14 +2191,10 @@ pub async fn set_ios_voice_processing_mode(
/// Set the preferred audio output route (Android/iOS).
#[frb(sync)]
pub fn set_audio_output_route(route: BridgeAudioRoute) {
runtime().block_on(async {
let _ = session().ios_handle_route_change(route.into()).await;
});
dispatch_platform_audio_event(PlatformAudioEvent::AudioOutputRoute(route.into()));
}
/// Called from Flutter when the app enters background/foreground.
#[frb(sync)]
pub fn record_lifecycle_event(state: String) {
runtime().block_on(async {
session().record_lifecycle_event(&state).await;
});
dispatch_platform_audio_event(PlatformAudioEvent::Lifecycle { state });
}