fix(audio): use WebRTC VAD on iOS
This commit is contained in:
@@ -17,7 +17,7 @@ sonora = "0.1"
|
|||||||
webrtc-vad = "0.4"
|
webrtc-vad = "0.4"
|
||||||
|
|
||||||
# ndarray is required by ort's tensor construction API and by
|
# ndarray is required by ort's tensor construction API and by
|
||||||
# Silero VAD ONNX inference across all platforms.
|
# Silero VAD ONNX inference on non-iOS targets.
|
||||||
ndarray = "0.17"
|
ndarray = "0.17"
|
||||||
|
|
||||||
# Opus encoder. tsclientlib already pulls this; we depend explicitly so
|
# Opus encoder. tsclientlib already pulls this; we depend explicitly so
|
||||||
@@ -50,12 +50,6 @@ coreaudio-rs = "0.14"
|
|||||||
# on the main queue to avoid the VPIO RPC timeout on iOS simulator.
|
# on the main queue to avoid the VPIO RPC timeout on iOS simulator.
|
||||||
dispatch2 = "0.3"
|
dispatch2 = "0.3"
|
||||||
|
|
||||||
[target.'cfg(target_os = "ios")'.dependencies]
|
|
||||||
# ONNX Runtime Rust binding for Silero VAD v6 (P1 VAD_002). The official
|
|
||||||
# iOS CocoaPod ships ONNX Runtime as a static framework, so iOS links it
|
|
||||||
# into chanora_bridge at build time instead of loading a dylib at runtime.
|
|
||||||
ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray"] }
|
|
||||||
|
|
||||||
[target.'cfg(not(target_os = "ios"))'.dependencies]
|
[target.'cfg(not(target_os = "ios"))'.dependencies]
|
||||||
ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "api-24"] }
|
ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "api-24"] }
|
||||||
|
|
||||||
|
|||||||
@@ -100,6 +100,16 @@ pub enum VadBackend {
|
|||||||
Disabled,
|
Disabled,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "ios")]
|
||||||
|
fn default_vad_backend() -> VadBackend {
|
||||||
|
VadBackend::WebrtcVad
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "ios"))]
|
||||||
|
fn default_vad_backend() -> VadBackend {
|
||||||
|
VadBackend::SileroOnnx
|
||||||
|
}
|
||||||
|
|
||||||
impl VadBackend {
|
impl VadBackend {
|
||||||
/// Stable bridge/debug string.
|
/// Stable bridge/debug string.
|
||||||
pub fn as_str(self) -> &'static str {
|
pub fn as_str(self) -> &'static str {
|
||||||
@@ -164,7 +174,7 @@ impl Default for AudioProcessingConfig {
|
|||||||
route: AudioRoute::Speaker,
|
route: AudioRoute::Speaker,
|
||||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||||
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
||||||
vad_backend: VadBackend::SileroOnnx,
|
vad_backend: default_vad_backend(),
|
||||||
aec: EffectOwner::Platform,
|
aec: EffectOwner::Platform,
|
||||||
// iOS VPIO owns NS/AGC on the default shipping path. Software
|
// iOS VPIO owns NS/AGC on the default shipping path. Software
|
||||||
// effects are opt-in through the experimental raw route only.
|
// effects are opt-in through the experimental raw route only.
|
||||||
@@ -242,6 +252,9 @@ mod tests {
|
|||||||
assert_eq!(config.aec, EffectOwner::Platform);
|
assert_eq!(config.aec, EffectOwner::Platform);
|
||||||
assert_eq!(config.ns, EffectOwner::Platform);
|
assert_eq!(config.ns, EffectOwner::Platform);
|
||||||
assert_eq!(config.agc, EffectOwner::Platform);
|
assert_eq!(config.agc, EffectOwner::Platform);
|
||||||
|
#[cfg(target_os = "ios")]
|
||||||
|
assert_eq!(config.vad_backend, VadBackend::WebrtcVad);
|
||||||
|
#[cfg(not(target_os = "ios"))]
|
||||||
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
|
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -116,9 +116,7 @@ mod inner {
|
|||||||
mic_gain: f32,
|
mic_gain: f32,
|
||||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||||
vad_detector: crate::vad::WebRtcFallbackVad,
|
vad_detector: crate::vad::WebRtcFallbackVad,
|
||||||
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
|
|
||||||
current_vad_backend: crate::VadBackend,
|
current_vad_backend: crate::VadBackend,
|
||||||
silero_model_epoch: u64,
|
|
||||||
capture_frame_seq: u64,
|
capture_frame_seq: u64,
|
||||||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||||||
/// Processing config — retained for route-change reloads.
|
/// Processing config — retained for route-change reloads.
|
||||||
@@ -154,9 +152,7 @@ mod inner {
|
|||||||
mic_gain: params.mic_gain,
|
mic_gain: params.mic_gain,
|
||||||
voice_activity_selector: params.voice_activity_selector.clone(),
|
voice_activity_selector: params.voice_activity_selector.clone(),
|
||||||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||||||
silero_vad_worker: None,
|
|
||||||
current_vad_backend: crate::VadBackend::WebrtcVad,
|
current_vad_backend: crate::VadBackend::WebrtcVad,
|
||||||
silero_model_epoch: crate::vad::silero_model_epoch(),
|
|
||||||
capture_frame_seq: 0,
|
capture_frame_seq: 0,
|
||||||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||||||
audio_processing_config: params.audio_processing_config.clone(),
|
audio_processing_config: params.audio_processing_config.clone(),
|
||||||
@@ -276,7 +272,6 @@ mod inner {
|
|||||||
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
if !voice_activity_mode {
|
if !voice_activity_mode {
|
||||||
self.silero_vad_worker = None;
|
|
||||||
self.current_vad_backend = crate::VadBackend::Disabled;
|
self.current_vad_backend = crate::VadBackend::Disabled;
|
||||||
self.fallback_warned_backend = None;
|
self.fallback_warned_backend = None;
|
||||||
self.audio_processing_stats.set_vad_fallback_active(false);
|
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||||
@@ -298,37 +293,20 @@ mod inner {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Switch VAD backend only while VoiceActivity mode is active.
|
if voice_activity_mode && vad_backend != self.current_vad_backend {
|
||||||
let silero_epoch = crate::vad::silero_model_epoch();
|
|
||||||
let silero_changed = voice_activity_mode
|
|
||||||
&& vad_backend == crate::VadBackend::SileroOnnx
|
|
||||||
&& silero_epoch != self.silero_model_epoch;
|
|
||||||
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_changed) {
|
|
||||||
self.current_vad_backend = vad_backend;
|
self.current_vad_backend = vad_backend;
|
||||||
self.silero_model_epoch = silero_epoch;
|
|
||||||
self.fallback_warned_backend = None;
|
self.fallback_warned_backend = None;
|
||||||
match vad_backend {
|
if vad_backend == crate::VadBackend::SileroOnnx {
|
||||||
crate::VadBackend::SileroOnnx => {
|
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||||||
let path = crate::vad::silero_model_bundle_path();
|
self.audio_processing_stats.set_vad_fallback_active(true);
|
||||||
self.silero_vad_worker =
|
} else {
|
||||||
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
|
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||||
if self.silero_vad_worker.is_none() {
|
|
||||||
tracing::warn!(
|
|
||||||
target: "chanora_audio",
|
|
||||||
"Silero VAD model not found at {path}; falling back to WebRTC VAD"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
self.silero_vad_worker = None;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
self.vad_state.reset();
|
self.vad_state.reset();
|
||||||
}
|
}
|
||||||
|
|
||||||
let (vad_probability, active) = if voice_activity_mode {
|
let (vad_probability, active) = if voice_activity_mode {
|
||||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||||
let capture_seq = self.capture_frame_seq;
|
|
||||||
let mut used_fallback_vad = false;
|
let mut used_fallback_vad = false;
|
||||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||||
crate::vad::VadOutput {
|
crate::vad::VadOutput {
|
||||||
@@ -336,35 +314,9 @@ mod inner {
|
|||||||
speech: true,
|
speech: true,
|
||||||
}
|
}
|
||||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
used_fallback_vad = true;
|
||||||
let enqueued = worker.try_send(capture_seq, &frame);
|
self.mark_vad_fallback_active(vad_backend);
|
||||||
if !worker.is_stale(capture_seq) {
|
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||||
let p = worker.latest_probability();
|
|
||||||
crate::vad::VadOutput {
|
|
||||||
probability: p,
|
|
||||||
speech: p >= 0.5,
|
|
||||||
}
|
|
||||||
} else if enqueued {
|
|
||||||
crate::vad::VadOutput {
|
|
||||||
probability: 0.0,
|
|
||||||
speech: false,
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
used_fallback_vad = true;
|
|
||||||
self.mark_vad_fallback_active(vad_backend);
|
|
||||||
crate::vad::VoiceActivityDetector::process_10ms(
|
|
||||||
&mut self.vad_detector,
|
|
||||||
&frame,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
used_fallback_vad = true;
|
|
||||||
self.mark_vad_fallback_active(vad_backend);
|
|
||||||
crate::vad::VoiceActivityDetector::process_10ms(
|
|
||||||
&mut self.vad_detector,
|
|
||||||
&frame,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -48,20 +48,15 @@
|
|||||||
//!
|
//!
|
||||||
//! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the
|
//! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the
|
||||||
//! AudioUnit internally holds the C `AudioUnit` opaque pointer and
|
//! AudioUnit internally holds the C `AudioUnit` opaque pointer and
|
||||||
//! the wrapper's destructor calls `AudioComponentInstanceDispose`,
|
//! the wrapper's destructor calls `AudioComponentInstanceDispose`.
|
||||||
//! which (per Apple's threading rules) must be called from the
|
//! `IosVoiceUnit` stores the unit as `Option<AudioUnit>` so initial
|
||||||
//! thread that owns the unit. We open the unit on the same thread
|
//! setup and lifecycle operations can dispatch CoreAudio
|
||||||
//! that calls `Self::start` (the tokio worker that runs
|
//! initialize/start/stop calls to `DispatchQueue::main()` while the
|
||||||
//! `chanora_core::ChanoraSession::start_audio`, the same pattern
|
//! wrapper keeps ownership and preserves the render/input callback
|
||||||
//! cpal + SDL use) and never move it. The outer `AudioEngine`
|
//! state in the surrounding `Arc`s. `restart`, `pause`, and
|
||||||
//! already carries an `unsafe impl Send` to satisfy the same
|
//! `resume` may temporarily move the unit through that helper, and
|
||||||
//! constraint for cpal's `!Send` Stream type; that impl covers
|
//! `Drop` stops it if still present before those callback `Arc`s are
|
||||||
//! VPIO too.
|
//! released.
|
||||||
//!
|
|
||||||
//! Dropping `IosVoiceUnit` calls `audio_unit.stop()` via the
|
|
||||||
//! wrapper's `Drop`, which detaches the render + input callbacks
|
|
||||||
//! and stops the unit. The AudioHandler + CaptureState `Arc`s the
|
|
||||||
//! callbacks held are then released.
|
|
||||||
//!
|
//!
|
||||||
//! ## What this file does NOT do
|
//! ## What this file does NOT do
|
||||||
//!
|
//!
|
||||||
@@ -143,13 +138,8 @@ struct IosCaptureState {
|
|||||||
mic_gain: f32,
|
mic_gain: f32,
|
||||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||||
vad_detector: crate::vad::WebRtcFallbackVad,
|
vad_detector: crate::vad::WebRtcFallbackVad,
|
||||||
/// Background Silero worker — enqueues frames off the realtime
|
|
||||||
/// callback and publishes the latest probability atomically.
|
|
||||||
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
|
|
||||||
/// Last VAD backend we configured — used to detect backend changes.
|
/// Last VAD backend we configured — used to detect backend changes.
|
||||||
current_vad_backend: crate::VadBackend,
|
current_vad_backend: crate::VadBackend,
|
||||||
/// Last observed configured Silero model epoch.
|
|
||||||
silero_model_epoch: u64,
|
|
||||||
fallback_warned_backend: Option<crate::VadBackend>,
|
fallback_warned_backend: Option<crate::VadBackend>,
|
||||||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||||
@@ -187,9 +177,7 @@ impl IosCaptureState {
|
|||||||
mic_gain: params.mic_gain,
|
mic_gain: params.mic_gain,
|
||||||
voice_activity_selector: params.voice_activity_selector.clone(),
|
voice_activity_selector: params.voice_activity_selector.clone(),
|
||||||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||||||
silero_vad_worker: None,
|
|
||||||
current_vad_backend: crate::VadBackend::WebrtcVad,
|
current_vad_backend: crate::VadBackend::WebrtcVad,
|
||||||
silero_model_epoch: crate::vad::silero_model_epoch(),
|
|
||||||
fallback_warned_backend: None,
|
fallback_warned_backend: None,
|
||||||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||||||
audio_processing_config: params.audio_processing_config.clone(),
|
audio_processing_config: params.audio_processing_config.clone(),
|
||||||
@@ -349,7 +337,7 @@ impl IosCaptureState {
|
|||||||
false,
|
false,
|
||||||
false,
|
false,
|
||||||
true,
|
true,
|
||||||
crate::VadBackend::SileroOnnx,
|
crate::VadBackend::WebrtcVad,
|
||||||
crate::voice_activity::VAD_HANGOVER_MS,
|
crate::voice_activity::VAD_HANGOVER_MS,
|
||||||
false,
|
false,
|
||||||
crate::AudioRoute::Unknown,
|
crate::AudioRoute::Unknown,
|
||||||
@@ -362,18 +350,12 @@ impl IosCaptureState {
|
|||||||
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
if !voice_activity_mode {
|
if !voice_activity_mode {
|
||||||
self.silero_vad_worker = None;
|
|
||||||
self.current_vad_backend = crate::VadBackend::Disabled;
|
self.current_vad_backend = crate::VadBackend::Disabled;
|
||||||
self.fallback_warned_backend = None;
|
self.fallback_warned_backend = None;
|
||||||
self.audio_processing_stats.set_vad_fallback_active(false);
|
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Switch VAD backend only while VoiceActivity mode is active.
|
// Switch VAD backend only while VoiceActivity mode is active.
|
||||||
let silero_model_epoch = crate::vad::silero_model_epoch();
|
|
||||||
let silero_model_changed = voice_activity_mode
|
|
||||||
&& vad_backend == crate::VadBackend::SileroOnnx
|
|
||||||
&& silero_model_epoch != self.silero_model_epoch;
|
|
||||||
|
|
||||||
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
|
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
|
||||||
if debug_wav_dump_enabled {
|
if debug_wav_dump_enabled {
|
||||||
if recorder_guard.is_none() {
|
if recorder_guard.is_none() {
|
||||||
@@ -387,35 +369,14 @@ impl IosCaptureState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_model_changed)
|
if voice_activity_mode && vad_backend != self.current_vad_backend {
|
||||||
{
|
|
||||||
self.current_vad_backend = vad_backend;
|
self.current_vad_backend = vad_backend;
|
||||||
self.fallback_warned_backend = None;
|
self.fallback_warned_backend = None;
|
||||||
self.silero_model_epoch = silero_model_epoch;
|
if vad_backend == crate::VadBackend::SileroOnnx {
|
||||||
match vad_backend {
|
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||||||
crate::VadBackend::SileroOnnx => {
|
self.audio_processing_stats.set_vad_fallback_active(true);
|
||||||
// Attempt to load Silero model from the well-known
|
} else {
|
||||||
// bundle path. The actual inference runs on a
|
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||||
// background worker; the callback only enqueues
|
|
||||||
// 10 ms frames and falls back to WebRTC if the
|
|
||||||
// worker is missing or stale.
|
|
||||||
let model_path = crate::vad::silero_model_bundle_path();
|
|
||||||
self.silero_vad_worker =
|
|
||||||
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path);
|
|
||||||
if self.silero_vad_worker.is_none() {
|
|
||||||
warn!(
|
|
||||||
target: "chanora_audio",
|
|
||||||
"Silero VAD model not found at {model_path}; falling back to WebRTC VAD"
|
|
||||||
);
|
|
||||||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
|
||||||
}
|
|
||||||
self.audio_processing_stats
|
|
||||||
.set_vad_fallback_active(self.silero_vad_worker.is_none());
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
self.silero_vad_worker = None;
|
|
||||||
self.audio_processing_stats.set_vad_fallback_active(false);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// Reset VAD state machine timers on backend switch.
|
// Reset VAD state machine timers on backend switch.
|
||||||
self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new(
|
self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new(
|
||||||
@@ -457,7 +418,6 @@ impl IosCaptureState {
|
|||||||
// VAD: only evaluate while VoiceActivity mode is active.
|
// VAD: only evaluate while VoiceActivity mode is active.
|
||||||
let (vad_probability, gate_open) = if voice_activity_mode {
|
let (vad_probability, gate_open) = if voice_activity_mode {
|
||||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||||
let capture_seq = self.capture_frame_seq;
|
|
||||||
let mut used_fallback_vad = false;
|
let mut used_fallback_vad = false;
|
||||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||||
crate::vad::VadOutput {
|
crate::vad::VadOutput {
|
||||||
@@ -465,32 +425,9 @@ impl IosCaptureState {
|
|||||||
speech: true,
|
speech: true,
|
||||||
}
|
}
|
||||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
used_fallback_vad = true;
|
||||||
let enqueued = worker.try_send(capture_seq, &frame);
|
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||||||
if !worker.is_stale(capture_seq) {
|
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||||
let probability = worker.latest_probability();
|
|
||||||
crate::vad::VadOutput {
|
|
||||||
probability,
|
|
||||||
speech: probability >= 0.5,
|
|
||||||
}
|
|
||||||
} else if enqueued {
|
|
||||||
crate::vad::VadOutput {
|
|
||||||
probability: 0.0,
|
|
||||||
speech: false,
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
used_fallback_vad = true;
|
|
||||||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
|
||||||
crate::vad::VoiceActivityDetector::process_10ms(
|
|
||||||
&mut self.vad_detector,
|
|
||||||
&frame,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
used_fallback_vad = true;
|
|
||||||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
|
||||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||||
};
|
};
|
||||||
@@ -578,10 +515,39 @@ pub struct IosVoiceUnit {
|
|||||||
// Drop = stop the audio unit (severs render callback). The
|
// Drop = stop the audio unit (severs render callback). The
|
||||||
// wrapper's own Drop calls AudioComponentInstanceDispose
|
// wrapper's own Drop calls AudioComponentInstanceDispose
|
||||||
// after stop returns.
|
// after stop returns.
|
||||||
unit: AudioUnit,
|
unit: Option<AudioUnit>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IosVoiceUnit {
|
impl IosVoiceUnit {
|
||||||
|
#[cfg(target_os = "ios")]
|
||||||
|
fn exec_main_queue_lifecycle(
|
||||||
|
&mut self,
|
||||||
|
op: impl FnOnce(&mut AudioUnit) -> Result<(), String> + Send + 'static,
|
||||||
|
) -> Result<(), AudioError> {
|
||||||
|
let unit = self
|
||||||
|
.unit
|
||||||
|
.take()
|
||||||
|
.ok_or_else(|| AudioError::Backend("vpio lifecycle: audio unit missing".to_string()))?;
|
||||||
|
|
||||||
|
let (tx, rx) = std::sync::mpsc::sync_channel::<Result<(), String>>(1);
|
||||||
|
let unit_arc = Arc::new(Mutex::new(Some(unit)));
|
||||||
|
let unit_arc2 = Arc::clone(&unit_arc);
|
||||||
|
|
||||||
|
dispatch2::DispatchQueue::main().exec_async(move || {
|
||||||
|
let mut guard = unit_arc2.lock().unwrap();
|
||||||
|
let unit = guard.as_mut().unwrap();
|
||||||
|
let _ = tx.send(op(unit));
|
||||||
|
});
|
||||||
|
|
||||||
|
let result = match rx.recv() {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(_) => Err("vpio lifecycle: main thread channel closed unexpectedly".to_string()),
|
||||||
|
};
|
||||||
|
|
||||||
|
self.unit = unit_arc.lock().unwrap().take();
|
||||||
|
result.map_err(AudioError::Backend)
|
||||||
|
}
|
||||||
|
|
||||||
/// Open a VoiceProcessingIO AudioUnit, pin its stream format
|
/// Open a VoiceProcessingIO AudioUnit, pin its stream format
|
||||||
/// to 48 kHz Int16 mono on both buses, install render + input
|
/// to 48 kHz Int16 mono on both buses, install render + input
|
||||||
/// callbacks, and start it. The unit begins pumping audio
|
/// callbacks, and start it. The unit begins pumping audio
|
||||||
@@ -994,7 +960,7 @@ impl IosVoiceUnit {
|
|||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Self { unit })
|
Ok(Self { unit: Some(unit) })
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restart the audio unit after route change handling.
|
/// Restart the audio unit after route change handling.
|
||||||
@@ -1003,23 +969,18 @@ impl IosVoiceUnit {
|
|||||||
/// VoiceProcessingIO unit through an uninitialize/reinitialize
|
/// VoiceProcessingIO unit through an uninitialize/reinitialize
|
||||||
/// cycle, then start again.
|
/// cycle, then start again.
|
||||||
///
|
///
|
||||||
/// Called from the Flutter method channel handler which runs on
|
|
||||||
/// the main isolate — that runs on the main thread — so the
|
|
||||||
/// CoreAudio RPC is already on the correct thread here.
|
|
||||||
#[cfg(target_os = "ios")]
|
#[cfg(target_os = "ios")]
|
||||||
pub fn restart(&mut self) -> Result<(), AudioError> {
|
pub fn restart(&mut self) -> Result<(), AudioError> {
|
||||||
self.unit
|
self.exec_main_queue_lifecycle(|unit| {
|
||||||
.stop()
|
unit.stop().map_err(|e| format!("vpio restart stop: {e}"))?;
|
||||||
.map_err(|e| AudioError::Backend(format!("vpio restart stop: {e}")))?;
|
unit.uninitialize()
|
||||||
self.unit
|
.map_err(|e| format!("vpio restart uninit: {e}"))?;
|
||||||
.uninitialize()
|
unit.initialize()
|
||||||
.map_err(|e| AudioError::Backend(format!("vpio restart uninit: {e}")))?;
|
.map_err(|e| format!("vpio restart init: {e}"))?;
|
||||||
self.unit
|
unit.start()
|
||||||
.initialize()
|
.map_err(|e| format!("vpio restart start: {e}"))?;
|
||||||
.map_err(|e| AudioError::Backend(format!("vpio restart init: {e}")))?;
|
Ok(())
|
||||||
self.unit
|
})?;
|
||||||
.start()
|
|
||||||
.map_err(|e| AudioError::Backend(format!("vpio restart start: {e}")))?;
|
|
||||||
info!(target: "chanora_audio", "ios VPIO audio unit restarted");
|
info!(target: "chanora_audio", "ios VPIO audio unit restarted");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1027,17 +988,17 @@ impl IosVoiceUnit {
|
|||||||
/// Pause the audio unit during an interruption.
|
/// Pause the audio unit during an interruption.
|
||||||
#[cfg(target_os = "ios")]
|
#[cfg(target_os = "ios")]
|
||||||
pub fn pause(&mut self) -> Result<(), AudioError> {
|
pub fn pause(&mut self) -> Result<(), AudioError> {
|
||||||
self.unit
|
self.exec_main_queue_lifecycle(|unit| {
|
||||||
.stop()
|
unit.stop().map_err(|e| format!("vpio pause stop: {e}"))
|
||||||
.map_err(|e| AudioError::Backend(format!("vpio pause stop: {e}")))
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resume the audio unit after an interruption.
|
/// Resume the audio unit after an interruption.
|
||||||
#[cfg(target_os = "ios")]
|
#[cfg(target_os = "ios")]
|
||||||
pub fn resume(&mut self) -> Result<(), AudioError> {
|
pub fn resume(&mut self) -> Result<(), AudioError> {
|
||||||
self.unit
|
self.exec_main_queue_lifecycle(|unit| {
|
||||||
.start()
|
unit.start().map_err(|e| format!("vpio resume start: {e}"))
|
||||||
.map_err(|e| AudioError::Backend(format!("vpio resume start: {e}")))
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1046,10 +1007,12 @@ impl Drop for IosVoiceUnit {
|
|||||||
// Stop the audio unit so the render callback no longer
|
// Stop the audio unit so the render callback no longer
|
||||||
// fires. The coreaudio-rs wrapper's own Drop calls
|
// fires. The coreaudio-rs wrapper's own Drop calls
|
||||||
// AudioComponentInstanceDispose afterwards.
|
// AudioComponentInstanceDispose afterwards.
|
||||||
if let Err(e) = self.unit.stop() {
|
if let Some(unit) = self.unit.as_mut() {
|
||||||
warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed");
|
if let Err(e) = unit.stop() {
|
||||||
} else {
|
warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed");
|
||||||
info!(target: "chanora_audio", "ios audio unit stopped");
|
} else {
|
||||||
|
info!(target: "chanora_audio", "ios audio unit stopped");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
|
|||||||
route,
|
route,
|
||||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||||
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
||||||
vad_backend: VadBackend::SileroOnnx,
|
vad_backend: VadBackend::WebrtcVad,
|
||||||
aec: EffectOwner::Platform,
|
aec: EffectOwner::Platform,
|
||||||
// VPIO owns NS and AGC on the shipping default path (IOSP_002/003).
|
// VPIO owns NS and AGC on the shipping default path (IOSP_002/003).
|
||||||
ns: EffectOwner::Platform,
|
ns: EffectOwner::Platform,
|
||||||
@@ -43,7 +43,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
|
|||||||
route,
|
route,
|
||||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||||
processing_backend: AudioBackend::Noop,
|
processing_backend: AudioBackend::Noop,
|
||||||
vad_backend: VadBackend::SileroOnnx,
|
vad_backend: VadBackend::WebrtcVad,
|
||||||
// No AEC needed for wired headset (no acoustic echo path).
|
// No AEC needed for wired headset (no acoustic echo path).
|
||||||
aec: EffectOwner::Off,
|
aec: EffectOwner::Off,
|
||||||
// Conservative NS/AGC: optional, not forced.
|
// Conservative NS/AGC: optional, not forced.
|
||||||
@@ -57,7 +57,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
|
|||||||
route,
|
route,
|
||||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||||
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
||||||
vad_backend: VadBackend::SileroOnnx,
|
vad_backend: VadBackend::WebrtcVad,
|
||||||
// BT HFP manages its own AEC in the headset firmware.
|
// BT HFP manages its own AEC in the headset firmware.
|
||||||
aec: EffectOwner::Off,
|
aec: EffectOwner::Off,
|
||||||
ns: EffectOwner::Conservative,
|
ns: EffectOwner::Conservative,
|
||||||
@@ -87,7 +87,7 @@ pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
|
|||||||
route,
|
route,
|
||||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||||
processing_backend: AudioBackend::Noop,
|
processing_backend: AudioBackend::Noop,
|
||||||
vad_backend: VadBackend::SileroOnnx,
|
vad_backend: VadBackend::WebrtcVad,
|
||||||
// Safe fallback: AEC off until route is classified.
|
// Safe fallback: AEC off until route is classified.
|
||||||
aec: EffectOwner::Off,
|
aec: EffectOwner::Off,
|
||||||
ns: EffectOwner::Off,
|
ns: EffectOwner::Off,
|
||||||
@@ -146,6 +146,7 @@ mod tests {
|
|||||||
cfg.ios_mode,
|
cfg.ios_mode,
|
||||||
IosVoiceProcessingMode::PlatformVoiceProcessing
|
IosVoiceProcessingMode::PlatformVoiceProcessing
|
||||||
);
|
);
|
||||||
|
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -156,6 +157,7 @@ mod tests {
|
|||||||
AudioBackend::PlatformVoiceProcessing
|
AudioBackend::PlatformVoiceProcessing
|
||||||
);
|
);
|
||||||
assert_eq!(cfg.aec, EffectOwner::Platform);
|
assert_eq!(cfg.aec, EffectOwner::Platform);
|
||||||
|
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -163,12 +165,14 @@ mod tests {
|
|||||||
let cfg = ios_route_policy(AudioRoute::WiredHeadset);
|
let cfg = ios_route_policy(AudioRoute::WiredHeadset);
|
||||||
assert_eq!(cfg.aec, EffectOwner::Off);
|
assert_eq!(cfg.aec, EffectOwner::Off);
|
||||||
assert_eq!(cfg.processing_backend, AudioBackend::Noop);
|
assert_eq!(cfg.processing_backend, AudioBackend::Noop);
|
||||||
|
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bluetooth_hfp_disables_app_aec() {
|
fn bluetooth_hfp_disables_app_aec() {
|
||||||
let cfg = ios_route_policy(AudioRoute::BluetoothHfp);
|
let cfg = ios_route_policy(AudioRoute::BluetoothHfp);
|
||||||
assert_eq!(cfg.aec, EffectOwner::Off);
|
assert_eq!(cfg.aec, EffectOwner::Off);
|
||||||
|
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -183,6 +187,7 @@ mod tests {
|
|||||||
fn unknown_route_safe_fallback_no_aec() {
|
fn unknown_route_safe_fallback_no_aec() {
|
||||||
let cfg = ios_route_policy(AudioRoute::Unknown);
|
let cfg = ios_route_policy(AudioRoute::Unknown);
|
||||||
assert_eq!(cfg.aec, EffectOwner::Off);
|
assert_eq!(cfg.aec, EffectOwner::Off);
|
||||||
|
assert_eq!(cfg.vad_backend, VadBackend::WebrtcVad);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
//! Voice activity detection backends and helpers.
|
//! Voice activity detection backends and helpers.
|
||||||
//!
|
//!
|
||||||
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into
|
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into
|
||||||
//! this module. The production path prefers a model-backed detector when
|
//! this module and uses the realtime-safe WebRTC fallback. Other
|
||||||
//! available, and otherwise uses the realtime-safe fallback below so
|
//! platforms may use a model-backed detector when available so
|
||||||
//! VoiceActivity mode never collapses back to Continuous transmit.
|
//! VoiceActivity mode never collapses back to Continuous transmit.
|
||||||
|
|
||||||
pub mod resampler;
|
pub mod resampler;
|
||||||
|
#[cfg(not(target_os = "ios"))]
|
||||||
pub mod silero_onnx;
|
pub mod silero_onnx;
|
||||||
|
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
@@ -15,6 +16,7 @@ use crate::frame::{f32_to_i16, i16_to_f32};
|
|||||||
use crate::AudioError;
|
use crate::AudioError;
|
||||||
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "ios"))]
|
||||||
pub use silero_onnx::SileroOnnxVad;
|
pub use silero_onnx::SileroOnnxVad;
|
||||||
|
|
||||||
/// Voice activity detector output for one 10 ms frame.
|
/// Voice activity detector output for one 10 ms frame.
|
||||||
@@ -111,11 +113,11 @@ fn silero_model_path_override() -> &'static RwLock<Option<String>> {
|
|||||||
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Configure the preferred Silero ONNX model path.
|
/// Configure the preferred Silero ONNX model path on supported platforms.
|
||||||
///
|
///
|
||||||
/// The path is validated eagerly. A successful call increments the
|
/// The path is validated eagerly. A successful call increments the
|
||||||
/// model epoch so running audio backends can reload the model without
|
/// model epoch so running non-iOS audio backends can reload the model
|
||||||
/// an app restart.
|
/// without an app restart.
|
||||||
pub fn set_silero_model_path(path: &str) -> Result<(), AudioError> {
|
pub fn set_silero_model_path(path: &str) -> Result<(), AudioError> {
|
||||||
let path = path.trim();
|
let path = path.trim();
|
||||||
if path.is_empty() {
|
if path.is_empty() {
|
||||||
@@ -141,13 +143,13 @@ pub fn silero_model_epoch() -> u64 {
|
|||||||
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
|
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return the expected path of the Silero VAD v6 ONNX model.
|
/// Return the expected path of the Silero VAD v6 ONNX model on
|
||||||
|
/// supported platforms.
|
||||||
/// The model is shipped as a Flutter asset and copied to the app's
|
/// The model is shipped as a Flutter asset and copied to the app's
|
||||||
/// data directory by the Dart-side asset loader.
|
/// data directory by the Dart-side asset loader.
|
||||||
///
|
///
|
||||||
/// On iOS/Android the model lives in the app's Documents/files
|
/// Android and macOS may use app data/Documents locations. Desktop
|
||||||
/// directory. On desktop, the caller should set the path explicitly
|
/// callers can set the path explicitly via `set_silero_model_path`.
|
||||||
/// via `set_silero_model_path`.
|
|
||||||
pub fn silero_model_bundle_path() -> String {
|
pub fn silero_model_bundle_path() -> String {
|
||||||
if let Ok(guard) = silero_model_path_override().read() {
|
if let Ok(guard) = silero_model_path_override().read() {
|
||||||
if let Some(path) = guard.as_ref() {
|
if let Some(path) = guard.as_ref() {
|
||||||
@@ -155,8 +157,9 @@ pub fn silero_model_bundle_path() -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// iOS: Documents directory (written by Flutter asset loader).
|
// macOS: Documents directory (written by Flutter asset loader).
|
||||||
// macOS: same Documents pattern.
|
// iOS keeps this fallback only for API compatibility; the ONNX
|
||||||
|
// detector is not compiled into iOS builds.
|
||||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||||
{
|
{
|
||||||
if let Ok(home) = std::env::var("HOME") {
|
if let Ok(home) = std::env::var("HOME") {
|
||||||
|
|||||||
Reference in New Issue
Block a user