edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize voice activity and audio routing

Browse Source
This commit is contained in:
Edison Jwa
2026-05-25 01:19:09 +09:00
parent eb9014cd81
commit 5515ff6643
34 changed files with 3054 additions and 1751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/ios_voice_unit.rs
+65 -90
View File
@@ -146,13 +146,10 @@ struct IosCaptureState {
/// Background Silero worker — enqueues frames off the realtime
/// callback and publishes the latest probability atomically.
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad: Option<crate::vad::TenOnnxVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
/// Last observed configured TEN model epoch.
ten_model_epoch: u64,
fallback_warned_backend: Option<crate::VadBackend>,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
@@ -191,10 +188,8 @@ impl IosCaptureState {
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config: params.audio_processing_config.clone(),
@@ -361,13 +356,23 @@ impl IosCaptureState {
crate::AudioBackend::PlatformVoiceProcessing,
));
// Switch VAD backend when the config changes.
let voice_activity_mode = self
.voice_activity_selector
.as_ref()
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_vad_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
// Switch VAD backend only while VoiceActivity mode is active.
let silero_model_epoch = crate::vad::silero_model_epoch();
let ten_model_epoch = crate::vad::ten_model_epoch();
let silero_model_changed = vad_backend == crate::VadBackend::SileroOnnx
let silero_model_changed = voice_activity_mode
&& vad_backend == crate::VadBackend::SileroOnnx
&& silero_model_epoch != self.silero_model_epoch;
let ten_model_changed =
vad_backend == crate::VadBackend::TenVad && ten_model_epoch != self.ten_model_epoch;
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
if debug_wav_dump_enabled {
@@ -382,11 +387,11 @@ impl IosCaptureState {
}
}
if vad_backend != self.current_vad_backend || silero_model_changed || ten_model_changed {
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_model_changed)
{
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
self.silero_model_epoch = silero_model_epoch;
self.ten_model_epoch = ten_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
// Attempt to load Silero model from the well-known
@@ -407,23 +412,8 @@ impl IosCaptureState {
self.audio_processing_stats
.set_vad_fallback_active(self.silero_vad_worker.is_none());
}
crate::VadBackend::TenVad => {
self.silero_vad_worker = None;
let model_path = crate::vad::ten_model_bundle_path();
self.ten_vad = crate::vad::TenOnnxVadWorker::try_new(&model_path);
if self.ten_vad.is_none() {
warn!(
target: "chanora_audio",
"TEN VAD ONNX model not available at {model_path}; falling back to WebRTC VAD"
);
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
}
self.audio_processing_stats
.set_vad_fallback_active(self.ten_vad.is_none());
}
_ => {
self.silero_vad_worker = None;
self.ten_vad = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
}
@@ -436,13 +426,16 @@ impl IosCaptureState {
self.vad_state.reset();
}
// Keep the VAD state machine aligned with the active config.
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
if voice_activity_mode {
// Keep the VAD state machine aligned with the active config only
// while VoiceActivity mode owns the transmit gate.
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
}
// VPIO owns AEC. Keep the legacy non-AEC conditioning path here until
// the raw WebRTC APM path is explicitly selected.
@@ -461,29 +454,37 @@ impl IosCaptureState {
self.sonora_processor.process_capture(&mut frame);
}
// VAD: use Silero if loaded, otherwise WebRTC fallback.
// Disabled backend → always open (Continuous-like for VAD mode).
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
// VAD: only evaluate while VoiceActivity mode is active.
let (vad_probability, gate_open) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
@@ -491,49 +492,23 @@ impl IosCaptureState {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
(vad.probability, self.vad_state.update(vad.speech))
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
(0.0, false)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let gate_open = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(selector) = &self.voice_activity_selector {
selector.set_voice_activity_open(gate_open && !output_muted);
selector.set_voice_activity_open(voice_activity_mode && gate_open && !output_muted);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
gate_open && !output_muted,
vad_probability,
voice_activity_mode && gate_open && !output_muted,
transmit_active,
);