edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize voice activity and audio routing

Browse Source
This commit is contained in:
Edison Jwa
2026-05-25 01:19:09 +09:00
parent eb9014cd81
commit 5515ff6643
34 changed files with 3054 additions and 1751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/android_voice_unit.rs
+169 -119
View File
@@ -145,16 +145,18 @@ struct AndroidCaptureState {
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
ten_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
input_sample_rate_hz: u32,
resample_pos: f64,
resample_last: i16,
resample_scratch: Vec<i16>,
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: usize,
fallback_warned_backend: Option<crate::VadBackend>,
@@ -171,29 +173,18 @@ impl AndroidCaptureState {
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
input_sample_rate_hz: u32,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("android")?;
// Android always uses software WebRTC APM for AEC/NS/AGC/HPF.
// The config's EffectOwner fields are resolved by open() AFTER
// hardware-effect binding; the processor is constructed here
// with all modules enabled regardless, so the resolved config
// (Platform vs WebrtcApm) only affects diagnostics, not behaviour.
// Seed the processor from the current shared config snapshot.
// `open()` may later resolve Platform-owned stages to WebRTC
// fallback (or keep them hardware-owned) once hardware-effect
// binding completes; that resolved config is pushed back into
// the live processor before the streams are started.
let webrtc_apm_config = audio_processing_config
.lock()
.map(|cfg| {
let mut c = crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg);
c.aec = true;
c.ns = true;
c.agc = true;
c
})
.unwrap_or(crate::processor::webrtc_apm::WebRtcApmConfig {
aec: true,
ns: true,
agc: true,
hpf: true,
..Default::default()
});
.map(|cfg| webrtc_apm_config_from_audio_config(&cfg))
.unwrap_or_default();
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
@@ -206,10 +197,8 @@ impl AndroidCaptureState {
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
@@ -218,6 +207,10 @@ impl AndroidCaptureState {
audio_processing_config,
audio_processing_stats,
render_reference,
input_sample_rate_hz: input_sample_rate_hz.max(1),
resample_pos: 0.0,
resample_last: 0,
resample_scratch: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
fallback_warned_backend: None,
@@ -227,6 +220,17 @@ impl AndroidCaptureState {
/// Consume i16 mono frames from Oboe. Accumulate to 10 ms chunks,
/// process each through WebRTC APM + VAD, then encode 20 ms frames.
fn ingest_i16(&mut self, samples: &[i16]) {
self.audio_processing_stats
.record_callback_frames(samples.len() as u64);
if self.input_sample_rate_hz != crate::frame::SAMPLE_RATE_HZ {
let resampled = self.resample_capture_to_48k(samples);
self.ingest_48k_i16(&resampled);
return;
}
self.ingest_48k_i16(samples);
}
fn ingest_48k_i16(&mut self, samples: &[i16]) {
let mut offset = 0;
while offset < samples.len() {
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
@@ -284,6 +288,41 @@ impl AndroidCaptureState {
}
}
fn resample_capture_to_48k(&mut self, samples: &[i16]) -> Vec<i16> {
if samples.is_empty() {
return Vec::new();
}
self.resample_scratch.clear();
let ratio = self.input_sample_rate_hz as f64 / crate::frame::SAMPLE_RATE_HZ as f64;
let mut pos = self.resample_pos;
while pos < samples.len() as f64 {
let i = pos.floor() as isize;
let frac = pos - i as f64;
let a = if i <= 0 {
self.resample_last as f64
} else {
samples[(i - 1) as usize] as f64
};
let b = if i < samples.len() as isize {
samples[i as usize] as f64
} else {
a
};
let value = (a + frac * (b - a))
.round()
.clamp(i16::MIN as f64, i16::MAX as f64) as i16;
self.resample_scratch.push(value);
pos += ratio;
}
self.resample_pos = pos - samples.len() as f64;
self.resample_last = *samples.last().unwrap_or(&self.resample_last);
self.resample_scratch.clone()
}
fn set_input_sample_rate_hz(&mut self, sample_rate_hz: u32) {
self.input_sample_rate_hz = sample_rate_hz.max(1);
}
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend != Some(failed_backend) {
self.fallback_warned_backend = Some(failed_backend);
@@ -317,6 +356,18 @@ impl AndroidCaptureState {
self.webrtc_apm_processor.process_render(&render_ref);
self.webrtc_apm_processor.process_capture(&mut frame);
let voice_activity_mode = self
.voice_activity_selector
.as_ref()
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_vad_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
let (vad_hangover, vad_backend) = self
.audio_processing_config
.try_lock()
@@ -325,30 +376,28 @@ impl AndroidCaptureState {
crate::voice_activity::VAD_HANGOVER_MS,
crate::VadBackend::WebrtcVad,
));
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
if voice_activity_mode {
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
}
// VAD backend switching (mirrors iOS Raw path).
// VAD backend switching only while VoiceActivity mode is active.
let silero_epoch = crate::vad::silero_model_epoch();
let ten_epoch = crate::vad::ten_model_epoch();
let silero_changed =
vad_backend == crate::VadBackend::SileroOnnx && silero_epoch != self.silero_model_epoch;
let ten_changed =
vad_backend == crate::VadBackend::TenVad && ten_epoch != self.ten_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed || ten_changed {
let silero_changed = voice_activity_mode
&& vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_changed) {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.ten_model_epoch = ten_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
let path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
self.ten_vad_worker = None;
if self.silero_vad_worker.is_none() {
warn!(
target: "chanora_audio",
@@ -356,48 +405,43 @@ impl AndroidCaptureState {
);
}
}
crate::VadBackend::TenVad => {
let path = crate::vad::ten_model_bundle_path();
self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path);
self.silero_vad_worker = None;
if self.ten_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"android: TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD"
);
}
}
_ => {
self.silero_vad_worker = None;
self.ten_vad_worker = None;
}
}
self.vad_state.reset();
}
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
// Warm-up: worker dispatched but hasn't finished yet.
// Default to no-speech until first result arrives (~100ms).
crate::vad::VadOutput {
probability: 0.0,
speech: false,
let (vad_probability, active) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
@@ -405,53 +449,28 @@ impl AndroidCaptureState {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
(vad.probability, self.vad_state.update(vad.speech))
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
(0.0, false)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let active = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(sel) = &self.voice_activity_selector {
sel.set_voice_activity_open(active && !output_muted);
sel.set_voice_activity_open(voice_activity_mode && active);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
active && !output_muted,
vad_probability,
voice_activity_mode && active,
self.transmit_active.load(Ordering::Relaxed),
);
self.audio_processing_stats
.record_capture_frame(frame.iter().all(|sample| sample.abs() < 1.0e-6));
if !self.transmit_active.load(Ordering::Relaxed) || output_muted {
if !self.transmit_active.load(Ordering::Relaxed) {
return;
}
@@ -511,6 +530,9 @@ struct OutputCallback {
event_tx: BackendEventTx,
scratch: Arc<Mutex<Vec<f32>>>,
render_reference: Arc<RenderReferenceBuffer>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
pending_render_ref: [f32; crate::frame::FRAME_10MS_SAMPLES],
pending_render_ref_len: usize,
}
impl AudioOutputCallback for OutputCallback {
@@ -552,19 +574,20 @@ impl AudioOutputCallback for OutputCallback {
gain,
muted,
);
self.audio_processing_stats
.update_render(crate::frame::dbfs(&scratch[..needed]), frames.len() as u32);
// Write the first 10 ms of render audio into the reference
// buffer for the capture-side AEC.
let mono_n = needed / 2;
let render_n = mono_n.min(crate::frame::FRAME_10MS_SAMPLES);
let mut ref_frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (i, chunk) in scratch[..render_n * 2].chunks_exact(2).enumerate() {
if i >= render_n {
break;
// Accumulate the full render callback into 10 ms mono chunks so
// AEC sees consistent reference timing even when output callbacks
// are shorter or longer than 10 ms.
for chunk in scratch[..needed].chunks_exact(2) {
self.pending_render_ref[self.pending_render_ref_len] = (chunk[0] + chunk[1]) * 0.5;
self.pending_render_ref_len += 1;
if self.pending_render_ref_len == crate::frame::FRAME_10MS_SAMPLES {
self.render_reference.write(&self.pending_render_ref);
self.pending_render_ref_len = 0;
}
ref_frame[i] = (chunk[0] + chunk[1]) * 0.5;
}
self.render_reference.write(&ref_frame);
}));
DataCallbackResult::Continue
}
@@ -640,6 +663,7 @@ impl AndroidVoiceUnit {
// Clone the APM config Arc before params is partially moved
// into the capture state constructor below.
let apm_config_clone = params.audio_processing_config.clone();
let audio_processing_stats = params.audio_processing_stats.clone();
let capture_state = Arc::new(Mutex::new(
AndroidCaptureState::new(
@@ -650,8 +674,9 @@ impl AndroidVoiceUnit {
params.mic_gain,
params.voice_activity_selector,
params.audio_processing_config,
params.audio_processing_stats,
audio_processing_stats.clone(),
render_ref_for_capture,
cfg.sample_rate,
)
.map_err(|e| BackendError::OpenFailed(format!("capture state init: {e}")))?,
));
@@ -768,6 +793,9 @@ impl AndroidVoiceUnit {
event_tx: event_tx.clone(),
scratch: scratch.clone(),
render_reference: render_ref_for_output,
audio_processing_stats: audio_processing_stats.clone(),
pending_render_ref: [0.0_f32; crate::frame::FRAME_10MS_SAMPLES],
pending_render_ref_len: 0,
};
let output_builder = output_builder.set_callback(output_cb);
@@ -785,6 +813,7 @@ impl AndroidVoiceUnit {
params.handler.clone(),
params.output_gain.clone(),
params.output_muted.clone(),
audio_processing_stats.clone(),
scratch.clone(),
render_ref_buf,
)?
@@ -869,6 +898,14 @@ impl AndroidVoiceUnit {
"android: audio processing config resolved (hardware effects: aec={hw_aec} ns={hw_ns} agc={hw_agc})"
);
}
if let Ok(mut capture) = capture_state.lock() {
capture.set_input_sample_rate_hz(input_sample_rate.max(1) as u32);
let resolved_cfg = apm_config_clone
.lock()
.map(|cfg| webrtc_apm_config_from_audio_config(&cfg))
.unwrap_or_default();
capture.webrtc_apm_processor.apply_config(resolved_cfg);
}
// --- SDD-112 item 10 / SDD-113 item 7 / SDD-116 item 3 ---
// Publish the diagnostics snapshot. Per-effect engagement is
@@ -910,6 +947,9 @@ impl AndroidVoiceUnit {
latency_tier: latency_tier_for(input_perf),
};
publish_android_audio_diagnostics(diagnostics);
params
.audio_processing_stats
.set_actual_sample_rate_hz(input_sample_rate.max(0) as u32);
Ok(Self {
input: input_stream,
@@ -1005,6 +1045,7 @@ impl AndroidVoiceUnit {
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
scratch: Arc<Mutex<Vec<f32>>>,
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<AudioStreamAsync<OboeOutput, OutputCallback>, BackendError> {
@@ -1015,6 +1056,9 @@ impl AndroidVoiceUnit {
event_tx: event_tx.clone(),
scratch,
render_reference,
audio_processing_stats,
pending_render_ref: [0.0_f32; crate::frame::FRAME_10MS_SAMPLES],
pending_render_ref_len: 0,
};
let builder = AudioStreamBuilder::default()
.set_direction::<OboeOutput>()
@@ -1038,6 +1082,12 @@ impl AndroidVoiceUnit {
}
}
fn webrtc_apm_config_from_audio_config(
config: &crate::AudioProcessingConfig,
) -> crate::processor::webrtc_apm::WebRtcApmConfig {
crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(config)
}
impl MobileVoiceAudioBackend for AndroidVoiceUnit {
fn start(&mut self) -> Result<(), BackendError> {
if let Some(s) = self.input.as_mut() {