feat: stabilize voice activity and audio routing

This commit is contained in:
Edison Jwa
2026-05-25 01:19:09 +09:00
parent eb9014cd81
commit 5515ff6643
34 changed files with 3054 additions and 1751 deletions
+1 -1
View File
@@ -17,7 +17,7 @@ sonora = "0.1"
webrtc-vad = "0.4"
# ndarray is required by ort's tensor construction API and by
# Silero / TEN VAD ONNX inference across all platforms.
# Silero VAD ONNX inference across all platforms.
ndarray = "0.17"
# Opus encoder. tsclientlib already pulls this; we depend explicitly so
+169 -119
View File
@@ -145,16 +145,18 @@ struct AndroidCaptureState {
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
ten_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
input_sample_rate_hz: u32,
resample_pos: f64,
resample_last: i16,
resample_scratch: Vec<i16>,
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: usize,
fallback_warned_backend: Option<crate::VadBackend>,
@@ -171,29 +173,18 @@ impl AndroidCaptureState {
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
input_sample_rate_hz: u32,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("android")?;
// Android always uses software WebRTC APM for AEC/NS/AGC/HPF.
// The config's EffectOwner fields are resolved by open() AFTER
// hardware-effect binding; the processor is constructed here
// with all modules enabled regardless, so the resolved config
// (Platform vs WebrtcApm) only affects diagnostics, not behaviour.
// Seed the processor from the current shared config snapshot.
// `open()` may later resolve Platform-owned stages to WebRTC
// fallback (or keep them hardware-owned) once hardware-effect
// binding completes; that resolved config is pushed back into
// the live processor before the streams are started.
let webrtc_apm_config = audio_processing_config
.lock()
.map(|cfg| {
let mut c = crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg);
c.aec = true;
c.ns = true;
c.agc = true;
c
})
.unwrap_or(crate::processor::webrtc_apm::WebRtcApmConfig {
aec: true,
ns: true,
agc: true,
hpf: true,
..Default::default()
});
.map(|cfg| webrtc_apm_config_from_audio_config(&cfg))
.unwrap_or_default();
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
@@ -206,10 +197,8 @@ impl AndroidCaptureState {
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
@@ -218,6 +207,10 @@ impl AndroidCaptureState {
audio_processing_config,
audio_processing_stats,
render_reference,
input_sample_rate_hz: input_sample_rate_hz.max(1),
resample_pos: 0.0,
resample_last: 0,
resample_scratch: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
fallback_warned_backend: None,
@@ -227,6 +220,17 @@ impl AndroidCaptureState {
/// Consume i16 mono frames from Oboe. Accumulate to 10 ms chunks,
/// process each through WebRTC APM + VAD, then encode 20 ms frames.
fn ingest_i16(&mut self, samples: &[i16]) {
self.audio_processing_stats
.record_callback_frames(samples.len() as u64);
if self.input_sample_rate_hz != crate::frame::SAMPLE_RATE_HZ {
let resampled = self.resample_capture_to_48k(samples);
self.ingest_48k_i16(&resampled);
return;
}
self.ingest_48k_i16(samples);
}
fn ingest_48k_i16(&mut self, samples: &[i16]) {
let mut offset = 0;
while offset < samples.len() {
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
@@ -284,6 +288,41 @@ impl AndroidCaptureState {
}
}
fn resample_capture_to_48k(&mut self, samples: &[i16]) -> Vec<i16> {
if samples.is_empty() {
return Vec::new();
}
self.resample_scratch.clear();
let ratio = self.input_sample_rate_hz as f64 / crate::frame::SAMPLE_RATE_HZ as f64;
let mut pos = self.resample_pos;
while pos < samples.len() as f64 {
let i = pos.floor() as isize;
let frac = pos - i as f64;
let a = if i <= 0 {
self.resample_last as f64
} else {
samples[(i - 1) as usize] as f64
};
let b = if i < samples.len() as isize {
samples[i as usize] as f64
} else {
a
};
let value = (a + frac * (b - a))
.round()
.clamp(i16::MIN as f64, i16::MAX as f64) as i16;
self.resample_scratch.push(value);
pos += ratio;
}
self.resample_pos = pos - samples.len() as f64;
self.resample_last = *samples.last().unwrap_or(&self.resample_last);
self.resample_scratch.clone()
}
fn set_input_sample_rate_hz(&mut self, sample_rate_hz: u32) {
self.input_sample_rate_hz = sample_rate_hz.max(1);
}
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend != Some(failed_backend) {
self.fallback_warned_backend = Some(failed_backend);
@@ -317,6 +356,18 @@ impl AndroidCaptureState {
self.webrtc_apm_processor.process_render(&render_ref);
self.webrtc_apm_processor.process_capture(&mut frame);
let voice_activity_mode = self
.voice_activity_selector
.as_ref()
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_vad_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
let (vad_hangover, vad_backend) = self
.audio_processing_config
.try_lock()
@@ -325,30 +376,28 @@ impl AndroidCaptureState {
crate::voice_activity::VAD_HANGOVER_MS,
crate::VadBackend::WebrtcVad,
));
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
if voice_activity_mode {
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
}
// VAD backend switching (mirrors iOS Raw path).
// VAD backend switching only while VoiceActivity mode is active.
let silero_epoch = crate::vad::silero_model_epoch();
let ten_epoch = crate::vad::ten_model_epoch();
let silero_changed =
vad_backend == crate::VadBackend::SileroOnnx && silero_epoch != self.silero_model_epoch;
let ten_changed =
vad_backend == crate::VadBackend::TenVad && ten_epoch != self.ten_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed || ten_changed {
let silero_changed = voice_activity_mode
&& vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_changed) {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.ten_model_epoch = ten_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
let path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
self.ten_vad_worker = None;
if self.silero_vad_worker.is_none() {
warn!(
target: "chanora_audio",
@@ -356,48 +405,43 @@ impl AndroidCaptureState {
);
}
}
crate::VadBackend::TenVad => {
let path = crate::vad::ten_model_bundle_path();
self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path);
self.silero_vad_worker = None;
if self.ten_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"android: TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD"
);
}
}
_ => {
self.silero_vad_worker = None;
self.ten_vad_worker = None;
}
}
self.vad_state.reset();
}
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
// Warm-up: worker dispatched but hasn't finished yet.
// Default to no-speech until first result arrives (~100ms).
crate::vad::VadOutput {
probability: 0.0,
speech: false,
let (vad_probability, active) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
@@ -405,53 +449,28 @@ impl AndroidCaptureState {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
(vad.probability, self.vad_state.update(vad.speech))
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
(0.0, false)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let active = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(sel) = &self.voice_activity_selector {
sel.set_voice_activity_open(active && !output_muted);
sel.set_voice_activity_open(voice_activity_mode && active);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
active && !output_muted,
vad_probability,
voice_activity_mode && active,
self.transmit_active.load(Ordering::Relaxed),
);
self.audio_processing_stats
.record_capture_frame(frame.iter().all(|sample| sample.abs() < 1.0e-6));
if !self.transmit_active.load(Ordering::Relaxed) || output_muted {
if !self.transmit_active.load(Ordering::Relaxed) {
return;
}
@@ -511,6 +530,9 @@ struct OutputCallback {
event_tx: BackendEventTx,
scratch: Arc<Mutex<Vec<f32>>>,
render_reference: Arc<RenderReferenceBuffer>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
pending_render_ref: [f32; crate::frame::FRAME_10MS_SAMPLES],
pending_render_ref_len: usize,
}
impl AudioOutputCallback for OutputCallback {
@@ -552,19 +574,20 @@ impl AudioOutputCallback for OutputCallback {
gain,
muted,
);
self.audio_processing_stats
.update_render(crate::frame::dbfs(&scratch[..needed]), frames.len() as u32);
// Write the first 10 ms of render audio into the reference
// buffer for the capture-side AEC.
let mono_n = needed / 2;
let render_n = mono_n.min(crate::frame::FRAME_10MS_SAMPLES);
let mut ref_frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (i, chunk) in scratch[..render_n * 2].chunks_exact(2).enumerate() {
if i >= render_n {
break;
// Accumulate the full render callback into 10 ms mono chunks so
// AEC sees consistent reference timing even when output callbacks
// are shorter or longer than 10 ms.
for chunk in scratch[..needed].chunks_exact(2) {
self.pending_render_ref[self.pending_render_ref_len] = (chunk[0] + chunk[1]) * 0.5;
self.pending_render_ref_len += 1;
if self.pending_render_ref_len == crate::frame::FRAME_10MS_SAMPLES {
self.render_reference.write(&self.pending_render_ref);
self.pending_render_ref_len = 0;
}
ref_frame[i] = (chunk[0] + chunk[1]) * 0.5;
}
self.render_reference.write(&ref_frame);
}));
DataCallbackResult::Continue
}
@@ -640,6 +663,7 @@ impl AndroidVoiceUnit {
// Clone the APM config Arc before params is partially moved
// into the capture state constructor below.
let apm_config_clone = params.audio_processing_config.clone();
let audio_processing_stats = params.audio_processing_stats.clone();
let capture_state = Arc::new(Mutex::new(
AndroidCaptureState::new(
@@ -650,8 +674,9 @@ impl AndroidVoiceUnit {
params.mic_gain,
params.voice_activity_selector,
params.audio_processing_config,
params.audio_processing_stats,
audio_processing_stats.clone(),
render_ref_for_capture,
cfg.sample_rate,
)
.map_err(|e| BackendError::OpenFailed(format!("capture state init: {e}")))?,
));
@@ -768,6 +793,9 @@ impl AndroidVoiceUnit {
event_tx: event_tx.clone(),
scratch: scratch.clone(),
render_reference: render_ref_for_output,
audio_processing_stats: audio_processing_stats.clone(),
pending_render_ref: [0.0_f32; crate::frame::FRAME_10MS_SAMPLES],
pending_render_ref_len: 0,
};
let output_builder = output_builder.set_callback(output_cb);
@@ -785,6 +813,7 @@ impl AndroidVoiceUnit {
params.handler.clone(),
params.output_gain.clone(),
params.output_muted.clone(),
audio_processing_stats.clone(),
scratch.clone(),
render_ref_buf,
)?
@@ -869,6 +898,14 @@ impl AndroidVoiceUnit {
"android: audio processing config resolved (hardware effects: aec={hw_aec} ns={hw_ns} agc={hw_agc})"
);
}
if let Ok(mut capture) = capture_state.lock() {
capture.set_input_sample_rate_hz(input_sample_rate.max(1) as u32);
let resolved_cfg = apm_config_clone
.lock()
.map(|cfg| webrtc_apm_config_from_audio_config(&cfg))
.unwrap_or_default();
capture.webrtc_apm_processor.apply_config(resolved_cfg);
}
// --- SDD-112 item 10 / SDD-113 item 7 / SDD-116 item 3 ---
// Publish the diagnostics snapshot. Per-effect engagement is
@@ -910,6 +947,9 @@ impl AndroidVoiceUnit {
latency_tier: latency_tier_for(input_perf),
};
publish_android_audio_diagnostics(diagnostics);
params
.audio_processing_stats
.set_actual_sample_rate_hz(input_sample_rate.max(0) as u32);
Ok(Self {
input: input_stream,
@@ -1005,6 +1045,7 @@ impl AndroidVoiceUnit {
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
scratch: Arc<Mutex<Vec<f32>>>,
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<AudioStreamAsync<OboeOutput, OutputCallback>, BackendError> {
@@ -1015,6 +1056,9 @@ impl AndroidVoiceUnit {
event_tx: event_tx.clone(),
scratch,
render_reference,
audio_processing_stats,
pending_render_ref: [0.0_f32; crate::frame::FRAME_10MS_SAMPLES],
pending_render_ref_len: 0,
};
let builder = AudioStreamBuilder::default()
.set_direction::<OboeOutput>()
@@ -1038,6 +1082,12 @@ impl AndroidVoiceUnit {
}
}
fn webrtc_apm_config_from_audio_config(
config: &crate::AudioProcessingConfig,
) -> crate::processor::webrtc_apm::WebRtcApmConfig {
crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(config)
}
impl MobileVoiceAudioBackend for AndroidVoiceUnit {
fn start(&mut self) -> Result<(), BackendError> {
if let Some(s) = self.input.as_mut() {
+59 -11
View File
@@ -92,9 +92,6 @@ impl AudioBackend {
pub enum VadBackend {
/// Silero ONNX VAD. P1 schema default when model/runtime exist.
SileroOnnx,
/// TEN VAD backend. Native TEN runtime is optional; unavailable
/// builds fall back to the realtime-safe WebRTC detector.
TenVad,
/// WebRTC-style fallback VAD.
WebrtcVad,
/// Debug-only energy VAD.
@@ -108,7 +105,6 @@ impl VadBackend {
pub fn as_str(self) -> &'static str {
match self {
Self::SileroOnnx => "silero_vad_onnx",
Self::TenVad => "ten_vad",
Self::WebrtcVad => "webrtc_vad",
Self::EnergyDebug => "energy_debug",
Self::Disabled => "disabled",
@@ -168,7 +164,7 @@ impl Default for AudioProcessingConfig {
route: AudioRoute::Speaker,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::TenVad,
vad_backend: VadBackend::SileroOnnx,
aec: EffectOwner::Platform,
// iOS VPIO owns NS/AGC on the default shipping path. Software
// effects are opt-in through the experimental raw route only.
@@ -246,7 +242,7 @@ mod tests {
assert_eq!(config.aec, EffectOwner::Platform);
assert_eq!(config.ns, EffectOwner::Platform);
assert_eq!(config.agc, EffectOwner::Platform);
assert_eq!(config.vad_backend, VadBackend::TenVad);
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
}
#[test]
@@ -259,11 +255,6 @@ mod tests {
assert!(config.validate_for_ios().is_err());
}
#[test]
fn ten_vad_has_stable_debug_string() {
assert_eq!(VadBackend::TenVad.as_str(), "ten_vad");
}
#[test]
fn raw_processing_allows_full_webrtc_apm_chain() {
let config = AudioProcessingConfig {
@@ -342,6 +333,16 @@ pub struct AudioProcessingStats {
pub callback_xruns: u64,
/// Clipped sample count.
pub clipped_samples: u64,
/// Number of effectively silent processed capture frames.
pub zero_frames: u64,
/// Number of processed capture frames.
pub capture_frames: u64,
/// Number of input callbacks carrying 10 ms of audio.
pub callbacks_10ms: u64,
/// Number of input callbacks carrying 20 ms of audio.
pub callbacks_20ms: u64,
/// Number of input callbacks carrying any other size.
pub callbacks_other: u64,
/// Sonora enabled.
pub sonora_enabled: bool,
/// Platform voice processing enabled.
@@ -361,6 +362,11 @@ pub struct SharedAudioProcessingStats {
output_underruns: AtomicU64,
callback_xruns: AtomicU64,
clipped_samples: AtomicU64,
zero_frames: AtomicU64,
capture_frames: AtomicU64,
callbacks_10ms: AtomicU64,
callbacks_20ms: AtomicU64,
callbacks_other: AtomicU64,
actual_sample_rate_hz: AtomicU32,
actual_io_buffer_frames: AtomicU32,
}
@@ -379,6 +385,11 @@ impl Default for SharedAudioProcessingStats {
output_underruns: AtomicU64::new(0),
callback_xruns: AtomicU64::new(0),
clipped_samples: AtomicU64::new(0),
zero_frames: AtomicU64::new(0),
capture_frames: AtomicU64::new(0),
callbacks_10ms: AtomicU64::new(0),
callbacks_20ms: AtomicU64::new(0),
callbacks_other: AtomicU64::new(0),
actual_sample_rate_hz: AtomicU32::new(crate::frame::SAMPLE_RATE_HZ),
actual_io_buffer_frames: AtomicU32::new(crate::frame::FRAME_20MS_SAMPLES as u32),
}
@@ -412,6 +423,12 @@ impl SharedAudioProcessingStats {
.store(io_buffer_frames, Ordering::Relaxed);
}
/// Record the actual device sample rate.
pub fn set_actual_sample_rate_hz(&self, sample_rate_hz: u32) {
self.actual_sample_rate_hz
.store(sample_rate_hz, Ordering::Relaxed);
}
/// Increment output underrun count.
pub fn increment_output_underrun(&self) {
self.output_underruns.fetch_add(1, Ordering::Relaxed);
@@ -427,6 +444,32 @@ impl SharedAudioProcessingStats {
self.clipped_samples.fetch_add(count, Ordering::Relaxed);
}
/// Record one processed capture frame and whether it was effectively silent.
pub fn record_capture_frame(&self, zero_frame: bool) {
self.capture_frames.fetch_add(1, Ordering::Relaxed);
if zero_frame {
self.zero_frames.fetch_add(1, Ordering::Relaxed);
}
}
/// Bucket callback delivery sizes to diagnose timing jitter and packetization.
pub fn record_callback_frames(&self, frames: u64) {
let sample_rate_hz = self.actual_sample_rate_hz.load(Ordering::Relaxed).max(1);
let frames_10ms = (sample_rate_hz / 100) as u64;
let frames_20ms = (sample_rate_hz / 50) as u64;
match frames {
value if value == frames_10ms => {
self.callbacks_10ms.fetch_add(1, Ordering::Relaxed);
}
value if value == frames_20ms => {
self.callbacks_20ms.fetch_add(1, Ordering::Relaxed);
}
_ => {
self.callbacks_other.fetch_add(1, Ordering::Relaxed);
}
}
}
/// Store whether the selected VAD backend is currently using a fallback.
pub fn set_vad_fallback_active(&self, active: bool) {
self.vad_fallback_active.store(active, Ordering::Relaxed);
@@ -452,6 +495,11 @@ impl SharedAudioProcessingStats {
output_underruns: self.output_underruns.load(Ordering::Relaxed),
callback_xruns: self.callback_xruns.load(Ordering::Relaxed),
clipped_samples: self.clipped_samples.load(Ordering::Relaxed),
zero_frames: self.zero_frames.load(Ordering::Relaxed),
capture_frames: self.capture_frames.load(Ordering::Relaxed),
callbacks_10ms: self.callbacks_10ms.load(Ordering::Relaxed),
callbacks_20ms: self.callbacks_20ms.load(Ordering::Relaxed),
callbacks_other: self.callbacks_other.load(Ordering::Relaxed),
sonora_enabled: config.processing_backend == AudioBackend::Sonora,
platform_voice_processing_enabled: config.processing_backend
== AudioBackend::PlatformVoiceProcessing,
+435 -101
View File
@@ -20,6 +20,18 @@ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
not(target_os = "android")
))]
use cpal::{SampleFormat, SizedSample};
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
use std::collections::hash_map::DefaultHasher;
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
use std::hash::{Hash, Hasher};
use tokio::sync::mpsc;
use tracing::{debug, info};
// `error!` and `warn!` are used only inside the cpal capture /
@@ -80,12 +92,77 @@ pub struct AudioDeviceList {
/// Info about a single audio device.
#[derive(Debug, Clone)]
pub struct AudioDeviceInfo {
/// Stable platform-reported device identifier.
pub id: String,
/// Human-readable device name from the OS.
pub name: String,
/// Additional device details useful for disambiguation.
pub details: String,
/// True if the OS reports this as the default device.
pub is_default: bool,
}
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
fn desktop_device_id(device: &cpal::Device) -> Option<String> {
device.id().ok().map(|id| id.to_string())
}
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
fn short_device_id(id: &str) -> String {
let mut hasher = DefaultHasher::new();
id.hash(&mut hasher);
format!("{:016x}", hasher.finish())
}
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
fn describe_device(device: &cpal::Device) -> Option<AudioDeviceInfo> {
let id = desktop_device_id(device)?;
let description = device.description().ok();
let name = description
.as_ref()
.map(|d| d.name().trim().to_owned())
.filter(|name| !name.is_empty())
.unwrap_or_else(|| format!("Device {}", &short_device_id(&id)[..8]));
let mut details = Vec::new();
if let Some(description) = description.as_ref() {
if let Some(manufacturer) = description.manufacturer() {
details.push(manufacturer.to_owned());
}
if let Some(driver) = description.driver() {
details.push(driver.to_owned());
}
let device_type = description.device_type();
if device_type != cpal::device_description::DeviceType::Unknown {
details.push(format!("{device_type}"));
}
let interface_type = description.interface_type();
if interface_type != cpal::device_description::InterfaceType::Unknown {
details.push(format!("{interface_type}"));
}
}
details.push(format!("id={}", short_device_id(&id)));
Some(AudioDeviceInfo {
id,
name,
details: details.join(" · "),
is_default: false,
})
}
/// Enumerate available audio input and output devices.
/// On mobile platforms (iOS, Android) returns an empty list because
/// device selection is managed by the OS audio session.
@@ -100,38 +177,30 @@ pub fn list_audio_devices() -> AudioDeviceList {
input_devices: Vec::new(),
output_devices: Vec::new(),
};
let Ok(host) = cpal::default_host() else {
return list;
};
let default_in = host.default_input_device();
let default_out = host.default_output_device();
let host = cpal::default_host();
let default_in = host
.default_input_device()
.and_then(|device| desktop_device_id(&device));
let default_out = host
.default_output_device()
.and_then(|device| desktop_device_id(&device));
if let Ok(devices) = host.input_devices() {
for d in devices {
let name = d
.description()
.map(|n| n.name().to_owned())
.unwrap_or_default();
if !name.is_empty() {
let is_default = default_in
if let Some(mut device) = describe_device(&d) {
device.is_default = default_in
.as_ref()
.is_some_and(|di| di.description().is_ok_and(|dn| dn.name() == name.as_str()));
list.input_devices
.push(AudioDeviceInfo { name, is_default });
.is_some_and(|default_id| default_id == &device.id);
list.input_devices.push(device);
}
}
}
if let Ok(devices) = host.output_devices() {
for d in devices {
let name = d
.description()
.map(|n| n.name().to_owned())
.unwrap_or_default();
if !name.is_empty() {
let is_default = default_out
if let Some(mut device) = describe_device(&d) {
device.is_default = default_out
.as_ref()
.is_some_and(|di| di.description().is_ok_and(|dn| dn.name() == name.as_str()));
list.output_devices
.push(AudioDeviceInfo { name, is_default });
.is_some_and(|default_id| default_id == &device.id);
list.output_devices.push(device);
}
}
}
@@ -172,12 +241,12 @@ pub struct AudioEngineConfig {
/// is rejected on Android because the P0 path intentionally has
/// no generic mobile-audio fallback.
pub mobile_voice_preset: bool,
/// Optional input device name override. When `None`, the system
/// default input device is used. Set to a device name from
/// Optional input device id override. When `None`, the system
/// default input device is used. Set to a device id from
/// [`list_audio_devices`] to pin a specific microphone.
pub input_device_name: Option<String>,
/// Optional output device name override.
pub output_device_name: Option<String>,
pub input_device_id: Option<String>,
/// Optional output device id override.
pub output_device_id: Option<String>,
/// Optional selector used by P1 VoiceActivity to publish VAD state.
#[doc(hidden)]
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
@@ -190,8 +259,8 @@ impl std::fmt::Debug for AudioEngineConfig {
.field("ptt_initial", &self.ptt_initial)
.field("effects", &self.effects)
.field("mobile_voice_preset", &self.mobile_voice_preset)
.field("input_device_name", &self.input_device_name)
.field("output_device_name", &self.output_device_name)
.field("input_device_id", &self.input_device_id)
.field("output_device_id", &self.output_device_id)
.field(
"voice_activity_selector",
&self.voice_activity_selector.as_ref().map(|_| "present"),
@@ -207,8 +276,8 @@ impl Default for AudioEngineConfig {
ptt_initial: false,
effects: crate::AudioEffects::default(),
mobile_voice_preset: true,
input_device_name: None,
output_device_name: None,
input_device_id: None,
output_device_id: None,
voice_activity_selector: None,
}
}
@@ -237,13 +306,13 @@ pub struct AudioEngine {
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
voice_out_tx: mpsc::Sender<OutPacket>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
mic_gain: f32,
#[cfg(any(target_os = "ios", target_os = "macos"))]
// Streams must be dropped to stop audio. Both are `!Send` because
// cpal's Stream isn't Send on some backends; we keep them in an
// Option wrapped by Mutex so stop() can move them out. On Linux
@@ -277,7 +346,11 @@ pub struct AudioEngine {
/// unbinds effects and stops the service in SDD-115 reverse
/// order.
#[cfg(target_os = "android")]
_android_voice_unit: Mutex<Option<crate::android_voice_unit::AndroidVoiceUnit>>,
_android_voice_unit: Arc<Mutex<Option<crate::android_voice_unit::AndroidVoiceUnit>>>,
/// Persist the Android stream request so route-change reopen uses the
/// same effect and latency policy as the original session start.
#[cfg(target_os = "android")]
android_voice_stream_config: Arc<Mutex<crate::mobile_voice_backend::AndroidVoiceStreamConfig>>,
/// SDD-108 §1/§2: refcount-composable audio-mode controller.
/// Snapshots `AudioManager.getMode()` on the 0 → 1 transition and
/// restores it on the 1 → 0 transition. Held in a `Mutex` so the
@@ -286,7 +359,7 @@ pub struct AudioEngine {
/// mode lifecycle is bound to the voice-session lifecycle
/// (SDD-108 §3).
#[cfg(target_os = "android")]
audio_mode_stack: Mutex<crate::mode_stack::ModeStack>,
audio_mode_stack: Arc<Mutex<crate::mode_stack::ModeStack>>,
// Hand the inbound-voice forwarder task a shutdown signal.
shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
/// True if the capture stream actually opened. If false (typical
@@ -385,6 +458,183 @@ fn open_ios_voice_backend(
}
impl AudioEngine {
#[cfg(target_os = "android")]
fn spawn_android_backend_event_task(
android_voice_unit: Arc<Mutex<Option<crate::android_voice_unit::AndroidVoiceUnit>>>,
android_voice_stream_config: Arc<
Mutex<crate::mobile_voice_backend::AndroidVoiceStreamConfig>,
>,
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_gate: crate::ptt::AudioTransmitGate,
frames_sent: Arc<AtomicU32>,
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
mic_gain: f32,
audio_mode_stack: Arc<Mutex<crate::mode_stack::ModeStack>>,
mut event_rx: crate::mobile_voice_backend::BackendEventRx,
) {
use crate::mobile_voice_backend::BackendEvent;
tokio::spawn(async move {
while let Some(event) = event_rx.recv().await {
match event {
BackendEvent::Disconnected => {
warn!(target: "chanora_audio", "android: backend disconnected; reopening voice unit");
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
crate::android_voice_unit::chanora_android_abandon_audio_focus();
crate::android_voice_unit::clear_global_event_sender();
if let Some(mut unit) = android_voice_unit.lock().unwrap().take() {
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
if let Err(e) = unit.close() {
warn!(
target: "chanora_audio",
error = %e,
"android: failed to close disconnected voice unit before reopen"
);
}
}
if crate::android_voice_unit::chanora_android_start_voice_service() {
info!(
target: "chanora_audio",
"android: voice foreground service restart dispatched after backend disconnect"
);
}
match android_get_audio_mode() {
Ok(current_mode) => {
if current_mode != ANDROID_MODE_IN_COMMUNICATION {
match android_set_audio_mode(ANDROID_MODE_IN_COMMUNICATION) {
Ok(()) => info!(
target: "chanora_audio",
prior_mode = current_mode,
"android: AudioManager mode re-engaged after backend disconnect"
),
Err(e) => warn!(
target: "chanora_audio",
error = %e,
prior_mode = current_mode,
"android: failed to re-engage MODE_IN_COMMUNICATION after backend disconnect"
),
}
}
}
Err(e) => warn!(
target: "chanora_audio",
error = %e,
"android: AudioManager.getMode failed during backend reopen"
),
}
let cfg_av = android_voice_stream_config.lock().unwrap().clone();
let params = crate::mobile_voice_backend::VoiceAudioParams {
voice_out_tx: voice_out_tx.clone(),
transmit_active: transmit_gate.flag_arc(),
frames_sent: frames_sent.clone(),
mic_gain,
handler: audio_handler.clone(),
output_gain: output_gain.clone(),
output_muted: output_muted.clone(),
voice_activity_selector: voice_activity_selector.clone(),
audio_processing_config: audio_processing_config.clone(),
audio_processing_stats: audio_processing_stats.clone(),
};
match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) {
Ok(mut reopened) => {
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
match reopened.start() {
Ok(()) => {
if let Some(next_rx) = reopened.take_event_rx() {
Self::spawn_android_backend_event_task(
android_voice_unit.clone(),
android_voice_stream_config.clone(),
voice_out_tx.clone(),
transmit_gate.clone(),
frames_sent.clone(),
audio_handler.clone(),
output_gain.clone(),
output_muted.clone(),
voice_activity_selector.clone(),
audio_processing_config.clone(),
audio_processing_stats.clone(),
mic_gain,
audio_mode_stack.clone(),
next_rx,
);
}
crate::android_voice_unit::register_global_event_sender(
reopened.event_sender(),
);
if crate::android_voice_unit::chanora_android_request_audio_focus() {
info!(
target: "chanora_audio",
"android: audio focus re-requested after backend disconnect"
);
}
if crate::android_voice_unit::chanora_android_start_bluetooth_sco() {
info!(
target: "chanora_audio",
"android: bluetooth route re-engaged after backend disconnect"
);
}
*android_voice_unit.lock().unwrap() = Some(reopened);
}
Err(e) => warn!(
target: "chanora_audio",
error = %e,
"android: reopened voice unit failed to start after backend disconnect"
),
}
}
Err(e) => warn!(
target: "chanora_audio",
error = %e,
"android: failed to reopen voice unit after backend disconnect"
),
}
}
BackendEvent::FocusLost => {
warn!(
target: "chanora_audio",
"android: audio focus lost permanently (SDD-109); engine should leave session"
);
}
BackendEvent::FocusTransient => {
info!(
target: "chanora_audio",
"android: transient audio focus loss (SDD-109); pausing capture"
);
}
BackendEvent::FocusTransientCanDuck => {
info!(
target: "chanora_audio",
"android: transient audio focus loss with ducking (SDD-109); continuing"
);
}
BackendEvent::FocusGain => {
info!(
target: "chanora_audio",
"android: audio focus regained (SDD-109); resuming capture"
);
}
BackendEvent::BluetoothScoStateChanged(s) => {
info!(
target: "chanora_audio",
sco_state = s,
"android: Bluetooth SCO state changed (SDD-110)"
);
}
}
}
});
}
/// Start the engine: open capture + playback streams, spawn the
/// inbound-voice forwarder, return a handle.
pub fn start(
@@ -461,21 +711,22 @@ impl AudioEngine {
"starting audio engine: cpal host selected"
);
/// Helper: find a device by name, falling back to default.
fn find_device(
/// Helper: find a device by stable id, falling back to default.
fn find_device<DefaultFn, AllFn, Devices>(
host: &cpal::Host,
default_fn: fn(&cpal::Host) -> Option<cpal::Device>,
all_fn: fn(&cpal::Host) -> Result<cpal::Devices, cpal::DevicesError>,
default_fn: DefaultFn,
all_fn: AllFn,
prefer: Option<&str>,
) -> Option<cpal::Device> {
if let Some(name) = prefer {
) -> Option<cpal::Device>
where
DefaultFn: Fn(&cpal::Host) -> Option<cpal::Device>,
AllFn: Fn(&cpal::Host) -> Result<Devices, cpal::DevicesError>,
Devices: IntoIterator<Item = cpal::Device>,
{
if let Some(id) = prefer {
if let Ok(devices) = all_fn(host) {
for d in devices {
let dn = d
.description()
.map(|n| n.name().to_owned())
.unwrap_or_default();
if dn == name {
if desktop_device_id(&d).as_deref() == Some(id) {
return Some(d);
}
}
@@ -488,7 +739,7 @@ impl AudioEngine {
&host,
cpal::Host::default_input_device,
cpal::Host::input_devices,
cfg.input_device_name.as_deref(),
cfg.input_device_id.as_deref(),
)
.ok_or(AudioError::NoInputDevice)?;
@@ -496,7 +747,7 @@ impl AudioEngine {
&host,
cpal::Host::default_output_device,
cpal::Host::output_devices,
cfg.output_device_name.as_deref(),
cfg.output_device_id.as_deref(),
)
.ok_or(AudioError::NoOutputDevice)?;
@@ -729,7 +980,7 @@ impl AudioEngine {
mut voice_in_rx: mpsc::Receiver<InboundVoice>,
transmit_gate: crate::ptt::AudioTransmitGate,
) -> Result<Self, AudioError> {
use crate::mobile_voice_backend::{BackendEvent, MobileVoiceAudioBackend};
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
info!(target: "chanora_audio", "starting audio engine: Android Oboe backend");
@@ -799,7 +1050,7 @@ impl AudioEngine {
..Default::default()
};
let params = crate::mobile_voice_backend::VoiceAudioParams {
voice_out_tx,
voice_out_tx: voice_out_tx.clone(),
transmit_active: transmit_flag_for_capture,
frames_sent: frames_sent.clone(),
mic_gain: cfg.mic_gain,
@@ -821,53 +1072,42 @@ impl AudioEngine {
)));
}
if let Some(mut event_rx) = android_voice_unit.take_event_rx() {
tokio::spawn(async move {
while let Some(event) = event_rx.recv().await {
match event {
BackendEvent::Disconnected => {
warn!(
target: "chanora_audio",
"android: backend disconnected event received; stream reconnect requires session restart"
);
}
BackendEvent::FocusLost => {
warn!(
target: "chanora_audio",
"android: audio focus lost permanently (SDD-109); engine should leave session"
);
}
BackendEvent::FocusTransient => {
info!(
target: "chanora_audio",
"android: transient audio focus loss (SDD-109); pausing capture"
);
}
BackendEvent::FocusTransientCanDuck => {
info!(
target: "chanora_audio",
"android: transient audio focus loss with ducking (SDD-109); continuing"
);
}
BackendEvent::FocusGain => {
info!(
target: "chanora_audio",
"android: audio focus regained (SDD-109); resuming capture"
);
}
BackendEvent::BluetoothScoStateChanged(s) => {
info!(
target: "chanora_audio",
sco_state = s,
"android: Bluetooth SCO state changed (SDD-110)"
);
}
}
}
});
let android_voice_unit = Arc::new(Mutex::new(Some(android_voice_unit)));
let android_voice_stream_config = Arc::new(Mutex::new(cfg_av.clone()));
let audio_mode_stack = Arc::new(Mutex::new(audio_mode_stack));
if let Some(event_rx) = android_voice_unit
.lock()
.unwrap()
.as_mut()
.and_then(|unit| unit.take_event_rx())
{
Self::spawn_android_backend_event_task(
android_voice_unit.clone(),
android_voice_stream_config.clone(),
voice_out_tx.clone(),
transmit_gate.clone(),
frames_sent.clone(),
audio_handler.clone(),
output_gain.clone(),
output_muted.clone(),
cfg.voice_activity_selector.clone(),
audio_processing_config.clone(),
audio_processing_stats.clone(),
cfg.mic_gain,
audio_mode_stack.clone(),
event_rx,
);
}
crate::android_voice_unit::register_global_event_sender(android_voice_unit.event_sender());
crate::android_voice_unit::register_global_event_sender(
android_voice_unit
.lock()
.unwrap()
.as_ref()
.expect("android voice unit installed")
.event_sender(),
);
if crate::android_voice_unit::chanora_android_request_audio_focus() {
info!(
@@ -932,8 +1172,12 @@ impl AudioEngine {
audio_processing_config,
audio_processing_stats,
audio_handler,
_android_voice_unit: Mutex::new(Some(android_voice_unit)),
audio_mode_stack: Mutex::new(audio_mode_stack),
voice_out_tx,
voice_activity_selector: cfg.voice_activity_selector.clone(),
mic_gain: cfg.mic_gain,
_android_voice_unit: android_voice_unit,
android_voice_stream_config,
audio_mode_stack,
shutdown_tx: Some(shutdown_tx),
capture_active,
})
@@ -1182,6 +1426,96 @@ impl AudioEngine {
}
}
/// Android-only: reopen the underlying Oboe voice backend after a
/// route/device change while preserving the engine-owned state.
pub fn android_restart_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(target_os = "android")]
{
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
crate::android_voice_unit::chanora_android_abandon_audio_focus();
crate::android_voice_unit::clear_global_event_sender();
if let Some(mut unit) = self._android_voice_unit.lock().unwrap().take() {
unit.close().map_err(|e| {
AudioError::Backend(format!(
"android: failed to close Oboe voice unit during route restart: {e}"
))
})?;
}
if crate::android_voice_unit::chanora_android_start_voice_service() {
info!(
target: "chanora_audio",
"android: voice foreground service restart dispatched after route change"
);
}
if crate::android_voice_unit::chanora_android_request_audio_focus() {
info!(
target: "chanora_audio",
"android: audio focus re-requested after route change"
);
}
let cfg_av = self.android_voice_stream_config.lock().unwrap().clone();
let params = crate::mobile_voice_backend::VoiceAudioParams {
voice_out_tx: self.voice_out_tx.clone(),
transmit_active: self.transmit_gate.flag_arc(),
frames_sent: self.frames_sent.clone(),
mic_gain: self.mic_gain,
handler: self.audio_handler.clone(),
output_gain: self.output_gain.clone(),
output_muted: self.output_muted.clone(),
voice_activity_selector: self.voice_activity_selector.clone(),
audio_processing_config: self.audio_processing_config.clone(),
audio_processing_stats: self.audio_processing_stats.clone(),
};
let mut reopened = crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params)
.map_err(|e| {
AudioError::Backend(format!("android: failed to reopen Oboe voice unit: {e}"))
})?;
reopened.start().map_err(|e| {
AudioError::Backend(format!("android: failed to restart Oboe voice unit: {e}"))
})?;
if let Some(event_rx) = reopened.take_event_rx() {
Self::spawn_android_backend_event_task(
self._android_voice_unit.clone(),
self.android_voice_stream_config.clone(),
self.voice_out_tx.clone(),
self.transmit_gate.clone(),
self.frames_sent.clone(),
self.audio_handler.clone(),
self.output_gain.clone(),
self.output_muted.clone(),
self.voice_activity_selector.clone(),
self.audio_processing_config.clone(),
self.audio_processing_stats.clone(),
self.mic_gain,
self.audio_mode_stack.clone(),
event_rx,
);
}
crate::android_voice_unit::register_global_event_sender(reopened.event_sender());
if crate::android_voice_unit::chanora_android_start_bluetooth_sco() {
info!(
target: "chanora_audio",
"android: bluetooth route re-engaged after route change"
);
}
let mut guard = self._android_voice_unit.lock().unwrap();
*guard = Some(reopened);
Ok(())
}
#[cfg(not(target_os = "android"))]
{
Ok(())
}
}
/// iOS-only: pause the underlying VoiceProcessingIO unit.
pub fn ios_pause_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(any(target_os = "ios", target_os = "macos"))]
@@ -2086,7 +2420,7 @@ pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> {
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
#[doc(hidden)]
pub mod bench_seam {
use super::{Arc, AtomicBool, AtomicU32, CaptureState, OpusEncoder, OutPacket};
use super::{Arc, AtomicBool, AtomicU32, CaptureState, OutPacket};
use tokio::sync::mpsc;
/// Opaque handle wrapping a CaptureState plus the dummy mpsc
+62 -89
View File
@@ -117,10 +117,8 @@ mod inner {
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
ten_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
/// Processing config — retained for route-change reloads.
@@ -157,10 +155,8 @@ mod inner {
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config: params.audio_processing_config.clone(),
@@ -274,6 +270,18 @@ mod inner {
rec.push_processed_mic(&frame);
}
let voice_activity_mode = self
.voice_activity_selector
.as_ref()
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_vad_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
let (vad_backend, vad_hangover) = self
.audio_processing_config
.try_lock()
@@ -282,30 +290,28 @@ mod inner {
crate::VadBackend::WebrtcVad,
crate::voice_activity::VAD_HANGOVER_MS,
));
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
if voice_activity_mode {
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
}
// Switch VAD backend when config changes.
// Switch VAD backend only while VoiceActivity mode is active.
let silero_epoch = crate::vad::silero_model_epoch();
let ten_epoch = crate::vad::ten_model_epoch();
let silero_changed = vad_backend == crate::VadBackend::SileroOnnx
let silero_changed = voice_activity_mode
&& vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
let ten_changed =
vad_backend == crate::VadBackend::TenVad && ten_epoch != self.ten_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed || ten_changed {
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_changed) {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.ten_model_epoch = ten_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
let path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
self.ten_vad_worker = None;
if self.silero_vad_worker.is_none() {
tracing::warn!(
target: "chanora_audio",
@@ -313,46 +319,43 @@ mod inner {
);
}
}
crate::VadBackend::TenVad => {
let path = crate::vad::ten_model_bundle_path();
self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path);
self.silero_vad_worker = None;
if self.ten_vad_worker.is_none() {
tracing::warn!(
target: "chanora_audio",
"TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD"
);
}
}
_ => {
self.silero_vad_worker = None;
self.ten_vad_worker = None;
}
}
self.vad_state.reset();
}
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
let (vad_probability, active) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
@@ -363,56 +366,26 @@ mod inner {
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
(vad.probability, self.vad_state.update(vad.speech))
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
(0.0, false)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let active = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(sel) = &self.voice_activity_selector {
sel.set_voice_activity_open(active && !output_muted);
sel.set_voice_activity_open(voice_activity_mode && active);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
active && !output_muted,
vad_probability,
voice_activity_mode && active,
self.transmit_active.load(Ordering::Relaxed),
);
if !self.transmit_active.load(Ordering::Relaxed) || output_muted {
if !self.transmit_active.load(Ordering::Relaxed) {
return;
}
+65 -90
View File
@@ -146,13 +146,10 @@ struct IosCaptureState {
/// Background Silero worker — enqueues frames off the realtime
/// callback and publishes the latest probability atomically.
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad: Option<crate::vad::TenOnnxVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
/// Last observed configured TEN model epoch.
ten_model_epoch: u64,
fallback_warned_backend: Option<crate::VadBackend>,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
@@ -191,10 +188,8 @@ impl IosCaptureState {
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config: params.audio_processing_config.clone(),
@@ -361,13 +356,23 @@ impl IosCaptureState {
crate::AudioBackend::PlatformVoiceProcessing,
));
// Switch VAD backend when the config changes.
let voice_activity_mode = self
.voice_activity_selector
.as_ref()
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
.unwrap_or(false);
if !voice_activity_mode {
self.silero_vad_worker = None;
self.current_vad_backend = crate::VadBackend::Disabled;
self.fallback_warned_backend = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
// Switch VAD backend only while VoiceActivity mode is active.
let silero_model_epoch = crate::vad::silero_model_epoch();
let ten_model_epoch = crate::vad::ten_model_epoch();
let silero_model_changed = vad_backend == crate::VadBackend::SileroOnnx
let silero_model_changed = voice_activity_mode
&& vad_backend == crate::VadBackend::SileroOnnx
&& silero_model_epoch != self.silero_model_epoch;
let ten_model_changed =
vad_backend == crate::VadBackend::TenVad && ten_model_epoch != self.ten_model_epoch;
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
if debug_wav_dump_enabled {
@@ -382,11 +387,11 @@ impl IosCaptureState {
}
}
if vad_backend != self.current_vad_backend || silero_model_changed || ten_model_changed {
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_model_changed)
{
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
self.silero_model_epoch = silero_model_epoch;
self.ten_model_epoch = ten_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
// Attempt to load Silero model from the well-known
@@ -407,23 +412,8 @@ impl IosCaptureState {
self.audio_processing_stats
.set_vad_fallback_active(self.silero_vad_worker.is_none());
}
crate::VadBackend::TenVad => {
self.silero_vad_worker = None;
let model_path = crate::vad::ten_model_bundle_path();
self.ten_vad = crate::vad::TenOnnxVadWorker::try_new(&model_path);
if self.ten_vad.is_none() {
warn!(
target: "chanora_audio",
"TEN VAD ONNX model not available at {model_path}; falling back to WebRTC VAD"
);
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
}
self.audio_processing_stats
.set_vad_fallback_active(self.ten_vad.is_none());
}
_ => {
self.silero_vad_worker = None;
self.ten_vad = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
}
@@ -436,13 +426,16 @@ impl IosCaptureState {
self.vad_state.reset();
}
// Keep the VAD state machine aligned with the active config.
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
if voice_activity_mode {
// Keep the VAD state machine aligned with the active config only
// while VoiceActivity mode owns the transmit gate.
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
}
// VPIO owns AEC. Keep the legacy non-AEC conditioning path here until
// the raw WebRTC APM path is explicitly selected.
@@ -461,29 +454,37 @@ impl IosCaptureState {
self.sonora_processor.process_capture(&mut frame);
}
// VAD: use Silero if loaded, otherwise WebRTC fallback.
// Disabled backend → always open (Continuous-like for VAD mode).
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
// VAD: only evaluate while VoiceActivity mode is active.
let (vad_probability, gate_open) = if voice_activity_mode {
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
@@ -491,49 +492,23 @@ impl IosCaptureState {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
(vad.probability, self.vad_state.update(vad.speech))
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
(0.0, false)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let gate_open = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(selector) = &self.voice_activity_selector {
selector.set_voice_activity_open(gate_open && !output_muted);
selector.set_voice_activity_open(voice_activity_mode && gate_open && !output_muted);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
gate_open && !output_muted,
vad_probability,
voice_activity_mode && gate_open && !output_muted,
transmit_active,
);
@@ -203,8 +203,8 @@ impl Default for AndroidVoiceStreamConfig {
Self {
sample_rate: 48_000,
channel_count: 1,
request_low_latency: true,
request_exclusive: true,
request_low_latency: false,
request_exclusive: false,
effects: AudioEffects::default(),
}
}
@@ -488,7 +488,7 @@ impl fmt::Display for AchievedInputPreset {
#[derive(Debug, Clone)]
pub struct AndroidAudioDiagnostics {
// Requested side (SDD-112 items 4..7) — fixed by SDD-112.
/// Requested performance mode (always "LowLatency" in P0).
/// Requested performance mode for the current Android request profile.
pub requested_performance_mode: &'static str,
/// Requested output usage (always "VoiceCommunication" in P0).
pub requested_usage: &'static str,
@@ -496,7 +496,7 @@ pub struct AndroidAudioDiagnostics {
pub requested_content_type: &'static str,
/// Requested input preset (always "VoiceCommunication" first).
pub requested_input_preset: &'static str,
/// Requested sharing mode (always "Exclusive" first).
/// Requested sharing mode for the current Android request profile.
pub requested_sharing_mode: &'static str,
/// Requested sample rate (Hz).
pub requested_sample_rate_hz: u32,
@@ -640,15 +640,15 @@ pub fn current_android_audio_diagnostics() -> Option<AndroidAudioDiagnostics> {
mod tests {
use super::*;
/// SWE4-UV-047: default config records requested low-latency,
/// exclusive sharing, mono 48 kHz, and default effects.
/// SWE4-UV-047: default config records the conservative shared
/// voice-input request profile, mono 48 kHz, and default effects.
#[test]
fn swe4_uv_047_default_config_records_requested_values() {
let cfg = AndroidVoiceStreamConfig::default();
assert_eq!(cfg.sample_rate, 48_000);
assert_eq!(cfg.channel_count, 1);
assert!(cfg.request_low_latency);
assert!(cfg.request_exclusive);
assert!(!cfg.request_low_latency);
assert!(!cfg.request_exclusive);
// AudioEffects defaults are all-on per DEC-007..010.
assert!(cfg.effects.aec);
assert!(cfg.effects.noise_suppression);
@@ -702,7 +702,9 @@ mod tests {
);
}
/// SWE4-UV-049: sharing-mode ladder is Exclusive -> Shared -> exhausted.
/// SWE4-UV-049: sharing-mode ladder remains Exclusive -> Shared -> exhausted.
/// The default request profile may start at Shared, but the ladder still
/// exists for explicitly opt-in low-latency / exclusive experiments.
#[test]
fn swe4_uv_049_sharing_mode_fallback_ladder_order() {
assert_eq!(
-74
View File
@@ -7,7 +7,6 @@
pub mod resampler;
pub mod silero_onnx;
pub mod ten_onnx;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{OnceLock, RwLock};
@@ -17,7 +16,6 @@ use crate::AudioError;
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
pub use silero_onnx::SileroOnnxVad;
pub use ten_onnx::{TenOnnxVad, TenOnnxVadWorker};
/// Voice activity detector output for one 10 ms frame.
#[derive(Debug, Clone, Copy)]
@@ -108,17 +106,11 @@ pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
static TEN_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
static TEN_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
}
fn ten_model_path_override() -> &'static RwLock<Option<String>> {
TEN_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
}
/// Configure the preferred Silero ONNX model path.
///
/// The path is validated eagerly. A successful call increments the
@@ -149,32 +141,6 @@ pub fn silero_model_epoch() -> u64 {
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
}
/// Configure the preferred TEN VAD ONNX model path.
pub fn set_ten_model_path(path: &str) -> Result<(), AudioError> {
let path = path.trim();
if path.is_empty() {
return Err(AudioError::InvalidAudioProcessingConfig(
"ten vad model path must not be empty".to_string(),
));
}
if !std::path::Path::new(path).is_file() {
return Err(AudioError::InvalidAudioProcessingConfig(format!(
"ten vad model path does not exist or is not a file: {path}"
)));
}
let mut guard = ten_model_path_override()
.write()
.map_err(|_| AudioError::Backend("ten vad model path lock poisoned".to_string()))?;
*guard = Some(path.to_string());
TEN_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
Ok(())
}
/// Monotonic counter incremented whenever the configured TEN model path changes.
pub fn ten_model_epoch() -> u64 {
TEN_MODEL_EPOCH.load(Ordering::Relaxed)
}
/// Return the expected path of the Silero VAD v6 ONNX model.
/// The model is shipped as a Flutter asset and copied to the app's
/// data directory by the Dart-side asset loader.
@@ -231,46 +197,6 @@ pub fn silero_model_bundle_path() -> String {
}
}
/// Return the expected path of the TEN VAD ONNX model copied by Flutter.
pub fn ten_model_bundle_path() -> String {
if let Ok(guard) = ten_model_path_override().read() {
if let Some(path) = guard.as_ref() {
return path.clone();
}
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
if let Ok(home) = std::env::var("HOME") {
let docs = format!("{home}/Documents/ten_vad.onnx");
if std::path::Path::new(&docs).exists() {
return docs;
}
let bundle = format!("{home}/../Library/ten_vad.onnx");
if std::path::Path::new(&bundle).exists() {
return bundle;
}
}
"ten_vad.onnx".to_string()
}
#[cfg(target_os = "android")]
{
"ten_vad.onnx".to_string()
}
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
{
if let Ok(cwd) = std::env::current_dir() {
let local = cwd.join("ten_vad.onnx");
if local.exists() {
return local.to_string_lossy().to_string();
}
}
"ten_vad.onnx".to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
-452
View File
@@ -1,452 +0,0 @@
//! TEN VAD ONNX backend.
//!
//! TEN's ONNX graph does not accept raw PCM. It expects the same feature
//! stack produced by TEN's `AUP_Aed_aivad_proc`: three context frames of
//! 40 log-mel powers plus one pitch feature, followed by four recurrent
//! state tensors. This module ports that preprocessing path to Rust and
//! keeps ONNX Runtime off the realtime callback where possible.
use crate::frame::f32_to_i16;
use rustfft::{num_complex::Complex32, FftPlanner};
use super::resampler::{Downsampler48to16, INPUT_FRAME_10MS};
use super::{VadOutput, VoiceActivityDetector};
const SAMPLE_RATE_16K: f32 = 16_000.0;
const HOP_16K: usize = 256;
const WINDOW_16K: usize = 768;
const FFT_SIZE: usize = 1024;
const N_BINS: usize = FFT_SIZE / 2 + 1;
const MEL_BANDS: usize = 40;
const FEATURE_LEN: usize = 41;
const CONTEXT: usize = 3;
const HIDDEN: usize = 64;
const POWER_NORMALIZER: f32 = 32768.0 * 32768.0;
const EPS: f32 = 1.0e-20;
const FEATURE_MEANS: [f32; FEATURE_LEN] = [
-8.198236, -6.2657166, -5.4838185, -4.7586913, -4.417089, -4.142893, -3.9128504, -3.845928,
-3.6570904, -3.7234187, -3.8761342, -3.843891, -3.6904051, -3.7560658, -3.6986961, -3.650463,
-3.7004688, -3.5673213, -3.4989002, -3.477807, -3.458816, -3.4449239, -3.4013286, -3.3062613,
-3.2785568, -3.2332509, -3.198616, -3.2045264, -3.2087986, -3.257838, -3.3813767, -3.5340214,
-3.640868, -3.7268589, -3.773731, -3.8046672, -3.832901, -3.8711205, -3.990593, -4.4802895,
92.3569,
];
const FEATURE_STDS: [f32; FEATURE_LEN] = [
5.166064, 4.9772096, 4.698896, 4.6306214, 4.634348, 4.641156, 4.6406765, 4.666367, 4.6505346,
4.640021, 4.6374, 4.620099, 4.5963163, 4.562655, 4.5543604, 4.5669107, 4.56249, 4.5624127,
4.5852995, 4.6001797, 4.592846, 4.5859227, 4.5834966, 4.626093, 4.626958, 4.6262894, 4.637006,
4.683016, 4.726814, 4.7342896, 4.753227, 4.849723, 4.869435, 4.884483, 4.921327, 4.9592123,
4.996619, 5.0448236, 5.072217, 5.0964394, 115.21369,
];
/// TEN VAD using ONNX Runtime and Rust-ported TEN feature preprocessing.
pub struct TenOnnxVad {
session: ort::session::Session,
downsampler: Downsampler48to16,
hop_accum: Vec<f32>,
sample_fifo: Vec<f32>,
feature_stack: [[f32; FEATURE_LEN]; CONTEXT],
states: [[f32; HIDDEN]; 4],
mel_filters: Vec<[f32; N_BINS]>,
fft: std::sync::Arc<dyn rustfft::Fft<f32>>,
fft_buffer: Vec<Complex32>,
last_probability: f32,
last_speech: bool,
}
unsafe impl Send for TenOnnxVad {}
impl TenOnnxVad {
/// Load TEN VAD ONNX model.
pub fn try_new(model_path: &str) -> Option<Self> {
if !std::path::Path::new(model_path).exists() {
tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model not found");
return None;
}
let session = match std::panic::catch_unwind(|| {
ort::session::Session::builder().and_then(|mut b| b.commit_from_file(model_path))
}) {
Ok(Ok(session)) => session,
Ok(Err(error)) => {
tracing::warn!(target: "chanora_audio", %error, path = model_path, "TEN VAD ONNX model load failed");
return None;
}
Err(_) => {
tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX Runtime panicked during load");
return None;
}
};
let mut fft_planner = FftPlanner::<f32>::new();
let fft = fft_planner.plan_fft_forward(FFT_SIZE);
tracing::info!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model loaded");
Some(Self {
session,
downsampler: Downsampler48to16::default(),
hop_accum: Vec::with_capacity(HOP_16K + super::resampler::OUTPUT_FRAME_10MS),
sample_fifo: Vec::with_capacity(WINDOW_16K + HOP_16K),
feature_stack: [[0.0; FEATURE_LEN]; CONTEXT],
states: [[0.0; HIDDEN]; 4],
mel_filters: build_mel_filters(),
fft,
fft_buffer: vec![Complex32::ZERO; FFT_SIZE],
last_probability: 0.0,
last_speech: false,
})
}
fn process_hop(&mut self, hop: &[f32]) {
self.sample_fifo.extend_from_slice(hop);
let frame = if self.sample_fifo.len() >= WINDOW_16K {
let start = self.sample_fifo.len() - WINDOW_16K;
self.sample_fifo[start..].to_vec()
} else {
let mut padded = vec![0.0; WINDOW_16K - self.sample_fifo.len()];
padded.extend_from_slice(&self.sample_fifo);
padded
};
if self.sample_fifo.len() > WINDOW_16K {
let excess = self.sample_fifo.len() - WINDOW_16K;
self.sample_fifo.drain(..excess);
}
let feature = compute_feature(
&self.mel_filters,
self.fft.as_ref(),
&mut self.fft_buffer,
&frame,
);
self.feature_stack.copy_within(1..CONTEXT, 0);
self.feature_stack[CONTEXT - 1] = feature;
self.run_onnx();
}
fn run_onnx(&mut self) {
use ndarray::{Array, IxDyn};
use ort::value::Value;
let input: Vec<f32> = self.feature_stack.iter().flatten().copied().collect();
let input_arr = match Array::from_shape_vec(IxDyn(&[1, CONTEXT, FEATURE_LEN]), input) {
Ok(v) => v,
Err(_) => return,
};
let state_arrs = [0, 1, 2, 3]
.map(|idx| Array::from_shape_vec(IxDyn(&[1, HIDDEN]), self.states[idx].to_vec()));
let input_val = match Value::from_array(input_arr) {
Ok(v) => v,
Err(error) => {
tracing::warn!(target: "chanora_audio", %error, "TEN VAD input tensor error");
return;
}
};
let state_vals = match state_arrs {
[Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d],
_ => return,
};
let state_vals = match state_vals.map(Value::from_array) {
[Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d],
_ => return,
};
let outputs = match self.session.run([
(&input_val).into(),
(&state_vals[0]).into(),
(&state_vals[1]).into(),
(&state_vals[2]).into(),
(&state_vals[3]).into(),
]) {
Ok(outputs) => outputs,
Err(error) => {
tracing::warn!(target: "chanora_audio", %error, "TEN VAD ONNX inference failed");
return;
}
};
if let Ok((_, prob)) = outputs["output_1"].try_extract_tensor::<f32>() {
if let Some(&p) = prob.first() {
self.last_probability = p.clamp(0.0, 1.0);
self.last_speech = self.last_probability >= 0.5;
}
}
for (idx, name) in ["output_2", "output_3", "output_6", "output_7"]
.iter()
.enumerate()
{
if let Ok((_, state)) = outputs[*name].try_extract_tensor::<f32>() {
let copy_len = state.len().min(HIDDEN);
self.states[idx][..copy_len].copy_from_slice(&state[..copy_len]);
}
}
}
}
fn compute_feature(
mel_filters: &[[f32; N_BINS]],
fft: &dyn rustfft::Fft<f32>,
fft_buffer: &mut [Complex32],
frame: &[f32],
) -> [f32; FEATURE_LEN] {
let power = power_spectrum(fft, fft_buffer, frame);
let mut feature = [0.0; FEATURE_LEN];
for band in 0..MEL_BANDS {
let energy = mel_filters[band]
.iter()
.zip(power.iter())
.map(|(w, p)| w * p)
.sum::<f32>()
/ POWER_NORMALIZER;
let log_energy = (energy + EPS).ln();
feature[band] = (log_energy - FEATURE_MEANS[band]) / (FEATURE_STDS[band] + EPS);
}
let pitch_hz = estimate_pitch_hz(frame);
feature[MEL_BANDS] = (pitch_hz - FEATURE_MEANS[MEL_BANDS]) / (FEATURE_STDS[MEL_BANDS] + EPS);
feature
}
impl VoiceActivityDetector for TenOnnxVad {
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
debug_assert_eq!(samples.len(), INPUT_FRAME_10MS);
let mut input = [0.0_f32; INPUT_FRAME_10MS];
input.copy_from_slice(samples);
let downsampled = self.downsampler.process_frame_10ms(&input);
self.hop_accum.extend_from_slice(&downsampled);
while self.hop_accum.len() >= HOP_16K {
let hop: Vec<f32> = self.hop_accum[..HOP_16K].to_vec();
self.hop_accum.drain(..HOP_16K);
self.process_hop(&hop);
}
VadOutput {
probability: self.last_probability,
speech: self.last_speech,
}
}
}
fn hz_to_mel(hz: f32) -> f32 {
2595.0 * (1.0 + hz / 700.0).log10()
}
fn mel_to_hz(mel: f32) -> f32 {
700.0 * (10.0_f32.powf(mel / 2595.0) - 1.0)
}
fn build_mel_filters() -> Vec<[f32; N_BINS]> {
let low_mel = hz_to_mel(0.0);
let high_mel = hz_to_mel(8000.0);
let mut bins = [0_usize; MEL_BANDS + 2];
for idx in 0..bins.len() {
let mel = idx as f32 * (high_mel - low_mel) / (MEL_BANDS as f32 + 1.0) + low_mel;
let hz = mel_to_hz(mel);
let mut bin = ((FFT_SIZE as f32 + 1.0) * hz / SAMPLE_RATE_16K).floor() as usize;
bin = bin.min(N_BINS - 1);
if idx > 0 && bin == bins[idx - 1] {
bin = (bin + 1).min(N_BINS - 1);
}
bins[idx] = bin;
}
let mut filters = vec![[0.0_f32; N_BINS]; MEL_BANDS];
for band in 0..MEL_BANDS {
let left = bins[band];
let center = bins[band + 1].max(left + 1);
let right = bins[band + 2].max(center + 1).min(N_BINS - 1);
for (i, weight) in filters[band]
.iter_mut()
.enumerate()
.take(center.min(N_BINS))
.skip(left)
{
*weight = (i - left) as f32 / (center - left) as f32;
}
for (i, weight) in filters[band]
.iter_mut()
.enumerate()
.take(right + 1)
.skip(center)
{
*weight = (right - i) as f32 / (right - center).max(1) as f32;
}
}
filters
}
fn power_spectrum(
fft: &dyn rustfft::Fft<f32>,
fft_buffer: &mut [Complex32],
frame: &[f32],
) -> [f32; N_BINS] {
debug_assert_eq!(fft_buffer.len(), FFT_SIZE);
fft_buffer.fill(Complex32::ZERO);
for (idx, sample) in frame.iter().take(WINDOW_16K).enumerate() {
let hann = 0.5 - 0.5 * (2.0 * std::f32::consts::PI * idx as f32 / WINDOW_16K as f32).cos();
fft_buffer[idx].re = f32_to_i16(*sample) as f32 * hann;
}
fft.process(fft_buffer);
let mut out = [0.0_f32; N_BINS];
for (dst, bin) in out.iter_mut().zip(fft_buffer.iter()) {
*dst = bin.norm_sqr();
}
out
}
fn estimate_pitch_hz(frame: &[f32]) -> f32 {
let min_lag = (SAMPLE_RATE_16K / 400.0) as usize;
let max_lag = (SAMPLE_RATE_16K / 60.0) as usize;
let mut best_lag = 0_usize;
let mut best_corr = 0.0_f32;
for lag in min_lag..=max_lag.min(frame.len().saturating_sub(1)) {
let mut corr = 0.0_f32;
let mut energy = 0.0_f32;
for i in lag..frame.len() {
corr += frame[i] * frame[i - lag];
energy += frame[i - lag] * frame[i - lag];
}
let norm = if energy > 1.0e-8 {
corr / energy.sqrt()
} else {
0.0
};
if norm > best_corr {
best_corr = norm;
best_lag = lag;
}
}
if best_lag == 0 || best_corr < 0.01 {
0.0
} else {
SAMPLE_RATE_16K / best_lag as f32
}
}
// ---------------------------------------------------------------------------
// Background worker — same pattern as SileroOnnxVadWorker so the realtime
// callback never blocks on STFT / pitch / ONNX inference.
// ---------------------------------------------------------------------------
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64};
use std::sync::Arc;
use std::thread::JoinHandle;
/// Maximum number of 10 ms frames the worker may lag before the callback
/// treats its output as stale and uses WebRTC fallback instead.
const TEN_MAX_STALE_FRAMES: u64 = 8;
struct TenFrameMessage {
seq: u64,
frame: [f32; INPUT_FRAME_10MS],
}
/// Background TEN VAD worker. The realtime callback only enqueues 10 ms
/// frames and reads the latest probability atomically.
pub struct TenOnnxVadWorker {
tx: Option<std::sync::mpsc::SyncSender<TenFrameMessage>>,
latest_probability: Arc<AtomicU32>,
latest_processed_seq: Arc<AtomicU64>,
alive: Arc<AtomicBool>,
handle: Option<JoinHandle<()>>,
}
impl TenOnnxVadWorker {
/// Start a background TEN worker if the model loads.
pub fn try_new(model_path: &str) -> Option<Self> {
let vad = TenOnnxVad::try_new(model_path)?;
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<TenFrameMessage>(128);
let prob_arc = latest_probability.clone();
let seq_arc = latest_processed_seq.clone();
let alive_arc = alive.clone();
let handle = std::thread::Builder::new()
.name("chanora-ten-vad".to_string())
.spawn(move || {
let mut vad = vad;
while alive_arc.load(std::sync::atomic::Ordering::Relaxed) {
let msg = match rx.recv() {
Ok(m) => m,
Err(_) => break,
};
let mut frame_f32 = [0.0_f32; INPUT_FRAME_10MS];
frame_f32.copy_from_slice(&msg.frame);
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame_f32);
prob_arc.store(
out.probability.clamp(0.0, 1.0).to_bits(),
std::sync::atomic::Ordering::Relaxed,
);
seq_arc.store(msg.seq, std::sync::atomic::Ordering::Relaxed);
}
})
.ok()?;
Some(Self {
tx: Some(tx),
latest_probability,
latest_processed_seq,
alive,
handle: Some(handle),
})
}
/// Best-effort enqueue of a 10 ms frame for background inference.
pub fn try_send(&self, seq: u64, frame: &[f32; INPUT_FRAME_10MS]) -> bool {
let Some(tx) = &self.tx else {
return false;
};
tx.try_send(TenFrameMessage { seq, frame: *frame }).is_ok()
}
/// Latest probability published by the background worker.
pub fn latest_probability(&self) -> f32 {
f32::from_bits(
self.latest_probability
.load(std::sync::atomic::Ordering::Relaxed),
)
}
/// True when the worker is too far behind to trust its output.
pub fn is_stale(&self, capture_seq: u64) -> bool {
let latest = self
.latest_processed_seq
.load(std::sync::atomic::Ordering::Relaxed);
latest == u64::MAX || capture_seq.saturating_sub(latest) > TEN_MAX_STALE_FRAMES
}
}
impl Drop for TenOnnxVadWorker {
fn drop(&mut self) {
self.alive
.store(false, std::sync::atomic::Ordering::Relaxed);
drop(self.tx.take());
if let Some(h) = self.handle.take() {
let _ = h.join();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mel_filter_bank_has_expected_shape() {
let filters = build_mel_filters();
assert_eq!(filters.len(), MEL_BANDS);
assert!(filters.iter().all(|f| f.iter().any(|&v| v > 0.0)));
}
#[test]
fn preprocessing_produces_finite_features() {
let filters = build_mel_filters();
let mut planner = FftPlanner::<f32>::new();
let fft = planner.plan_fft_forward(FFT_SIZE);
let mut fft_buffer = vec![Complex32::ZERO; FFT_SIZE];
let frame = vec![0.0_f32; WINDOW_16K];
let feature = compute_feature(&filters, fft.as_ref(), &mut fft_buffer, &frame);
assert!(feature.iter().all(|v| v.is_finite()));
}
}
+78 -30
View File
@@ -145,7 +145,7 @@ pub(crate) fn publish_permission_state(permission: String, state: PermissionStat
/// flood the diagnostic export during transient packet loss;
/// users can still raise verbosity via `RUST_LOG=info`.
const DEFAULT_LOG_FILTER: &str =
"info,tsproto::resend=error,tsproto::packet_codec=error,tsclientlib=error";
"info,tsproto::resend=error,tsproto::packet_codec=error,tsclientlib=error,ts_bookkeeping::messages::s2c=error";
/// Initialise the bridge. Must be called once on Dart side before
/// any other API call. Sets up panic logging.
@@ -921,8 +921,6 @@ pub enum BridgeAudioBackend {
pub enum BridgeVadBackend {
/// Silero ONNX VAD.
SileroOnnx,
/// TEN VAD.
TenVad,
/// WebRTC fallback VAD.
WebrtcVad,
/// Debug energy VAD.
@@ -1014,6 +1012,16 @@ pub struct BridgeAudioProcessingStats {
pub callback_xruns: u64,
/// Clipped samples.
pub clipped_samples: u64,
/// Number of effectively silent processed capture frames.
pub zero_frames: u64,
/// Number of processed capture frames.
pub capture_frames: u64,
/// Input callbacks carrying 10 ms of audio.
pub callbacks_10ms: u64,
/// Input callbacks carrying 20 ms of audio.
pub callbacks_20ms: u64,
/// Input callbacks carrying other sizes.
pub callbacks_other: u64,
/// Sonora enabled.
pub sonora_enabled: bool,
/// Platform voice processing enabled.
@@ -1092,7 +1100,6 @@ impl From<BridgeVadBackend> for chanora_core::VadBackend {
fn from(backend: BridgeVadBackend) -> Self {
match backend {
BridgeVadBackend::SileroOnnx => Self::SileroOnnx,
BridgeVadBackend::TenVad => Self::TenVad,
BridgeVadBackend::WebrtcVad => Self::WebrtcVad,
BridgeVadBackend::EnergyDebug => Self::EnergyDebug,
BridgeVadBackend::Disabled => Self::Disabled,
@@ -1104,7 +1111,6 @@ impl From<chanora_core::VadBackend> for BridgeVadBackend {
fn from(backend: chanora_core::VadBackend) -> Self {
match backend {
chanora_core::VadBackend::SileroOnnx => Self::SileroOnnx,
chanora_core::VadBackend::TenVad => Self::TenVad,
chanora_core::VadBackend::WebrtcVad => Self::WebrtcVad,
chanora_core::VadBackend::EnergyDebug => Self::EnergyDebug,
chanora_core::VadBackend::Disabled => Self::Disabled,
@@ -1196,6 +1202,11 @@ impl From<chanora_core::AudioProcessingStats> for BridgeAudioProcessingStats {
output_underruns: stats.output_underruns,
callback_xruns: stats.callback_xruns,
clipped_samples: stats.clipped_samples,
zero_frames: stats.zero_frames,
capture_frames: stats.capture_frames,
callbacks_10ms: stats.callbacks_10ms,
callbacks_20ms: stats.callbacks_20ms,
callbacks_other: stats.callbacks_other,
sonora_enabled: stats.sonora_enabled,
platform_voice_processing_enabled: stats.platform_voice_processing_enabled,
}
@@ -1236,8 +1247,31 @@ pub fn export_diagnostics() -> String {
let android_audio_yaml =
chanora_audio::mobile_voice_backend::current_android_audio_diagnostics()
.map(|d| d.to_yaml_fragment());
let audio_health = runtime().block_on(async { session().audio_processing_stats().await.ok() });
let network_info = runtime().block_on(async { session().network_diagnostics_summary().await });
let protocol_events = runtime().block_on(async { session().drain_protocol_events().await });
let android_audio_yaml = android_audio_yaml.map(|mut yaml| {
if let Some(stats) = audio_health {
yaml.push_str(&format!(
" health:\n\
\x20\x20\x20\x20capture_frames: {}\n\
\x20\x20\x20\x20zero_frames: {}\n\
\x20\x20\x20\x20callbacks_10ms: {}\n\
\x20\x20\x20\x20callbacks_20ms: {}\n\
\x20\x20\x20\x20callbacks_other: {}\n\
\x20\x20\x20\x20callback_xruns: {}\n\
\x20\x20\x20\x20clipped_samples: {}\n",
stats.capture_frames,
stats.zero_frames,
stats.callbacks_10ms,
stats.callbacks_20ms,
stats.callbacks_other,
stats.callback_xruns,
stats.clipped_samples,
));
}
yaml
});
match chanora_core::DiagnosticExport::from_sink(log_sink(), metadata) {
Ok(exp) => exp
.with_android_audio(android_audio_yaml)
@@ -1506,6 +1540,11 @@ pub enum BridgeEvent {
/// Target scope (server/channel/private/poke).
target: BridgeMessageTarget,
},
/// Human-readable server activity surfaced from protocol bookkeeping events.
ServerActivity {
/// TeamSpeak-style activity line.
message: String,
},
/// Audio route changed (speaker/earpiece/BT/wired).
AudioRouteChanged {
/// The new audio route.
@@ -1692,6 +1731,9 @@ impl From<chanora_core::SessionEvent> for BridgeEvent {
message,
target: target.into(),
},
chanora_core::SessionEvent::ServerActivity { message } => {
BridgeEvent::ServerActivity { message }
}
chanora_core::SessionEvent::AudioRouteChanged { route } => {
BridgeEvent::AudioRouteChanged {
route: route.into(),
@@ -1848,10 +1890,16 @@ pub async fn audio_processing_stats() -> Result<BridgeAudioProcessingStats, Brid
/// Audio device info from the platform.
#[derive(Debug, Clone)]
pub struct BridgeAudioDevice {
/// Stable platform-reported device identifier.
pub id: String,
/// Human-readable device name.
pub name: String,
/// Additional device details useful for disambiguation.
pub details: String,
/// True if the OS reports this as the default device.
pub is_default: bool,
/// True if Chanora currently has this device pinned.
pub is_selected: bool,
}
/// List of available audio devices.
@@ -1864,14 +1912,26 @@ pub struct BridgeAudioDeviceList {
}
/// List available audio input and output devices from the platform.
pub fn list_audio_devices() -> BridgeAudioDeviceList {
let list = chanora_audio::list_audio_devices();
BridgeAudioDeviceList {
pub async fn list_audio_devices() -> Result<BridgeAudioDeviceList, BridgeError> {
let (list, selected_input, selected_output) = runtime()
.spawn(async move {
let selected_input = session().preferred_input_device().await;
let selected_output = session().preferred_output_device().await;
let list = chanora_audio::list_audio_devices();
(list, selected_input, selected_output)
})
.await
.map_err(|e| task_join_error("list_audio_devices", e))?;
Ok(BridgeAudioDeviceList {
input_devices: list
.input_devices
.into_iter()
.map(|d| BridgeAudioDevice {
is_selected: selected_input.as_ref().is_some_and(|id| id == &d.id),
id: d.id,
name: d.name,
details: d.details,
is_default: d.is_default,
})
.collect(),
@@ -1879,27 +1939,30 @@ pub fn list_audio_devices() -> BridgeAudioDeviceList {
.output_devices
.into_iter()
.map(|d| BridgeAudioDevice {
is_selected: selected_output.as_ref().is_some_and(|id| id == &d.id),
id: d.id,
name: d.name,
details: d.details,
is_default: d.is_default,
})
.collect(),
}
})
}
/// Set the preferred input device by name. Takes effect on next
/// Set the preferred input device by id. Takes effect on next
/// `start_audio`.
pub async fn set_input_device(name: Option<String>) -> Result<(), BridgeError> {
pub async fn set_input_device(id: Option<String>) -> Result<(), BridgeError> {
runtime()
.spawn(async move { session().set_input_device(name).await })
.spawn(async move { session().set_input_device(id).await })
.await
.map_err(|e| task_join_error("set_input_device", e))??;
Ok(())
}
/// Set the preferred output device by name.
pub async fn set_output_device(name: Option<String>) -> Result<(), BridgeError> {
/// Set the preferred output device by id.
pub async fn set_output_device(id: Option<String>) -> Result<(), BridgeError> {
runtime()
.spawn(async move { session().set_output_device(name).await })
.spawn(async move { session().set_output_device(id).await })
.await
.map_err(|e| task_join_error("set_output_device", e))??;
Ok(())
@@ -1919,21 +1982,6 @@ pub async fn set_vad_model_path(path: String) -> Result<(), BridgeError> {
Ok(())
}
/// Configure the TEN VAD ONNX model path.
pub async fn set_ten_vad_model_path(path: String) -> Result<(), BridgeError> {
if path.trim().is_empty() {
return Err(BridgeError::InvalidCommand(
"ten vad model path must not be empty".to_string(),
));
}
runtime()
.spawn(async move { chanora_audio::vad::set_ten_model_path(&path) })
.await
.map_err(|e| task_join_error("set_ten_vad_model_path", e))?
.map_err(|e| BridgeError::Unmapped(format!("set_ten_vad_model_path: {e}")))?;
Ok(())
}
/// Enable or disable audio debug WAV dumping.
pub async fn enable_audio_debug_wav_dump(enabled: bool) -> Result<(), BridgeError> {
runtime()
+76 -68
View File
@@ -38,7 +38,7 @@ flutter_rust_bridge::frb_generated_boilerplate!(
default_rust_auto_opaque = RustAutoOpaqueMoi,
);
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.12.0";
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = 1433826599;
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -560177922;
// Section: executor
@@ -743,7 +743,7 @@ fn wire__crate__api__list_audio_devices_impl(
rust_vec_len_: i32,
data_len_: i32,
) {
FLUTTER_RUST_BRIDGE_HANDLER.wrap_normal::<flutter_rust_bridge::for_generated::SseCodec, _, _>(
FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
flutter_rust_bridge::for_generated::TaskInfo {
debug_name: "list_audio_devices",
port: Some(port_),
@@ -760,11 +760,14 @@ fn wire__crate__api__list_audio_devices_impl(
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
deserializer.end();
move |context| {
transform_result_sse::<_, ()>((move || {
let output_ok = Result::<_, ()>::Ok(crate::api::list_audio_devices())?;
Ok(output_ok)
})())
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::list_audio_devices().await?;
Ok(output_ok)
})()
.await,
)
}
},
)
@@ -1141,12 +1144,12 @@ fn wire__crate__api__set_input_device_impl(
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
let api_name = <Option<String>>::sse_decode(&mut deserializer);
let api_id = <Option<String>>::sse_decode(&mut deserializer);
deserializer.end();
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::set_input_device(api_name).await?;
let output_ok = crate::api::set_input_device(api_id).await?;
Ok(output_ok)
})()
.await,
@@ -1282,12 +1285,12 @@ fn wire__crate__api__set_output_device_impl(
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
let api_name = <Option<String>>::sse_decode(&mut deserializer);
let api_id = <Option<String>>::sse_decode(&mut deserializer);
deserializer.end();
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::set_output_device(api_name).await?;
let output_ok = crate::api::set_output_device(api_id).await?;
Ok(output_ok)
})()
.await,
@@ -1478,42 +1481,6 @@ fn wire__crate__api__set_release_tail_ms_impl(
},
)
}
fn wire__crate__api__set_ten_vad_model_path_impl(
port_: flutter_rust_bridge::for_generated::MessagePort,
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
rust_vec_len_: i32,
data_len_: i32,
) {
FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
flutter_rust_bridge::for_generated::TaskInfo {
debug_name: "set_ten_vad_model_path",
port: Some(port_),
mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal,
},
move || {
let message = unsafe {
flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(
ptr_,
rust_vec_len_,
data_len_,
)
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
let api_path = <String>::sse_decode(&mut deserializer);
deserializer.end();
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::set_ten_vad_model_path(api_path).await?;
Ok(output_ok)
})()
.await,
)
}
},
)
}
fn wire__crate__api__set_transmit_mode_impl(
port_: flutter_rust_bridge::for_generated::MessagePort,
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
@@ -1783,11 +1750,17 @@ impl SseDecode for crate::api::BridgeAudioBackend {
impl SseDecode for crate::api::BridgeAudioDevice {
// Codec=Sse (Serialization based), see doc to use other codecs
fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self {
let mut var_id = <String>::sse_decode(deserializer);
let mut var_name = <String>::sse_decode(deserializer);
let mut var_details = <String>::sse_decode(deserializer);
let mut var_isDefault = <bool>::sse_decode(deserializer);
let mut var_isSelected = <bool>::sse_decode(deserializer);
return crate::api::BridgeAudioDevice {
id: var_id,
name: var_name,
details: var_details,
is_default: var_isDefault,
is_selected: var_isSelected,
};
}
}
@@ -1859,6 +1832,11 @@ impl SseDecode for crate::api::BridgeAudioProcessingStats {
let mut var_outputUnderruns = <u64>::sse_decode(deserializer);
let mut var_callbackXruns = <u64>::sse_decode(deserializer);
let mut var_clippedSamples = <u64>::sse_decode(deserializer);
let mut var_zeroFrames = <u64>::sse_decode(deserializer);
let mut var_captureFrames = <u64>::sse_decode(deserializer);
let mut var_callbacks10Ms = <u64>::sse_decode(deserializer);
let mut var_callbacks20Ms = <u64>::sse_decode(deserializer);
let mut var_callbacksOther = <u64>::sse_decode(deserializer);
let mut var_sonoraEnabled = <bool>::sse_decode(deserializer);
let mut var_platformVoiceProcessingEnabled = <bool>::sse_decode(deserializer);
return crate::api::BridgeAudioProcessingStats {
@@ -1879,6 +1857,11 @@ impl SseDecode for crate::api::BridgeAudioProcessingStats {
output_underruns: var_outputUnderruns,
callback_xruns: var_callbackXruns,
clipped_samples: var_clippedSamples,
zero_frames: var_zeroFrames,
capture_frames: var_captureFrames,
callbacks_10ms: var_callbacks10Ms,
callbacks_20ms: var_callbacks20Ms,
callbacks_other: var_callbacksOther,
sonora_enabled: var_sonoraEnabled,
platform_voice_processing_enabled: var_platformVoiceProcessingEnabled,
};
@@ -2147,6 +2130,12 @@ impl SseDecode for crate::api::BridgeEvent {
};
}
12 => {
let mut var_message = <String>::sse_decode(deserializer);
return crate::api::BridgeEvent::ServerActivity {
message: var_message,
};
}
13 => {
let mut var_route = <crate::api::BridgeAudioRoute>::sse_decode(deserializer);
return crate::api::BridgeEvent::AudioRouteChanged { route: var_route };
}
@@ -2291,10 +2280,9 @@ impl SseDecode for crate::api::BridgeVadBackend {
let mut inner = <i32>::sse_decode(deserializer);
return match inner {
0 => crate::api::BridgeVadBackend::SileroOnnx,
1 => crate::api::BridgeVadBackend::TenVad,
2 => crate::api::BridgeVadBackend::WebrtcVad,
3 => crate::api::BridgeVadBackend::EnergyDebug,
4 => crate::api::BridgeVadBackend::Disabled,
1 => crate::api::BridgeVadBackend::WebrtcVad,
2 => crate::api::BridgeVadBackend::EnergyDebug,
3 => crate::api::BridgeVadBackend::Disabled,
_ => unreachable!("Invalid variant for BridgeVadBackend: {}", inner),
};
}
@@ -2544,13 +2532,12 @@ fn pde_ffi_dispatcher_primary_impl(
39 => wire__crate__api__set_ptt_impl(port, ptr, rust_vec_len, data_len),
40 => wire__crate__api__set_ptt_binding_impl(port, ptr, rust_vec_len, data_len),
41 => wire__crate__api__set_release_tail_ms_impl(port, ptr, rust_vec_len, data_len),
42 => wire__crate__api__set_ten_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
43 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
44 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
45 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len),
46 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len),
47 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len),
48 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len),
42 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
43 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
44 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len),
45 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len),
46 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len),
47 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len),
_ => unreachable!(),
}
}
@@ -2609,8 +2596,11 @@ impl flutter_rust_bridge::IntoIntoDart<crate::api::BridgeAudioBackend>
impl flutter_rust_bridge::IntoDart for crate::api::BridgeAudioDevice {
fn into_dart(self) -> flutter_rust_bridge::for_generated::DartAbi {
[
self.id.into_into_dart().into_dart(),
self.name.into_into_dart().into_dart(),
self.details.into_into_dart().into_dart(),
self.is_default.into_into_dart().into_dart(),
self.is_selected.into_into_dart().into_dart(),
]
.into_dart()
}
@@ -2697,6 +2687,11 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeAudioProcessingStats {
self.output_underruns.into_into_dart().into_dart(),
self.callback_xruns.into_into_dart().into_dart(),
self.clipped_samples.into_into_dart().into_dart(),
self.zero_frames.into_into_dart().into_dart(),
self.capture_frames.into_into_dart().into_dart(),
self.callbacks_10ms.into_into_dart().into_dart(),
self.callbacks_20ms.into_into_dart().into_dart(),
self.callbacks_other.into_into_dart().into_dart(),
self.sonora_enabled.into_into_dart().into_dart(),
self.platform_voice_processing_enabled
.into_into_dart()
@@ -2973,8 +2968,11 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeEvent {
target.into_into_dart().into_dart(),
]
.into_dart(),
crate::api::BridgeEvent::ServerActivity { message } => {
[12.into_dart(), message.into_into_dart().into_dart()].into_dart()
}
crate::api::BridgeEvent::AudioRouteChanged { route } => {
[12.into_dart(), route.into_into_dart().into_dart()].into_dart()
[13.into_dart(), route.into_into_dart().into_dart()].into_dart()
}
_ => {
unimplemented!("");
@@ -3170,10 +3168,9 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeVadBackend {
fn into_dart(self) -> flutter_rust_bridge::for_generated::DartAbi {
match self {
Self::SileroOnnx => 0.into_dart(),
Self::TenVad => 1.into_dart(),
Self::WebrtcVad => 2.into_dart(),
Self::EnergyDebug => 3.into_dart(),
Self::Disabled => 4.into_dart(),
Self::WebrtcVad => 1.into_dart(),
Self::EnergyDebug => 2.into_dart(),
Self::Disabled => 3.into_dart(),
_ => unreachable!(),
}
}
@@ -3313,8 +3310,11 @@ impl SseEncode for crate::api::BridgeAudioBackend {
impl SseEncode for crate::api::BridgeAudioDevice {
// Codec=Sse (Serialization based), see doc to use other codecs
fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) {
<String>::sse_encode(self.id, serializer);
<String>::sse_encode(self.name, serializer);
<String>::sse_encode(self.details, serializer);
<bool>::sse_encode(self.is_default, serializer);
<bool>::sse_encode(self.is_selected, serializer);
}
}
@@ -3368,6 +3368,11 @@ impl SseEncode for crate::api::BridgeAudioProcessingStats {
<u64>::sse_encode(self.output_underruns, serializer);
<u64>::sse_encode(self.callback_xruns, serializer);
<u64>::sse_encode(self.clipped_samples, serializer);
<u64>::sse_encode(self.zero_frames, serializer);
<u64>::sse_encode(self.capture_frames, serializer);
<u64>::sse_encode(self.callbacks_10ms, serializer);
<u64>::sse_encode(self.callbacks_20ms, serializer);
<u64>::sse_encode(self.callbacks_other, serializer);
<bool>::sse_encode(self.sonora_enabled, serializer);
<bool>::sse_encode(self.platform_voice_processing_enabled, serializer);
}
@@ -3595,8 +3600,12 @@ impl SseEncode for crate::api::BridgeEvent {
<String>::sse_encode(message, serializer);
<crate::api::BridgeMessageTarget>::sse_encode(target, serializer);
}
crate::api::BridgeEvent::AudioRouteChanged { route } => {
crate::api::BridgeEvent::ServerActivity { message } => {
<i32>::sse_encode(12, serializer);
<String>::sse_encode(message, serializer);
}
crate::api::BridgeEvent::AudioRouteChanged { route } => {
<i32>::sse_encode(13, serializer);
<crate::api::BridgeAudioRoute>::sse_encode(route, serializer);
}
_ => {
@@ -3734,10 +3743,9 @@ impl SseEncode for crate::api::BridgeVadBackend {
<i32>::sse_encode(
match self {
crate::api::BridgeVadBackend::SileroOnnx => 0,
crate::api::BridgeVadBackend::TenVad => 1,
crate::api::BridgeVadBackend::WebrtcVad => 2,
crate::api::BridgeVadBackend::EnergyDebug => 3,
crate::api::BridgeVadBackend::Disabled => 4,
crate::api::BridgeVadBackend::WebrtcVad => 1,
crate::api::BridgeVadBackend::EnergyDebug => 2,
crate::api::BridgeVadBackend::Disabled => 3,
_ => {
unimplemented!("");
}
+47 -51
View File
@@ -659,17 +659,11 @@ async fn connection_task(
..
} = &ev
{
let own_client = con
.get_state()
.ok()
.map(|state| state.own_client);
let own_client = con.get_state().ok().map(|state| state.own_client);
if own_client == Some(*client_id) {
let current_channel = con
.get_state()
.ok()
.and_then(|state| {
state.clients.get(client_id).map(|client| client.channel.0)
});
let current_channel = con.get_state().ok().and_then(|state| {
state.clients.get(client_id).map(|client| client.channel.0)
});
if let Some(current_channel) = current_channel {
let matched: Vec<MessageHandle> = pending_moves
.iter()
@@ -682,7 +676,9 @@ async fn connection_task(
})
.collect();
for handle in matched {
if let Some((_, reply, _)) = pending_moves.remove(&handle) {
if let Some((_, reply, _)) =
pending_moves.remove(&handle)
{
info!(
target: "chanora_protocol",
channel_id = current_channel,
@@ -727,7 +723,9 @@ async fn connection_task(
}
}
StreamItem::MessageResult(handle, result) => {
if let Some((_target_channel, reply, _deadline)) = pending_moves.remove(&handle) {
if let Some((_target_channel, reply, _deadline)) =
pending_moves.remove(&handle)
{
let mapped = match result {
Ok(()) => Ok(()),
Err(cmd_err) => {
@@ -814,36 +812,32 @@ async fn connection_task(
channel_id,
password,
reply,
}) => {
match move_self_to(&mut con, channel_id, password.as_deref()) {
Ok(handle) => {
let deadline = std::time::Instant::now() + Duration::from_secs(3);
pending_moves.insert(handle, (channel_id, Some(reply), deadline));
}
Err(e) => {
let _ = reply.send(Err(e));
}
}) => match move_self_to(&mut con, channel_id, password.as_deref()) {
Ok(handle) => {
let deadline = std::time::Instant::now() + Duration::from_secs(3);
pending_moves.insert(handle, (channel_id, Some(reply), deadline));
}
}
Err(e) => {
let _ = reply.send(Err(e));
}
},
Ok(Request::MoveToChannelNoWait {
channel_id,
password,
}) => {
match move_self_to(&mut con, channel_id, password.as_deref()) {
Ok(handle) => {
let deadline = std::time::Instant::now() + Duration::from_secs(3);
pending_moves.insert(handle, (channel_id, None, deadline));
}
Err(e) => {
warn!(
target: "chanora_protocol",
error = %e,
channel_id,
"fire-and-forget client_move could not be queued"
);
}
}) => match move_self_to(&mut con, channel_id, password.as_deref()) {
Ok(handle) => {
let deadline = std::time::Instant::now() + Duration::from_secs(3);
pending_moves.insert(handle, (channel_id, None, deadline));
}
}
Err(e) => {
warn!(
target: "chanora_protocol",
error = %e,
channel_id,
"fire-and-forget client_move could not be queued"
);
}
},
Ok(Request::SetMuted {
input,
output,
@@ -1261,10 +1255,7 @@ fn activity_channel_group_name(
.map(|group| quoted(&group.name))
}
fn activity_server_group_name(
con: &Connection,
id: tsclientlib::ServerGroupId,
) -> Option<String> {
fn activity_server_group_name(con: &Connection, id: tsclientlib::ServerGroupId) -> Option<String> {
con.get_state()
.ok()
.and_then(|state| state.server_groups.get(&id))
@@ -1282,7 +1273,11 @@ fn format_server_activity(con: &Connection, ev: &tsclientlib::events::Event) ->
use tsproto_types::Reason;
match ev {
Event::PropertyAdded { id: PropertyId::Client(client_id), extra, .. } => {
Event::PropertyAdded {
id: PropertyId::Client(client_id),
extra,
..
} => {
if extra.reason.is_none() {
return None;
}
@@ -1294,7 +1289,12 @@ fn format_server_activity(con: &Connection, ev: &tsclientlib::events::Event) ->
channel
))
}
Event::PropertyRemoved { id: PropertyId::Client(_), old, extra, .. } => {
Event::PropertyRemoved {
id: PropertyId::Client(_),
old,
extra,
..
} => {
let PropertyValue::Client(client) = old else {
return None;
};
@@ -1305,12 +1305,9 @@ fn format_server_activity(con: &Connection, ev: &tsclientlib::events::Event) ->
Some(Reason::Clientdisconnect) => {
Some(format!("{} disconnected (Leaving)", quoted(&client.name)))
}
Some(Reason::ClientdisconnectServerShutdown) | Some(Reason::Serverstop) => {
Some(format!(
"{} disconnected (server shutdown)",
quoted(&client.name)
))
}
Some(Reason::ClientdisconnectServerShutdown) | Some(Reason::Serverstop) => Some(
format!("{} disconnected (server shutdown)", quoted(&client.name)),
),
_ => Some(format!(
"{} dropped (connection lost)",
quoted(&client.name)
@@ -1329,8 +1326,7 @@ fn format_server_activity(con: &Connection, ev: &tsclientlib::events::Event) ->
let client = activity_client(con, *client_id)?;
let from = activity_channel_name(con, *from_channel_id)?;
let to = activity_channel_name(con, client.channel)?;
if invoker.as_ref().map(|invoker| invoker.id) == Some(*client_id) || invoker.is_none()
{
if invoker.as_ref().map(|invoker| invoker.id) == Some(*client_id) || invoker.is_none() {
Some(format!(
"{} switched from channel {} to {}",
quoted(&client.name),