feat: stabilize voice activity and audio routing
This commit is contained in:
@@ -17,7 +17,7 @@ sonora = "0.1"
|
||||
webrtc-vad = "0.4"
|
||||
|
||||
# ndarray is required by ort's tensor construction API and by
|
||||
# Silero / TEN VAD ONNX inference across all platforms.
|
||||
# Silero VAD ONNX inference across all platforms.
|
||||
ndarray = "0.17"
|
||||
|
||||
# Opus encoder. tsclientlib already pulls this; we depend explicitly so
|
||||
|
||||
@@ -145,16 +145,18 @@ struct AndroidCaptureState {
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad,
|
||||
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
|
||||
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
|
||||
current_vad_backend: crate::VadBackend,
|
||||
silero_model_epoch: u64,
|
||||
ten_model_epoch: u64,
|
||||
capture_frame_seq: u64,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||||
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
input_sample_rate_hz: u32,
|
||||
resample_pos: f64,
|
||||
resample_last: i16,
|
||||
resample_scratch: Vec<i16>,
|
||||
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_10ms_len: usize,
|
||||
fallback_warned_backend: Option<crate::VadBackend>,
|
||||
@@ -171,29 +173,18 @@ impl AndroidCaptureState {
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
input_sample_rate_hz: u32,
|
||||
) -> Result<Self, AudioError> {
|
||||
let encoder = crate::opus_voice::new_voip_encoder("android")?;
|
||||
// Android always uses software WebRTC APM for AEC/NS/AGC/HPF.
|
||||
// The config's EffectOwner fields are resolved by open() AFTER
|
||||
// hardware-effect binding; the processor is constructed here
|
||||
// with all modules enabled regardless, so the resolved config
|
||||
// (Platform vs WebrtcApm) only affects diagnostics, not behaviour.
|
||||
// Seed the processor from the current shared config snapshot.
|
||||
// `open()` may later resolve Platform-owned stages to WebRTC
|
||||
// fallback (or keep them hardware-owned) once hardware-effect
|
||||
// binding completes; that resolved config is pushed back into
|
||||
// the live processor before the streams are started.
|
||||
let webrtc_apm_config = audio_processing_config
|
||||
.lock()
|
||||
.map(|cfg| {
|
||||
let mut c = crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg);
|
||||
c.aec = true;
|
||||
c.ns = true;
|
||||
c.agc = true;
|
||||
c
|
||||
})
|
||||
.unwrap_or(crate::processor::webrtc_apm::WebRtcApmConfig {
|
||||
aec: true,
|
||||
ns: true,
|
||||
agc: true,
|
||||
hpf: true,
|
||||
..Default::default()
|
||||
});
|
||||
.map(|cfg| webrtc_apm_config_from_audio_config(&cfg))
|
||||
.unwrap_or_default();
|
||||
Ok(Self {
|
||||
encoder,
|
||||
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
|
||||
@@ -206,10 +197,8 @@ impl AndroidCaptureState {
|
||||
voice_activity_selector,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||||
silero_vad_worker: None,
|
||||
ten_vad_worker: None,
|
||||
current_vad_backend: crate::VadBackend::WebrtcVad,
|
||||
silero_model_epoch: crate::vad::silero_model_epoch(),
|
||||
ten_model_epoch: crate::vad::ten_model_epoch(),
|
||||
capture_frame_seq: 0,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||||
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
|
||||
@@ -218,6 +207,10 @@ impl AndroidCaptureState {
|
||||
audio_processing_config,
|
||||
audio_processing_stats,
|
||||
render_reference,
|
||||
input_sample_rate_hz: input_sample_rate_hz.max(1),
|
||||
resample_pos: 0.0,
|
||||
resample_last: 0,
|
||||
resample_scratch: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
|
||||
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_10ms_len: 0,
|
||||
fallback_warned_backend: None,
|
||||
@@ -227,6 +220,17 @@ impl AndroidCaptureState {
|
||||
/// Consume i16 mono frames from Oboe. Accumulate to 10 ms chunks,
|
||||
/// process each through WebRTC APM + VAD, then encode 20 ms frames.
|
||||
fn ingest_i16(&mut self, samples: &[i16]) {
|
||||
self.audio_processing_stats
|
||||
.record_callback_frames(samples.len() as u64);
|
||||
if self.input_sample_rate_hz != crate::frame::SAMPLE_RATE_HZ {
|
||||
let resampled = self.resample_capture_to_48k(samples);
|
||||
self.ingest_48k_i16(&resampled);
|
||||
return;
|
||||
}
|
||||
self.ingest_48k_i16(samples);
|
||||
}
|
||||
|
||||
fn ingest_48k_i16(&mut self, samples: &[i16]) {
|
||||
let mut offset = 0;
|
||||
while offset < samples.len() {
|
||||
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
|
||||
@@ -284,6 +288,41 @@ impl AndroidCaptureState {
|
||||
}
|
||||
}
|
||||
|
||||
fn resample_capture_to_48k(&mut self, samples: &[i16]) -> Vec<i16> {
|
||||
if samples.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
self.resample_scratch.clear();
|
||||
let ratio = self.input_sample_rate_hz as f64 / crate::frame::SAMPLE_RATE_HZ as f64;
|
||||
let mut pos = self.resample_pos;
|
||||
while pos < samples.len() as f64 {
|
||||
let i = pos.floor() as isize;
|
||||
let frac = pos - i as f64;
|
||||
let a = if i <= 0 {
|
||||
self.resample_last as f64
|
||||
} else {
|
||||
samples[(i - 1) as usize] as f64
|
||||
};
|
||||
let b = if i < samples.len() as isize {
|
||||
samples[i as usize] as f64
|
||||
} else {
|
||||
a
|
||||
};
|
||||
let value = (a + frac * (b - a))
|
||||
.round()
|
||||
.clamp(i16::MIN as f64, i16::MAX as f64) as i16;
|
||||
self.resample_scratch.push(value);
|
||||
pos += ratio;
|
||||
}
|
||||
self.resample_pos = pos - samples.len() as f64;
|
||||
self.resample_last = *samples.last().unwrap_or(&self.resample_last);
|
||||
self.resample_scratch.clone()
|
||||
}
|
||||
|
||||
fn set_input_sample_rate_hz(&mut self, sample_rate_hz: u32) {
|
||||
self.input_sample_rate_hz = sample_rate_hz.max(1);
|
||||
}
|
||||
|
||||
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
|
||||
if self.fallback_warned_backend != Some(failed_backend) {
|
||||
self.fallback_warned_backend = Some(failed_backend);
|
||||
@@ -317,6 +356,18 @@ impl AndroidCaptureState {
|
||||
self.webrtc_apm_processor.process_render(&render_ref);
|
||||
self.webrtc_apm_processor.process_capture(&mut frame);
|
||||
|
||||
let voice_activity_mode = self
|
||||
.voice_activity_selector
|
||||
.as_ref()
|
||||
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
||||
.unwrap_or(false);
|
||||
if !voice_activity_mode {
|
||||
self.silero_vad_worker = None;
|
||||
self.current_vad_backend = crate::VadBackend::Disabled;
|
||||
self.fallback_warned_backend = None;
|
||||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||
}
|
||||
|
||||
let (vad_hangover, vad_backend) = self
|
||||
.audio_processing_config
|
||||
.try_lock()
|
||||
@@ -325,30 +376,28 @@ impl AndroidCaptureState {
|
||||
crate::voice_activity::VAD_HANGOVER_MS,
|
||||
crate::VadBackend::WebrtcVad,
|
||||
));
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
if voice_activity_mode {
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
}
|
||||
|
||||
// VAD backend switching (mirrors iOS Raw path).
|
||||
// VAD backend switching only while VoiceActivity mode is active.
|
||||
let silero_epoch = crate::vad::silero_model_epoch();
|
||||
let ten_epoch = crate::vad::ten_model_epoch();
|
||||
let silero_changed =
|
||||
vad_backend == crate::VadBackend::SileroOnnx && silero_epoch != self.silero_model_epoch;
|
||||
let ten_changed =
|
||||
vad_backend == crate::VadBackend::TenVad && ten_epoch != self.ten_model_epoch;
|
||||
if vad_backend != self.current_vad_backend || silero_changed || ten_changed {
|
||||
let silero_changed = voice_activity_mode
|
||||
&& vad_backend == crate::VadBackend::SileroOnnx
|
||||
&& silero_epoch != self.silero_model_epoch;
|
||||
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_changed) {
|
||||
self.current_vad_backend = vad_backend;
|
||||
self.silero_model_epoch = silero_epoch;
|
||||
self.ten_model_epoch = ten_epoch;
|
||||
self.fallback_warned_backend = None;
|
||||
match vad_backend {
|
||||
crate::VadBackend::SileroOnnx => {
|
||||
let path = crate::vad::silero_model_bundle_path();
|
||||
self.silero_vad_worker =
|
||||
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
|
||||
self.ten_vad_worker = None;
|
||||
if self.silero_vad_worker.is_none() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
@@ -356,48 +405,43 @@ impl AndroidCaptureState {
|
||||
);
|
||||
}
|
||||
}
|
||||
crate::VadBackend::TenVad => {
|
||||
let path = crate::vad::ten_model_bundle_path();
|
||||
self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path);
|
||||
self.silero_vad_worker = None;
|
||||
if self.ten_vad_worker.is_none() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android: TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD"
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
self.silero_vad_worker = None;
|
||||
self.ten_vad_worker = None;
|
||||
}
|
||||
}
|
||||
self.vad_state.reset();
|
||||
}
|
||||
|
||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||
let capture_seq = self.capture_frame_seq;
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let p = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability: p,
|
||||
speech: p >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
// Warm-up: worker dispatched but hasn't finished yet.
|
||||
// Default to no-speech until first result arrives (~100ms).
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
let (vad_probability, active) = if voice_activity_mode {
|
||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||
let capture_seq = self.capture_frame_seq;
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let p = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability: p,
|
||||
speech: p >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(
|
||||
&mut self.vad_detector,
|
||||
&frame,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
@@ -405,53 +449,28 @@ impl AndroidCaptureState {
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::TenVad {
|
||||
if let Some(worker) = self.ten_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let p = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability: p,
|
||||
speech: p >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
(vad.probability, self.vad_state.update(vad.speech))
|
||||
} else {
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
(0.0, false)
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
let active = self.vad_state.update(vad.speech);
|
||||
let output_muted = self.output_muted.load(Ordering::Relaxed);
|
||||
if let Some(sel) = &self.voice_activity_selector {
|
||||
sel.set_voice_activity_open(active && !output_muted);
|
||||
sel.set_voice_activity_open(voice_activity_mode && active);
|
||||
}
|
||||
self.audio_processing_stats.update_capture(
|
||||
input_dbfs,
|
||||
crate::frame::dbfs(&frame),
|
||||
vad.probability,
|
||||
active && !output_muted,
|
||||
vad_probability,
|
||||
voice_activity_mode && active,
|
||||
self.transmit_active.load(Ordering::Relaxed),
|
||||
);
|
||||
self.audio_processing_stats
|
||||
.record_capture_frame(frame.iter().all(|sample| sample.abs() < 1.0e-6));
|
||||
|
||||
if !self.transmit_active.load(Ordering::Relaxed) || output_muted {
|
||||
if !self.transmit_active.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -511,6 +530,9 @@ struct OutputCallback {
|
||||
event_tx: BackendEventTx,
|
||||
scratch: Arc<Mutex<Vec<f32>>>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
pending_render_ref: [f32; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_render_ref_len: usize,
|
||||
}
|
||||
|
||||
impl AudioOutputCallback for OutputCallback {
|
||||
@@ -552,19 +574,20 @@ impl AudioOutputCallback for OutputCallback {
|
||||
gain,
|
||||
muted,
|
||||
);
|
||||
self.audio_processing_stats
|
||||
.update_render(crate::frame::dbfs(&scratch[..needed]), frames.len() as u32);
|
||||
|
||||
// Write the first 10 ms of render audio into the reference
|
||||
// buffer for the capture-side AEC.
|
||||
let mono_n = needed / 2;
|
||||
let render_n = mono_n.min(crate::frame::FRAME_10MS_SAMPLES);
|
||||
let mut ref_frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||||
for (i, chunk) in scratch[..render_n * 2].chunks_exact(2).enumerate() {
|
||||
if i >= render_n {
|
||||
break;
|
||||
// Accumulate the full render callback into 10 ms mono chunks so
|
||||
// AEC sees consistent reference timing even when output callbacks
|
||||
// are shorter or longer than 10 ms.
|
||||
for chunk in scratch[..needed].chunks_exact(2) {
|
||||
self.pending_render_ref[self.pending_render_ref_len] = (chunk[0] + chunk[1]) * 0.5;
|
||||
self.pending_render_ref_len += 1;
|
||||
if self.pending_render_ref_len == crate::frame::FRAME_10MS_SAMPLES {
|
||||
self.render_reference.write(&self.pending_render_ref);
|
||||
self.pending_render_ref_len = 0;
|
||||
}
|
||||
ref_frame[i] = (chunk[0] + chunk[1]) * 0.5;
|
||||
}
|
||||
self.render_reference.write(&ref_frame);
|
||||
}));
|
||||
DataCallbackResult::Continue
|
||||
}
|
||||
@@ -640,6 +663,7 @@ impl AndroidVoiceUnit {
|
||||
// Clone the APM config Arc before params is partially moved
|
||||
// into the capture state constructor below.
|
||||
let apm_config_clone = params.audio_processing_config.clone();
|
||||
let audio_processing_stats = params.audio_processing_stats.clone();
|
||||
|
||||
let capture_state = Arc::new(Mutex::new(
|
||||
AndroidCaptureState::new(
|
||||
@@ -650,8 +674,9 @@ impl AndroidVoiceUnit {
|
||||
params.mic_gain,
|
||||
params.voice_activity_selector,
|
||||
params.audio_processing_config,
|
||||
params.audio_processing_stats,
|
||||
audio_processing_stats.clone(),
|
||||
render_ref_for_capture,
|
||||
cfg.sample_rate,
|
||||
)
|
||||
.map_err(|e| BackendError::OpenFailed(format!("capture state init: {e}")))?,
|
||||
));
|
||||
@@ -768,6 +793,9 @@ impl AndroidVoiceUnit {
|
||||
event_tx: event_tx.clone(),
|
||||
scratch: scratch.clone(),
|
||||
render_reference: render_ref_for_output,
|
||||
audio_processing_stats: audio_processing_stats.clone(),
|
||||
pending_render_ref: [0.0_f32; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_render_ref_len: 0,
|
||||
};
|
||||
let output_builder = output_builder.set_callback(output_cb);
|
||||
|
||||
@@ -785,6 +813,7 @@ impl AndroidVoiceUnit {
|
||||
params.handler.clone(),
|
||||
params.output_gain.clone(),
|
||||
params.output_muted.clone(),
|
||||
audio_processing_stats.clone(),
|
||||
scratch.clone(),
|
||||
render_ref_buf,
|
||||
)?
|
||||
@@ -869,6 +898,14 @@ impl AndroidVoiceUnit {
|
||||
"android: audio processing config resolved (hardware effects: aec={hw_aec} ns={hw_ns} agc={hw_agc})"
|
||||
);
|
||||
}
|
||||
if let Ok(mut capture) = capture_state.lock() {
|
||||
capture.set_input_sample_rate_hz(input_sample_rate.max(1) as u32);
|
||||
let resolved_cfg = apm_config_clone
|
||||
.lock()
|
||||
.map(|cfg| webrtc_apm_config_from_audio_config(&cfg))
|
||||
.unwrap_or_default();
|
||||
capture.webrtc_apm_processor.apply_config(resolved_cfg);
|
||||
}
|
||||
|
||||
// --- SDD-112 item 10 / SDD-113 item 7 / SDD-116 item 3 ---
|
||||
// Publish the diagnostics snapshot. Per-effect engagement is
|
||||
@@ -910,6 +947,9 @@ impl AndroidVoiceUnit {
|
||||
latency_tier: latency_tier_for(input_perf),
|
||||
};
|
||||
publish_android_audio_diagnostics(diagnostics);
|
||||
params
|
||||
.audio_processing_stats
|
||||
.set_actual_sample_rate_hz(input_sample_rate.max(0) as u32);
|
||||
|
||||
Ok(Self {
|
||||
input: input_stream,
|
||||
@@ -1005,6 +1045,7 @@ impl AndroidVoiceUnit {
|
||||
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||||
output_gain: Arc<AtomicU32>,
|
||||
output_muted: Arc<AtomicBool>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
scratch: Arc<Mutex<Vec<f32>>>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
) -> Result<AudioStreamAsync<OboeOutput, OutputCallback>, BackendError> {
|
||||
@@ -1015,6 +1056,9 @@ impl AndroidVoiceUnit {
|
||||
event_tx: event_tx.clone(),
|
||||
scratch,
|
||||
render_reference,
|
||||
audio_processing_stats,
|
||||
pending_render_ref: [0.0_f32; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_render_ref_len: 0,
|
||||
};
|
||||
let builder = AudioStreamBuilder::default()
|
||||
.set_direction::<OboeOutput>()
|
||||
@@ -1038,6 +1082,12 @@ impl AndroidVoiceUnit {
|
||||
}
|
||||
}
|
||||
|
||||
fn webrtc_apm_config_from_audio_config(
|
||||
config: &crate::AudioProcessingConfig,
|
||||
) -> crate::processor::webrtc_apm::WebRtcApmConfig {
|
||||
crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(config)
|
||||
}
|
||||
|
||||
impl MobileVoiceAudioBackend for AndroidVoiceUnit {
|
||||
fn start(&mut self) -> Result<(), BackendError> {
|
||||
if let Some(s) = self.input.as_mut() {
|
||||
|
||||
@@ -92,9 +92,6 @@ impl AudioBackend {
|
||||
pub enum VadBackend {
|
||||
/// Silero ONNX VAD. P1 schema default when model/runtime exist.
|
||||
SileroOnnx,
|
||||
/// TEN VAD backend. Native TEN runtime is optional; unavailable
|
||||
/// builds fall back to the realtime-safe WebRTC detector.
|
||||
TenVad,
|
||||
/// WebRTC-style fallback VAD.
|
||||
WebrtcVad,
|
||||
/// Debug-only energy VAD.
|
||||
@@ -108,7 +105,6 @@ impl VadBackend {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::SileroOnnx => "silero_vad_onnx",
|
||||
Self::TenVad => "ten_vad",
|
||||
Self::WebrtcVad => "webrtc_vad",
|
||||
Self::EnergyDebug => "energy_debug",
|
||||
Self::Disabled => "disabled",
|
||||
@@ -168,7 +164,7 @@ impl Default for AudioProcessingConfig {
|
||||
route: AudioRoute::Speaker,
|
||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
||||
vad_backend: VadBackend::TenVad,
|
||||
vad_backend: VadBackend::SileroOnnx,
|
||||
aec: EffectOwner::Platform,
|
||||
// iOS VPIO owns NS/AGC on the default shipping path. Software
|
||||
// effects are opt-in through the experimental raw route only.
|
||||
@@ -246,7 +242,7 @@ mod tests {
|
||||
assert_eq!(config.aec, EffectOwner::Platform);
|
||||
assert_eq!(config.ns, EffectOwner::Platform);
|
||||
assert_eq!(config.agc, EffectOwner::Platform);
|
||||
assert_eq!(config.vad_backend, VadBackend::TenVad);
|
||||
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -259,11 +255,6 @@ mod tests {
|
||||
assert!(config.validate_for_ios().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ten_vad_has_stable_debug_string() {
|
||||
assert_eq!(VadBackend::TenVad.as_str(), "ten_vad");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_processing_allows_full_webrtc_apm_chain() {
|
||||
let config = AudioProcessingConfig {
|
||||
@@ -342,6 +333,16 @@ pub struct AudioProcessingStats {
|
||||
pub callback_xruns: u64,
|
||||
/// Clipped sample count.
|
||||
pub clipped_samples: u64,
|
||||
/// Number of effectively silent processed capture frames.
|
||||
pub zero_frames: u64,
|
||||
/// Number of processed capture frames.
|
||||
pub capture_frames: u64,
|
||||
/// Number of input callbacks carrying 10 ms of audio.
|
||||
pub callbacks_10ms: u64,
|
||||
/// Number of input callbacks carrying 20 ms of audio.
|
||||
pub callbacks_20ms: u64,
|
||||
/// Number of input callbacks carrying any other size.
|
||||
pub callbacks_other: u64,
|
||||
/// Sonora enabled.
|
||||
pub sonora_enabled: bool,
|
||||
/// Platform voice processing enabled.
|
||||
@@ -361,6 +362,11 @@ pub struct SharedAudioProcessingStats {
|
||||
output_underruns: AtomicU64,
|
||||
callback_xruns: AtomicU64,
|
||||
clipped_samples: AtomicU64,
|
||||
zero_frames: AtomicU64,
|
||||
capture_frames: AtomicU64,
|
||||
callbacks_10ms: AtomicU64,
|
||||
callbacks_20ms: AtomicU64,
|
||||
callbacks_other: AtomicU64,
|
||||
actual_sample_rate_hz: AtomicU32,
|
||||
actual_io_buffer_frames: AtomicU32,
|
||||
}
|
||||
@@ -379,6 +385,11 @@ impl Default for SharedAudioProcessingStats {
|
||||
output_underruns: AtomicU64::new(0),
|
||||
callback_xruns: AtomicU64::new(0),
|
||||
clipped_samples: AtomicU64::new(0),
|
||||
zero_frames: AtomicU64::new(0),
|
||||
capture_frames: AtomicU64::new(0),
|
||||
callbacks_10ms: AtomicU64::new(0),
|
||||
callbacks_20ms: AtomicU64::new(0),
|
||||
callbacks_other: AtomicU64::new(0),
|
||||
actual_sample_rate_hz: AtomicU32::new(crate::frame::SAMPLE_RATE_HZ),
|
||||
actual_io_buffer_frames: AtomicU32::new(crate::frame::FRAME_20MS_SAMPLES as u32),
|
||||
}
|
||||
@@ -412,6 +423,12 @@ impl SharedAudioProcessingStats {
|
||||
.store(io_buffer_frames, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Record the actual device sample rate.
|
||||
pub fn set_actual_sample_rate_hz(&self, sample_rate_hz: u32) {
|
||||
self.actual_sample_rate_hz
|
||||
.store(sample_rate_hz, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Increment output underrun count.
|
||||
pub fn increment_output_underrun(&self) {
|
||||
self.output_underruns.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -427,6 +444,32 @@ impl SharedAudioProcessingStats {
|
||||
self.clipped_samples.fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Record one processed capture frame and whether it was effectively silent.
|
||||
pub fn record_capture_frame(&self, zero_frame: bool) {
|
||||
self.capture_frames.fetch_add(1, Ordering::Relaxed);
|
||||
if zero_frame {
|
||||
self.zero_frames.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Bucket callback delivery sizes to diagnose timing jitter and packetization.
|
||||
pub fn record_callback_frames(&self, frames: u64) {
|
||||
let sample_rate_hz = self.actual_sample_rate_hz.load(Ordering::Relaxed).max(1);
|
||||
let frames_10ms = (sample_rate_hz / 100) as u64;
|
||||
let frames_20ms = (sample_rate_hz / 50) as u64;
|
||||
match frames {
|
||||
value if value == frames_10ms => {
|
||||
self.callbacks_10ms.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
value if value == frames_20ms => {
|
||||
self.callbacks_20ms.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
_ => {
|
||||
self.callbacks_other.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Store whether the selected VAD backend is currently using a fallback.
|
||||
pub fn set_vad_fallback_active(&self, active: bool) {
|
||||
self.vad_fallback_active.store(active, Ordering::Relaxed);
|
||||
@@ -452,6 +495,11 @@ impl SharedAudioProcessingStats {
|
||||
output_underruns: self.output_underruns.load(Ordering::Relaxed),
|
||||
callback_xruns: self.callback_xruns.load(Ordering::Relaxed),
|
||||
clipped_samples: self.clipped_samples.load(Ordering::Relaxed),
|
||||
zero_frames: self.zero_frames.load(Ordering::Relaxed),
|
||||
capture_frames: self.capture_frames.load(Ordering::Relaxed),
|
||||
callbacks_10ms: self.callbacks_10ms.load(Ordering::Relaxed),
|
||||
callbacks_20ms: self.callbacks_20ms.load(Ordering::Relaxed),
|
||||
callbacks_other: self.callbacks_other.load(Ordering::Relaxed),
|
||||
sonora_enabled: config.processing_backend == AudioBackend::Sonora,
|
||||
platform_voice_processing_enabled: config.processing_backend
|
||||
== AudioBackend::PlatformVoiceProcessing,
|
||||
|
||||
+435
-101
@@ -20,6 +20,18 @@ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||||
not(target_os = "android")
|
||||
))]
|
||||
use cpal::{SampleFormat, SizedSample};
|
||||
#[cfg(all(
|
||||
not(target_os = "ios"),
|
||||
not(target_os = "macos"),
|
||||
not(target_os = "android")
|
||||
))]
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
#[cfg(all(
|
||||
not(target_os = "ios"),
|
||||
not(target_os = "macos"),
|
||||
not(target_os = "android")
|
||||
))]
|
||||
use std::hash::{Hash, Hasher};
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info};
|
||||
// `error!` and `warn!` are used only inside the cpal capture /
|
||||
@@ -80,12 +92,77 @@ pub struct AudioDeviceList {
|
||||
/// Info about a single audio device.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AudioDeviceInfo {
|
||||
/// Stable platform-reported device identifier.
|
||||
pub id: String,
|
||||
/// Human-readable device name from the OS.
|
||||
pub name: String,
|
||||
/// Additional device details useful for disambiguation.
|
||||
pub details: String,
|
||||
/// True if the OS reports this as the default device.
|
||||
pub is_default: bool,
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
not(target_os = "ios"),
|
||||
not(target_os = "macos"),
|
||||
not(target_os = "android")
|
||||
))]
|
||||
fn desktop_device_id(device: &cpal::Device) -> Option<String> {
|
||||
device.id().ok().map(|id| id.to_string())
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
not(target_os = "ios"),
|
||||
not(target_os = "macos"),
|
||||
not(target_os = "android")
|
||||
))]
|
||||
fn short_device_id(id: &str) -> String {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
id.hash(&mut hasher);
|
||||
format!("{:016x}", hasher.finish())
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
not(target_os = "ios"),
|
||||
not(target_os = "macos"),
|
||||
not(target_os = "android")
|
||||
))]
|
||||
fn describe_device(device: &cpal::Device) -> Option<AudioDeviceInfo> {
|
||||
let id = desktop_device_id(device)?;
|
||||
let description = device.description().ok();
|
||||
let name = description
|
||||
.as_ref()
|
||||
.map(|d| d.name().trim().to_owned())
|
||||
.filter(|name| !name.is_empty())
|
||||
.unwrap_or_else(|| format!("Device {}", &short_device_id(&id)[..8]));
|
||||
|
||||
let mut details = Vec::new();
|
||||
if let Some(description) = description.as_ref() {
|
||||
if let Some(manufacturer) = description.manufacturer() {
|
||||
details.push(manufacturer.to_owned());
|
||||
}
|
||||
if let Some(driver) = description.driver() {
|
||||
details.push(driver.to_owned());
|
||||
}
|
||||
let device_type = description.device_type();
|
||||
if device_type != cpal::device_description::DeviceType::Unknown {
|
||||
details.push(format!("{device_type}"));
|
||||
}
|
||||
let interface_type = description.interface_type();
|
||||
if interface_type != cpal::device_description::InterfaceType::Unknown {
|
||||
details.push(format!("{interface_type}"));
|
||||
}
|
||||
}
|
||||
details.push(format!("id={}", short_device_id(&id)));
|
||||
|
||||
Some(AudioDeviceInfo {
|
||||
id,
|
||||
name,
|
||||
details: details.join(" · "),
|
||||
is_default: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Enumerate available audio input and output devices.
|
||||
/// On mobile platforms (iOS, Android) returns an empty list because
|
||||
/// device selection is managed by the OS audio session.
|
||||
@@ -100,38 +177,30 @@ pub fn list_audio_devices() -> AudioDeviceList {
|
||||
input_devices: Vec::new(),
|
||||
output_devices: Vec::new(),
|
||||
};
|
||||
let Ok(host) = cpal::default_host() else {
|
||||
return list;
|
||||
};
|
||||
let default_in = host.default_input_device();
|
||||
let default_out = host.default_output_device();
|
||||
let host = cpal::default_host();
|
||||
let default_in = host
|
||||
.default_input_device()
|
||||
.and_then(|device| desktop_device_id(&device));
|
||||
let default_out = host
|
||||
.default_output_device()
|
||||
.and_then(|device| desktop_device_id(&device));
|
||||
if let Ok(devices) = host.input_devices() {
|
||||
for d in devices {
|
||||
let name = d
|
||||
.description()
|
||||
.map(|n| n.name().to_owned())
|
||||
.unwrap_or_default();
|
||||
if !name.is_empty() {
|
||||
let is_default = default_in
|
||||
if let Some(mut device) = describe_device(&d) {
|
||||
device.is_default = default_in
|
||||
.as_ref()
|
||||
.is_some_and(|di| di.description().is_ok_and(|dn| dn.name() == name.as_str()));
|
||||
list.input_devices
|
||||
.push(AudioDeviceInfo { name, is_default });
|
||||
.is_some_and(|default_id| default_id == &device.id);
|
||||
list.input_devices.push(device);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Ok(devices) = host.output_devices() {
|
||||
for d in devices {
|
||||
let name = d
|
||||
.description()
|
||||
.map(|n| n.name().to_owned())
|
||||
.unwrap_or_default();
|
||||
if !name.is_empty() {
|
||||
let is_default = default_out
|
||||
if let Some(mut device) = describe_device(&d) {
|
||||
device.is_default = default_out
|
||||
.as_ref()
|
||||
.is_some_and(|di| di.description().is_ok_and(|dn| dn.name() == name.as_str()));
|
||||
list.output_devices
|
||||
.push(AudioDeviceInfo { name, is_default });
|
||||
.is_some_and(|default_id| default_id == &device.id);
|
||||
list.output_devices.push(device);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -172,12 +241,12 @@ pub struct AudioEngineConfig {
|
||||
/// is rejected on Android because the P0 path intentionally has
|
||||
/// no generic mobile-audio fallback.
|
||||
pub mobile_voice_preset: bool,
|
||||
/// Optional input device name override. When `None`, the system
|
||||
/// default input device is used. Set to a device name from
|
||||
/// Optional input device id override. When `None`, the system
|
||||
/// default input device is used. Set to a device id from
|
||||
/// [`list_audio_devices`] to pin a specific microphone.
|
||||
pub input_device_name: Option<String>,
|
||||
/// Optional output device name override.
|
||||
pub output_device_name: Option<String>,
|
||||
pub input_device_id: Option<String>,
|
||||
/// Optional output device id override.
|
||||
pub output_device_id: Option<String>,
|
||||
/// Optional selector used by P1 VoiceActivity to publish VAD state.
|
||||
#[doc(hidden)]
|
||||
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
@@ -190,8 +259,8 @@ impl std::fmt::Debug for AudioEngineConfig {
|
||||
.field("ptt_initial", &self.ptt_initial)
|
||||
.field("effects", &self.effects)
|
||||
.field("mobile_voice_preset", &self.mobile_voice_preset)
|
||||
.field("input_device_name", &self.input_device_name)
|
||||
.field("output_device_name", &self.output_device_name)
|
||||
.field("input_device_id", &self.input_device_id)
|
||||
.field("output_device_id", &self.output_device_id)
|
||||
.field(
|
||||
"voice_activity_selector",
|
||||
&self.voice_activity_selector.as_ref().map(|_| "present"),
|
||||
@@ -207,8 +276,8 @@ impl Default for AudioEngineConfig {
|
||||
ptt_initial: false,
|
||||
effects: crate::AudioEffects::default(),
|
||||
mobile_voice_preset: true,
|
||||
input_device_name: None,
|
||||
output_device_name: None,
|
||||
input_device_id: None,
|
||||
output_device_id: None,
|
||||
voice_activity_selector: None,
|
||||
}
|
||||
}
|
||||
@@ -237,13 +306,13 @@ pub struct AudioEngine {
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
|
||||
mic_gain: f32,
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
// Streams must be dropped to stop audio. Both are `!Send` because
|
||||
// cpal's Stream isn't Send on some backends; we keep them in an
|
||||
// Option wrapped by Mutex so stop() can move them out. On Linux
|
||||
@@ -277,7 +346,11 @@ pub struct AudioEngine {
|
||||
/// unbinds effects and stops the service in SDD-115 reverse
|
||||
/// order.
|
||||
#[cfg(target_os = "android")]
|
||||
_android_voice_unit: Mutex<Option<crate::android_voice_unit::AndroidVoiceUnit>>,
|
||||
_android_voice_unit: Arc<Mutex<Option<crate::android_voice_unit::AndroidVoiceUnit>>>,
|
||||
/// Persist the Android stream request so route-change reopen uses the
|
||||
/// same effect and latency policy as the original session start.
|
||||
#[cfg(target_os = "android")]
|
||||
android_voice_stream_config: Arc<Mutex<crate::mobile_voice_backend::AndroidVoiceStreamConfig>>,
|
||||
/// SDD-108 §1/§2: refcount-composable audio-mode controller.
|
||||
/// Snapshots `AudioManager.getMode()` on the 0 → 1 transition and
|
||||
/// restores it on the 1 → 0 transition. Held in a `Mutex` so the
|
||||
@@ -286,7 +359,7 @@ pub struct AudioEngine {
|
||||
/// mode lifecycle is bound to the voice-session lifecycle
|
||||
/// (SDD-108 §3).
|
||||
#[cfg(target_os = "android")]
|
||||
audio_mode_stack: Mutex<crate::mode_stack::ModeStack>,
|
||||
audio_mode_stack: Arc<Mutex<crate::mode_stack::ModeStack>>,
|
||||
// Hand the inbound-voice forwarder task a shutdown signal.
|
||||
shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
/// True if the capture stream actually opened. If false (typical
|
||||
@@ -385,6 +458,183 @@ fn open_ios_voice_backend(
|
||||
}
|
||||
|
||||
impl AudioEngine {
|
||||
#[cfg(target_os = "android")]
|
||||
fn spawn_android_backend_event_task(
|
||||
android_voice_unit: Arc<Mutex<Option<crate::android_voice_unit::AndroidVoiceUnit>>>,
|
||||
android_voice_stream_config: Arc<
|
||||
Mutex<crate::mobile_voice_backend::AndroidVoiceStreamConfig>,
|
||||
>,
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||||
output_gain: Arc<AtomicU32>,
|
||||
output_muted: Arc<AtomicBool>,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
mic_gain: f32,
|
||||
audio_mode_stack: Arc<Mutex<crate::mode_stack::ModeStack>>,
|
||||
mut event_rx: crate::mobile_voice_backend::BackendEventRx,
|
||||
) {
|
||||
use crate::mobile_voice_backend::BackendEvent;
|
||||
|
||||
tokio::spawn(async move {
|
||||
while let Some(event) = event_rx.recv().await {
|
||||
match event {
|
||||
BackendEvent::Disconnected => {
|
||||
warn!(target: "chanora_audio", "android: backend disconnected; reopening voice unit");
|
||||
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
|
||||
crate::android_voice_unit::chanora_android_abandon_audio_focus();
|
||||
crate::android_voice_unit::clear_global_event_sender();
|
||||
|
||||
if let Some(mut unit) = android_voice_unit.lock().unwrap().take() {
|
||||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||||
if let Err(e) = unit.close() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
error = %e,
|
||||
"android: failed to close disconnected voice unit before reopen"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if crate::android_voice_unit::chanora_android_start_voice_service() {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: voice foreground service restart dispatched after backend disconnect"
|
||||
);
|
||||
}
|
||||
|
||||
match android_get_audio_mode() {
|
||||
Ok(current_mode) => {
|
||||
if current_mode != ANDROID_MODE_IN_COMMUNICATION {
|
||||
match android_set_audio_mode(ANDROID_MODE_IN_COMMUNICATION) {
|
||||
Ok(()) => info!(
|
||||
target: "chanora_audio",
|
||||
prior_mode = current_mode,
|
||||
"android: AudioManager mode re-engaged after backend disconnect"
|
||||
),
|
||||
Err(e) => warn!(
|
||||
target: "chanora_audio",
|
||||
error = %e,
|
||||
prior_mode = current_mode,
|
||||
"android: failed to re-engage MODE_IN_COMMUNICATION after backend disconnect"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(
|
||||
target: "chanora_audio",
|
||||
error = %e,
|
||||
"android: AudioManager.getMode failed during backend reopen"
|
||||
),
|
||||
}
|
||||
|
||||
let cfg_av = android_voice_stream_config.lock().unwrap().clone();
|
||||
let params = crate::mobile_voice_backend::VoiceAudioParams {
|
||||
voice_out_tx: voice_out_tx.clone(),
|
||||
transmit_active: transmit_gate.flag_arc(),
|
||||
frames_sent: frames_sent.clone(),
|
||||
mic_gain,
|
||||
handler: audio_handler.clone(),
|
||||
output_gain: output_gain.clone(),
|
||||
output_muted: output_muted.clone(),
|
||||
voice_activity_selector: voice_activity_selector.clone(),
|
||||
audio_processing_config: audio_processing_config.clone(),
|
||||
audio_processing_stats: audio_processing_stats.clone(),
|
||||
};
|
||||
|
||||
match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) {
|
||||
Ok(mut reopened) => {
|
||||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||||
match reopened.start() {
|
||||
Ok(()) => {
|
||||
if let Some(next_rx) = reopened.take_event_rx() {
|
||||
Self::spawn_android_backend_event_task(
|
||||
android_voice_unit.clone(),
|
||||
android_voice_stream_config.clone(),
|
||||
voice_out_tx.clone(),
|
||||
transmit_gate.clone(),
|
||||
frames_sent.clone(),
|
||||
audio_handler.clone(),
|
||||
output_gain.clone(),
|
||||
output_muted.clone(),
|
||||
voice_activity_selector.clone(),
|
||||
audio_processing_config.clone(),
|
||||
audio_processing_stats.clone(),
|
||||
mic_gain,
|
||||
audio_mode_stack.clone(),
|
||||
next_rx,
|
||||
);
|
||||
}
|
||||
crate::android_voice_unit::register_global_event_sender(
|
||||
reopened.event_sender(),
|
||||
);
|
||||
if crate::android_voice_unit::chanora_android_request_audio_focus() {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: audio focus re-requested after backend disconnect"
|
||||
);
|
||||
}
|
||||
if crate::android_voice_unit::chanora_android_start_bluetooth_sco() {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: bluetooth route re-engaged after backend disconnect"
|
||||
);
|
||||
}
|
||||
*android_voice_unit.lock().unwrap() = Some(reopened);
|
||||
}
|
||||
Err(e) => warn!(
|
||||
target: "chanora_audio",
|
||||
error = %e,
|
||||
"android: reopened voice unit failed to start after backend disconnect"
|
||||
),
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(
|
||||
target: "chanora_audio",
|
||||
error = %e,
|
||||
"android: failed to reopen voice unit after backend disconnect"
|
||||
),
|
||||
}
|
||||
}
|
||||
BackendEvent::FocusLost => {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android: audio focus lost permanently (SDD-109); engine should leave session"
|
||||
);
|
||||
}
|
||||
BackendEvent::FocusTransient => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: transient audio focus loss (SDD-109); pausing capture"
|
||||
);
|
||||
}
|
||||
BackendEvent::FocusTransientCanDuck => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: transient audio focus loss with ducking (SDD-109); continuing"
|
||||
);
|
||||
}
|
||||
BackendEvent::FocusGain => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: audio focus regained (SDD-109); resuming capture"
|
||||
);
|
||||
}
|
||||
BackendEvent::BluetoothScoStateChanged(s) => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
sco_state = s,
|
||||
"android: Bluetooth SCO state changed (SDD-110)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Start the engine: open capture + playback streams, spawn the
|
||||
/// inbound-voice forwarder, return a handle.
|
||||
pub fn start(
|
||||
@@ -461,21 +711,22 @@ impl AudioEngine {
|
||||
"starting audio engine: cpal host selected"
|
||||
);
|
||||
|
||||
/// Helper: find a device by name, falling back to default.
|
||||
fn find_device(
|
||||
/// Helper: find a device by stable id, falling back to default.
|
||||
fn find_device<DefaultFn, AllFn, Devices>(
|
||||
host: &cpal::Host,
|
||||
default_fn: fn(&cpal::Host) -> Option<cpal::Device>,
|
||||
all_fn: fn(&cpal::Host) -> Result<cpal::Devices, cpal::DevicesError>,
|
||||
default_fn: DefaultFn,
|
||||
all_fn: AllFn,
|
||||
prefer: Option<&str>,
|
||||
) -> Option<cpal::Device> {
|
||||
if let Some(name) = prefer {
|
||||
) -> Option<cpal::Device>
|
||||
where
|
||||
DefaultFn: Fn(&cpal::Host) -> Option<cpal::Device>,
|
||||
AllFn: Fn(&cpal::Host) -> Result<Devices, cpal::DevicesError>,
|
||||
Devices: IntoIterator<Item = cpal::Device>,
|
||||
{
|
||||
if let Some(id) = prefer {
|
||||
if let Ok(devices) = all_fn(host) {
|
||||
for d in devices {
|
||||
let dn = d
|
||||
.description()
|
||||
.map(|n| n.name().to_owned())
|
||||
.unwrap_or_default();
|
||||
if dn == name {
|
||||
if desktop_device_id(&d).as_deref() == Some(id) {
|
||||
return Some(d);
|
||||
}
|
||||
}
|
||||
@@ -488,7 +739,7 @@ impl AudioEngine {
|
||||
&host,
|
||||
cpal::Host::default_input_device,
|
||||
cpal::Host::input_devices,
|
||||
cfg.input_device_name.as_deref(),
|
||||
cfg.input_device_id.as_deref(),
|
||||
)
|
||||
.ok_or(AudioError::NoInputDevice)?;
|
||||
|
||||
@@ -496,7 +747,7 @@ impl AudioEngine {
|
||||
&host,
|
||||
cpal::Host::default_output_device,
|
||||
cpal::Host::output_devices,
|
||||
cfg.output_device_name.as_deref(),
|
||||
cfg.output_device_id.as_deref(),
|
||||
)
|
||||
.ok_or(AudioError::NoOutputDevice)?;
|
||||
|
||||
@@ -729,7 +980,7 @@ impl AudioEngine {
|
||||
mut voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||||
) -> Result<Self, AudioError> {
|
||||
use crate::mobile_voice_backend::{BackendEvent, MobileVoiceAudioBackend};
|
||||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||||
|
||||
info!(target: "chanora_audio", "starting audio engine: Android Oboe backend");
|
||||
|
||||
@@ -799,7 +1050,7 @@ impl AudioEngine {
|
||||
..Default::default()
|
||||
};
|
||||
let params = crate::mobile_voice_backend::VoiceAudioParams {
|
||||
voice_out_tx,
|
||||
voice_out_tx: voice_out_tx.clone(),
|
||||
transmit_active: transmit_flag_for_capture,
|
||||
frames_sent: frames_sent.clone(),
|
||||
mic_gain: cfg.mic_gain,
|
||||
@@ -821,53 +1072,42 @@ impl AudioEngine {
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some(mut event_rx) = android_voice_unit.take_event_rx() {
|
||||
tokio::spawn(async move {
|
||||
while let Some(event) = event_rx.recv().await {
|
||||
match event {
|
||||
BackendEvent::Disconnected => {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android: backend disconnected event received; stream reconnect requires session restart"
|
||||
);
|
||||
}
|
||||
BackendEvent::FocusLost => {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android: audio focus lost permanently (SDD-109); engine should leave session"
|
||||
);
|
||||
}
|
||||
BackendEvent::FocusTransient => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: transient audio focus loss (SDD-109); pausing capture"
|
||||
);
|
||||
}
|
||||
BackendEvent::FocusTransientCanDuck => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: transient audio focus loss with ducking (SDD-109); continuing"
|
||||
);
|
||||
}
|
||||
BackendEvent::FocusGain => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: audio focus regained (SDD-109); resuming capture"
|
||||
);
|
||||
}
|
||||
BackendEvent::BluetoothScoStateChanged(s) => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
sco_state = s,
|
||||
"android: Bluetooth SCO state changed (SDD-110)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let android_voice_unit = Arc::new(Mutex::new(Some(android_voice_unit)));
|
||||
let android_voice_stream_config = Arc::new(Mutex::new(cfg_av.clone()));
|
||||
let audio_mode_stack = Arc::new(Mutex::new(audio_mode_stack));
|
||||
|
||||
if let Some(event_rx) = android_voice_unit
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.and_then(|unit| unit.take_event_rx())
|
||||
{
|
||||
Self::spawn_android_backend_event_task(
|
||||
android_voice_unit.clone(),
|
||||
android_voice_stream_config.clone(),
|
||||
voice_out_tx.clone(),
|
||||
transmit_gate.clone(),
|
||||
frames_sent.clone(),
|
||||
audio_handler.clone(),
|
||||
output_gain.clone(),
|
||||
output_muted.clone(),
|
||||
cfg.voice_activity_selector.clone(),
|
||||
audio_processing_config.clone(),
|
||||
audio_processing_stats.clone(),
|
||||
cfg.mic_gain,
|
||||
audio_mode_stack.clone(),
|
||||
event_rx,
|
||||
);
|
||||
}
|
||||
|
||||
crate::android_voice_unit::register_global_event_sender(android_voice_unit.event_sender());
|
||||
crate::android_voice_unit::register_global_event_sender(
|
||||
android_voice_unit
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.expect("android voice unit installed")
|
||||
.event_sender(),
|
||||
);
|
||||
|
||||
if crate::android_voice_unit::chanora_android_request_audio_focus() {
|
||||
info!(
|
||||
@@ -932,8 +1172,12 @@ impl AudioEngine {
|
||||
audio_processing_config,
|
||||
audio_processing_stats,
|
||||
audio_handler,
|
||||
_android_voice_unit: Mutex::new(Some(android_voice_unit)),
|
||||
audio_mode_stack: Mutex::new(audio_mode_stack),
|
||||
voice_out_tx,
|
||||
voice_activity_selector: cfg.voice_activity_selector.clone(),
|
||||
mic_gain: cfg.mic_gain,
|
||||
_android_voice_unit: android_voice_unit,
|
||||
android_voice_stream_config,
|
||||
audio_mode_stack,
|
||||
shutdown_tx: Some(shutdown_tx),
|
||||
capture_active,
|
||||
})
|
||||
@@ -1182,6 +1426,96 @@ impl AudioEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Android-only: reopen the underlying Oboe voice backend after a
|
||||
/// route/device change while preserving the engine-owned state.
|
||||
pub fn android_restart_voice_unit(&self) -> Result<(), AudioError> {
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||||
|
||||
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
|
||||
crate::android_voice_unit::chanora_android_abandon_audio_focus();
|
||||
crate::android_voice_unit::clear_global_event_sender();
|
||||
|
||||
if let Some(mut unit) = self._android_voice_unit.lock().unwrap().take() {
|
||||
unit.close().map_err(|e| {
|
||||
AudioError::Backend(format!(
|
||||
"android: failed to close Oboe voice unit during route restart: {e}"
|
||||
))
|
||||
})?;
|
||||
}
|
||||
|
||||
if crate::android_voice_unit::chanora_android_start_voice_service() {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: voice foreground service restart dispatched after route change"
|
||||
);
|
||||
}
|
||||
|
||||
if crate::android_voice_unit::chanora_android_request_audio_focus() {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: audio focus re-requested after route change"
|
||||
);
|
||||
}
|
||||
|
||||
let cfg_av = self.android_voice_stream_config.lock().unwrap().clone();
|
||||
let params = crate::mobile_voice_backend::VoiceAudioParams {
|
||||
voice_out_tx: self.voice_out_tx.clone(),
|
||||
transmit_active: self.transmit_gate.flag_arc(),
|
||||
frames_sent: self.frames_sent.clone(),
|
||||
mic_gain: self.mic_gain,
|
||||
handler: self.audio_handler.clone(),
|
||||
output_gain: self.output_gain.clone(),
|
||||
output_muted: self.output_muted.clone(),
|
||||
voice_activity_selector: self.voice_activity_selector.clone(),
|
||||
audio_processing_config: self.audio_processing_config.clone(),
|
||||
audio_processing_stats: self.audio_processing_stats.clone(),
|
||||
};
|
||||
let mut reopened = crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params)
|
||||
.map_err(|e| {
|
||||
AudioError::Backend(format!("android: failed to reopen Oboe voice unit: {e}"))
|
||||
})?;
|
||||
reopened.start().map_err(|e| {
|
||||
AudioError::Backend(format!("android: failed to restart Oboe voice unit: {e}"))
|
||||
})?;
|
||||
if let Some(event_rx) = reopened.take_event_rx() {
|
||||
Self::spawn_android_backend_event_task(
|
||||
self._android_voice_unit.clone(),
|
||||
self.android_voice_stream_config.clone(),
|
||||
self.voice_out_tx.clone(),
|
||||
self.transmit_gate.clone(),
|
||||
self.frames_sent.clone(),
|
||||
self.audio_handler.clone(),
|
||||
self.output_gain.clone(),
|
||||
self.output_muted.clone(),
|
||||
self.voice_activity_selector.clone(),
|
||||
self.audio_processing_config.clone(),
|
||||
self.audio_processing_stats.clone(),
|
||||
self.mic_gain,
|
||||
self.audio_mode_stack.clone(),
|
||||
event_rx,
|
||||
);
|
||||
}
|
||||
crate::android_voice_unit::register_global_event_sender(reopened.event_sender());
|
||||
|
||||
if crate::android_voice_unit::chanora_android_start_bluetooth_sco() {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
"android: bluetooth route re-engaged after route change"
|
||||
);
|
||||
}
|
||||
|
||||
let mut guard = self._android_voice_unit.lock().unwrap();
|
||||
*guard = Some(reopened);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(not(target_os = "android"))]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// iOS-only: pause the underlying VoiceProcessingIO unit.
|
||||
pub fn ios_pause_voice_unit(&self) -> Result<(), AudioError> {
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
@@ -2086,7 +2420,7 @@ pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> {
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||||
#[doc(hidden)]
|
||||
pub mod bench_seam {
|
||||
use super::{Arc, AtomicBool, AtomicU32, CaptureState, OpusEncoder, OutPacket};
|
||||
use super::{Arc, AtomicBool, AtomicU32, CaptureState, OutPacket};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
/// Opaque handle wrapping a CaptureState plus the dummy mpsc
|
||||
|
||||
@@ -117,10 +117,8 @@ mod inner {
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad,
|
||||
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
|
||||
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
|
||||
current_vad_backend: crate::VadBackend,
|
||||
silero_model_epoch: u64,
|
||||
ten_model_epoch: u64,
|
||||
capture_frame_seq: u64,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||||
/// Processing config — retained for route-change reloads.
|
||||
@@ -157,10 +155,8 @@ mod inner {
|
||||
voice_activity_selector: params.voice_activity_selector.clone(),
|
||||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||||
silero_vad_worker: None,
|
||||
ten_vad_worker: None,
|
||||
current_vad_backend: crate::VadBackend::WebrtcVad,
|
||||
silero_model_epoch: crate::vad::silero_model_epoch(),
|
||||
ten_model_epoch: crate::vad::ten_model_epoch(),
|
||||
capture_frame_seq: 0,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||||
audio_processing_config: params.audio_processing_config.clone(),
|
||||
@@ -274,6 +270,18 @@ mod inner {
|
||||
rec.push_processed_mic(&frame);
|
||||
}
|
||||
|
||||
let voice_activity_mode = self
|
||||
.voice_activity_selector
|
||||
.as_ref()
|
||||
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
||||
.unwrap_or(false);
|
||||
if !voice_activity_mode {
|
||||
self.silero_vad_worker = None;
|
||||
self.current_vad_backend = crate::VadBackend::Disabled;
|
||||
self.fallback_warned_backend = None;
|
||||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||
}
|
||||
|
||||
let (vad_backend, vad_hangover) = self
|
||||
.audio_processing_config
|
||||
.try_lock()
|
||||
@@ -282,30 +290,28 @@ mod inner {
|
||||
crate::VadBackend::WebrtcVad,
|
||||
crate::voice_activity::VAD_HANGOVER_MS,
|
||||
));
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
if voice_activity_mode {
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
}
|
||||
|
||||
// Switch VAD backend when config changes.
|
||||
// Switch VAD backend only while VoiceActivity mode is active.
|
||||
let silero_epoch = crate::vad::silero_model_epoch();
|
||||
let ten_epoch = crate::vad::ten_model_epoch();
|
||||
let silero_changed = vad_backend == crate::VadBackend::SileroOnnx
|
||||
let silero_changed = voice_activity_mode
|
||||
&& vad_backend == crate::VadBackend::SileroOnnx
|
||||
&& silero_epoch != self.silero_model_epoch;
|
||||
let ten_changed =
|
||||
vad_backend == crate::VadBackend::TenVad && ten_epoch != self.ten_model_epoch;
|
||||
if vad_backend != self.current_vad_backend || silero_changed || ten_changed {
|
||||
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_changed) {
|
||||
self.current_vad_backend = vad_backend;
|
||||
self.silero_model_epoch = silero_epoch;
|
||||
self.ten_model_epoch = ten_epoch;
|
||||
self.fallback_warned_backend = None;
|
||||
match vad_backend {
|
||||
crate::VadBackend::SileroOnnx => {
|
||||
let path = crate::vad::silero_model_bundle_path();
|
||||
self.silero_vad_worker =
|
||||
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
|
||||
self.ten_vad_worker = None;
|
||||
if self.silero_vad_worker.is_none() {
|
||||
tracing::warn!(
|
||||
target: "chanora_audio",
|
||||
@@ -313,46 +319,43 @@ mod inner {
|
||||
);
|
||||
}
|
||||
}
|
||||
crate::VadBackend::TenVad => {
|
||||
let path = crate::vad::ten_model_bundle_path();
|
||||
self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path);
|
||||
self.silero_vad_worker = None;
|
||||
if self.ten_vad_worker.is_none() {
|
||||
tracing::warn!(
|
||||
target: "chanora_audio",
|
||||
"TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD"
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
self.silero_vad_worker = None;
|
||||
self.ten_vad_worker = None;
|
||||
}
|
||||
}
|
||||
self.vad_state.reset();
|
||||
}
|
||||
|
||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||
let capture_seq = self.capture_frame_seq;
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let p = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability: p,
|
||||
speech: p >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
let (vad_probability, active) = if voice_activity_mode {
|
||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||
let capture_seq = self.capture_frame_seq;
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let p = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability: p,
|
||||
speech: p >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(
|
||||
&mut self.vad_detector,
|
||||
&frame,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
@@ -363,56 +366,26 @@ mod inner {
|
||||
)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::TenVad {
|
||||
if let Some(worker) = self.ten_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let p = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability: p,
|
||||
speech: p >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(
|
||||
&mut self.vad_detector,
|
||||
&frame,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
(vad.probability, self.vad_state.update(vad.speech))
|
||||
} else {
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
(0.0, false)
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
let active = self.vad_state.update(vad.speech);
|
||||
let output_muted = self.output_muted.load(Ordering::Relaxed);
|
||||
if let Some(sel) = &self.voice_activity_selector {
|
||||
sel.set_voice_activity_open(active && !output_muted);
|
||||
sel.set_voice_activity_open(voice_activity_mode && active);
|
||||
}
|
||||
self.audio_processing_stats.update_capture(
|
||||
input_dbfs,
|
||||
crate::frame::dbfs(&frame),
|
||||
vad.probability,
|
||||
active && !output_muted,
|
||||
vad_probability,
|
||||
voice_activity_mode && active,
|
||||
self.transmit_active.load(Ordering::Relaxed),
|
||||
);
|
||||
|
||||
if !self.transmit_active.load(Ordering::Relaxed) || output_muted {
|
||||
if !self.transmit_active.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -146,13 +146,10 @@ struct IosCaptureState {
|
||||
/// Background Silero worker — enqueues frames off the realtime
|
||||
/// callback and publishes the latest probability atomically.
|
||||
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
|
||||
ten_vad: Option<crate::vad::TenOnnxVadWorker>,
|
||||
/// Last VAD backend we configured — used to detect backend changes.
|
||||
current_vad_backend: crate::VadBackend,
|
||||
/// Last observed configured Silero model epoch.
|
||||
silero_model_epoch: u64,
|
||||
/// Last observed configured TEN model epoch.
|
||||
ten_model_epoch: u64,
|
||||
fallback_warned_backend: Option<crate::VadBackend>,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
@@ -191,10 +188,8 @@ impl IosCaptureState {
|
||||
voice_activity_selector: params.voice_activity_selector.clone(),
|
||||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||||
silero_vad_worker: None,
|
||||
ten_vad: None,
|
||||
current_vad_backend: crate::VadBackend::WebrtcVad,
|
||||
silero_model_epoch: crate::vad::silero_model_epoch(),
|
||||
ten_model_epoch: crate::vad::ten_model_epoch(),
|
||||
fallback_warned_backend: None,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||||
audio_processing_config: params.audio_processing_config.clone(),
|
||||
@@ -361,13 +356,23 @@ impl IosCaptureState {
|
||||
crate::AudioBackend::PlatformVoiceProcessing,
|
||||
));
|
||||
|
||||
// Switch VAD backend when the config changes.
|
||||
let voice_activity_mode = self
|
||||
.voice_activity_selector
|
||||
.as_ref()
|
||||
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
||||
.unwrap_or(false);
|
||||
if !voice_activity_mode {
|
||||
self.silero_vad_worker = None;
|
||||
self.current_vad_backend = crate::VadBackend::Disabled;
|
||||
self.fallback_warned_backend = None;
|
||||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||
}
|
||||
|
||||
// Switch VAD backend only while VoiceActivity mode is active.
|
||||
let silero_model_epoch = crate::vad::silero_model_epoch();
|
||||
let ten_model_epoch = crate::vad::ten_model_epoch();
|
||||
let silero_model_changed = vad_backend == crate::VadBackend::SileroOnnx
|
||||
let silero_model_changed = voice_activity_mode
|
||||
&& vad_backend == crate::VadBackend::SileroOnnx
|
||||
&& silero_model_epoch != self.silero_model_epoch;
|
||||
let ten_model_changed =
|
||||
vad_backend == crate::VadBackend::TenVad && ten_model_epoch != self.ten_model_epoch;
|
||||
|
||||
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
|
||||
if debug_wav_dump_enabled {
|
||||
@@ -382,11 +387,11 @@ impl IosCaptureState {
|
||||
}
|
||||
}
|
||||
|
||||
if vad_backend != self.current_vad_backend || silero_model_changed || ten_model_changed {
|
||||
if voice_activity_mode && (vad_backend != self.current_vad_backend || silero_model_changed)
|
||||
{
|
||||
self.current_vad_backend = vad_backend;
|
||||
self.fallback_warned_backend = None;
|
||||
self.silero_model_epoch = silero_model_epoch;
|
||||
self.ten_model_epoch = ten_model_epoch;
|
||||
match vad_backend {
|
||||
crate::VadBackend::SileroOnnx => {
|
||||
// Attempt to load Silero model from the well-known
|
||||
@@ -407,23 +412,8 @@ impl IosCaptureState {
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(self.silero_vad_worker.is_none());
|
||||
}
|
||||
crate::VadBackend::TenVad => {
|
||||
self.silero_vad_worker = None;
|
||||
let model_path = crate::vad::ten_model_bundle_path();
|
||||
self.ten_vad = crate::vad::TenOnnxVadWorker::try_new(&model_path);
|
||||
if self.ten_vad.is_none() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"TEN VAD ONNX model not available at {model_path}; falling back to WebRTC VAD"
|
||||
);
|
||||
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
|
||||
}
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(self.ten_vad.is_none());
|
||||
}
|
||||
_ => {
|
||||
self.silero_vad_worker = None;
|
||||
self.ten_vad = None;
|
||||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||
}
|
||||
}
|
||||
@@ -436,13 +426,16 @@ impl IosCaptureState {
|
||||
self.vad_state.reset();
|
||||
}
|
||||
|
||||
// Keep the VAD state machine aligned with the active config.
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
|
||||
if voice_activity_mode {
|
||||
// Keep the VAD state machine aligned with the active config only
|
||||
// while VoiceActivity mode owns the transmit gate.
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
}
|
||||
|
||||
// VPIO owns AEC. Keep the legacy non-AEC conditioning path here until
|
||||
// the raw WebRTC APM path is explicitly selected.
|
||||
@@ -461,29 +454,37 @@ impl IosCaptureState {
|
||||
self.sonora_processor.process_capture(&mut frame);
|
||||
}
|
||||
|
||||
// VAD: use Silero if loaded, otherwise WebRTC fallback.
|
||||
// Disabled backend → always open (Continuous-like for VAD mode).
|
||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||
let capture_seq = self.capture_frame_seq;
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let probability = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability,
|
||||
speech: probability >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
// VAD: only evaluate while VoiceActivity mode is active.
|
||||
let (vad_probability, gate_open) = if voice_activity_mode {
|
||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||
let capture_seq = self.capture_frame_seq;
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let probability = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability,
|
||||
speech: probability >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(
|
||||
&mut self.vad_detector,
|
||||
&frame,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
@@ -491,49 +492,23 @@ impl IosCaptureState {
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::TenVad {
|
||||
if let Some(worker) = self.ten_vad.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if !worker.is_stale(capture_seq) {
|
||||
let probability = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability,
|
||||
speech: probability >= 0.5,
|
||||
}
|
||||
} else if enqueued {
|
||||
crate::vad::VadOutput {
|
||||
probability: 0.0,
|
||||
speech: false,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
(vad.probability, self.vad_state.update(vad.speech))
|
||||
} else {
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
(0.0, false)
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
let gate_open = self.vad_state.update(vad.speech);
|
||||
let output_muted = self.output_muted.load(Ordering::Relaxed);
|
||||
if let Some(selector) = &self.voice_activity_selector {
|
||||
selector.set_voice_activity_open(gate_open && !output_muted);
|
||||
selector.set_voice_activity_open(voice_activity_mode && gate_open && !output_muted);
|
||||
}
|
||||
self.audio_processing_stats.update_capture(
|
||||
input_dbfs,
|
||||
crate::frame::dbfs(&frame),
|
||||
vad.probability,
|
||||
gate_open && !output_muted,
|
||||
vad_probability,
|
||||
voice_activity_mode && gate_open && !output_muted,
|
||||
transmit_active,
|
||||
);
|
||||
|
||||
|
||||
@@ -203,8 +203,8 @@ impl Default for AndroidVoiceStreamConfig {
|
||||
Self {
|
||||
sample_rate: 48_000,
|
||||
channel_count: 1,
|
||||
request_low_latency: true,
|
||||
request_exclusive: true,
|
||||
request_low_latency: false,
|
||||
request_exclusive: false,
|
||||
effects: AudioEffects::default(),
|
||||
}
|
||||
}
|
||||
@@ -488,7 +488,7 @@ impl fmt::Display for AchievedInputPreset {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AndroidAudioDiagnostics {
|
||||
// Requested side (SDD-112 items 4..7) — fixed by SDD-112.
|
||||
/// Requested performance mode (always "LowLatency" in P0).
|
||||
/// Requested performance mode for the current Android request profile.
|
||||
pub requested_performance_mode: &'static str,
|
||||
/// Requested output usage (always "VoiceCommunication" in P0).
|
||||
pub requested_usage: &'static str,
|
||||
@@ -496,7 +496,7 @@ pub struct AndroidAudioDiagnostics {
|
||||
pub requested_content_type: &'static str,
|
||||
/// Requested input preset (always "VoiceCommunication" first).
|
||||
pub requested_input_preset: &'static str,
|
||||
/// Requested sharing mode (always "Exclusive" first).
|
||||
/// Requested sharing mode for the current Android request profile.
|
||||
pub requested_sharing_mode: &'static str,
|
||||
/// Requested sample rate (Hz).
|
||||
pub requested_sample_rate_hz: u32,
|
||||
@@ -640,15 +640,15 @@ pub fn current_android_audio_diagnostics() -> Option<AndroidAudioDiagnostics> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// SWE4-UV-047: default config records requested low-latency,
|
||||
/// exclusive sharing, mono 48 kHz, and default effects.
|
||||
/// SWE4-UV-047: default config records the conservative shared
|
||||
/// voice-input request profile, mono 48 kHz, and default effects.
|
||||
#[test]
|
||||
fn swe4_uv_047_default_config_records_requested_values() {
|
||||
let cfg = AndroidVoiceStreamConfig::default();
|
||||
assert_eq!(cfg.sample_rate, 48_000);
|
||||
assert_eq!(cfg.channel_count, 1);
|
||||
assert!(cfg.request_low_latency);
|
||||
assert!(cfg.request_exclusive);
|
||||
assert!(!cfg.request_low_latency);
|
||||
assert!(!cfg.request_exclusive);
|
||||
// AudioEffects defaults are all-on per DEC-007..010.
|
||||
assert!(cfg.effects.aec);
|
||||
assert!(cfg.effects.noise_suppression);
|
||||
@@ -702,7 +702,9 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// SWE4-UV-049: sharing-mode ladder is Exclusive -> Shared -> exhausted.
|
||||
/// SWE4-UV-049: sharing-mode ladder remains Exclusive -> Shared -> exhausted.
|
||||
/// The default request profile may start at Shared, but the ladder still
|
||||
/// exists for explicitly opt-in low-latency / exclusive experiments.
|
||||
#[test]
|
||||
fn swe4_uv_049_sharing_mode_fallback_ladder_order() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
|
||||
pub mod resampler;
|
||||
pub mod silero_onnx;
|
||||
pub mod ten_onnx;
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
@@ -17,7 +16,6 @@ use crate::AudioError;
|
||||
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||
|
||||
pub use silero_onnx::SileroOnnxVad;
|
||||
pub use ten_onnx::{TenOnnxVad, TenOnnxVadWorker};
|
||||
|
||||
/// Voice activity detector output for one 10 ms frame.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -108,17 +106,11 @@ pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16
|
||||
|
||||
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
static TEN_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static TEN_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
fn ten_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
TEN_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
/// Configure the preferred Silero ONNX model path.
|
||||
///
|
||||
/// The path is validated eagerly. A successful call increments the
|
||||
@@ -149,32 +141,6 @@ pub fn silero_model_epoch() -> u64 {
|
||||
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Configure the preferred TEN VAD ONNX model path.
|
||||
pub fn set_ten_model_path(path: &str) -> Result<(), AudioError> {
|
||||
let path = path.trim();
|
||||
if path.is_empty() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"ten vad model path must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
if !std::path::Path::new(path).is_file() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(format!(
|
||||
"ten vad model path does not exist or is not a file: {path}"
|
||||
)));
|
||||
}
|
||||
let mut guard = ten_model_path_override()
|
||||
.write()
|
||||
.map_err(|_| AudioError::Backend("ten vad model path lock poisoned".to_string()))?;
|
||||
*guard = Some(path.to_string());
|
||||
TEN_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Monotonic counter incremented whenever the configured TEN model path changes.
|
||||
pub fn ten_model_epoch() -> u64 {
|
||||
TEN_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Return the expected path of the Silero VAD v6 ONNX model.
|
||||
/// The model is shipped as a Flutter asset and copied to the app's
|
||||
/// data directory by the Dart-side asset loader.
|
||||
@@ -231,46 +197,6 @@ pub fn silero_model_bundle_path() -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the expected path of the TEN VAD ONNX model copied by Flutter.
|
||||
pub fn ten_model_bundle_path() -> String {
|
||||
if let Ok(guard) = ten_model_path_override().read() {
|
||||
if let Some(path) = guard.as_ref() {
|
||||
return path.clone();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
let docs = format!("{home}/Documents/ten_vad.onnx");
|
||||
if std::path::Path::new(&docs).exists() {
|
||||
return docs;
|
||||
}
|
||||
let bundle = format!("{home}/../Library/ten_vad.onnx");
|
||||
if std::path::Path::new(&bundle).exists() {
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||||
{
|
||||
if let Ok(cwd) = std::env::current_dir() {
|
||||
let local = cwd.join("ten_vad.onnx");
|
||||
if local.exists() {
|
||||
return local.to_string_lossy().to_string();
|
||||
}
|
||||
}
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,452 +0,0 @@
|
||||
//! TEN VAD ONNX backend.
|
||||
//!
|
||||
//! TEN's ONNX graph does not accept raw PCM. It expects the same feature
|
||||
//! stack produced by TEN's `AUP_Aed_aivad_proc`: three context frames of
|
||||
//! 40 log-mel powers plus one pitch feature, followed by four recurrent
|
||||
//! state tensors. This module ports that preprocessing path to Rust and
|
||||
//! keeps ONNX Runtime off the realtime callback where possible.
|
||||
|
||||
use crate::frame::f32_to_i16;
|
||||
|
||||
use rustfft::{num_complex::Complex32, FftPlanner};
|
||||
|
||||
use super::resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||
use super::{VadOutput, VoiceActivityDetector};
|
||||
|
||||
const SAMPLE_RATE_16K: f32 = 16_000.0;
|
||||
const HOP_16K: usize = 256;
|
||||
const WINDOW_16K: usize = 768;
|
||||
const FFT_SIZE: usize = 1024;
|
||||
const N_BINS: usize = FFT_SIZE / 2 + 1;
|
||||
const MEL_BANDS: usize = 40;
|
||||
const FEATURE_LEN: usize = 41;
|
||||
const CONTEXT: usize = 3;
|
||||
const HIDDEN: usize = 64;
|
||||
const POWER_NORMALIZER: f32 = 32768.0 * 32768.0;
|
||||
const EPS: f32 = 1.0e-20;
|
||||
|
||||
const FEATURE_MEANS: [f32; FEATURE_LEN] = [
|
||||
-8.198236, -6.2657166, -5.4838185, -4.7586913, -4.417089, -4.142893, -3.9128504, -3.845928,
|
||||
-3.6570904, -3.7234187, -3.8761342, -3.843891, -3.6904051, -3.7560658, -3.6986961, -3.650463,
|
||||
-3.7004688, -3.5673213, -3.4989002, -3.477807, -3.458816, -3.4449239, -3.4013286, -3.3062613,
|
||||
-3.2785568, -3.2332509, -3.198616, -3.2045264, -3.2087986, -3.257838, -3.3813767, -3.5340214,
|
||||
-3.640868, -3.7268589, -3.773731, -3.8046672, -3.832901, -3.8711205, -3.990593, -4.4802895,
|
||||
92.3569,
|
||||
];
|
||||
|
||||
const FEATURE_STDS: [f32; FEATURE_LEN] = [
|
||||
5.166064, 4.9772096, 4.698896, 4.6306214, 4.634348, 4.641156, 4.6406765, 4.666367, 4.6505346,
|
||||
4.640021, 4.6374, 4.620099, 4.5963163, 4.562655, 4.5543604, 4.5669107, 4.56249, 4.5624127,
|
||||
4.5852995, 4.6001797, 4.592846, 4.5859227, 4.5834966, 4.626093, 4.626958, 4.6262894, 4.637006,
|
||||
4.683016, 4.726814, 4.7342896, 4.753227, 4.849723, 4.869435, 4.884483, 4.921327, 4.9592123,
|
||||
4.996619, 5.0448236, 5.072217, 5.0964394, 115.21369,
|
||||
];
|
||||
|
||||
/// TEN VAD using ONNX Runtime and Rust-ported TEN feature preprocessing.
|
||||
pub struct TenOnnxVad {
|
||||
session: ort::session::Session,
|
||||
downsampler: Downsampler48to16,
|
||||
hop_accum: Vec<f32>,
|
||||
sample_fifo: Vec<f32>,
|
||||
feature_stack: [[f32; FEATURE_LEN]; CONTEXT],
|
||||
states: [[f32; HIDDEN]; 4],
|
||||
mel_filters: Vec<[f32; N_BINS]>,
|
||||
fft: std::sync::Arc<dyn rustfft::Fft<f32>>,
|
||||
fft_buffer: Vec<Complex32>,
|
||||
last_probability: f32,
|
||||
last_speech: bool,
|
||||
}
|
||||
|
||||
unsafe impl Send for TenOnnxVad {}
|
||||
|
||||
impl TenOnnxVad {
|
||||
/// Load TEN VAD ONNX model.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
if !std::path::Path::new(model_path).exists() {
|
||||
tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model not found");
|
||||
return None;
|
||||
}
|
||||
let session = match std::panic::catch_unwind(|| {
|
||||
ort::session::Session::builder().and_then(|mut b| b.commit_from_file(model_path))
|
||||
}) {
|
||||
Ok(Ok(session)) => session,
|
||||
Ok(Err(error)) => {
|
||||
tracing::warn!(target: "chanora_audio", %error, path = model_path, "TEN VAD ONNX model load failed");
|
||||
return None;
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX Runtime panicked during load");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let mut fft_planner = FftPlanner::<f32>::new();
|
||||
let fft = fft_planner.plan_fft_forward(FFT_SIZE);
|
||||
tracing::info!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model loaded");
|
||||
Some(Self {
|
||||
session,
|
||||
downsampler: Downsampler48to16::default(),
|
||||
hop_accum: Vec::with_capacity(HOP_16K + super::resampler::OUTPUT_FRAME_10MS),
|
||||
sample_fifo: Vec::with_capacity(WINDOW_16K + HOP_16K),
|
||||
feature_stack: [[0.0; FEATURE_LEN]; CONTEXT],
|
||||
states: [[0.0; HIDDEN]; 4],
|
||||
mel_filters: build_mel_filters(),
|
||||
fft,
|
||||
fft_buffer: vec![Complex32::ZERO; FFT_SIZE],
|
||||
last_probability: 0.0,
|
||||
last_speech: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn process_hop(&mut self, hop: &[f32]) {
|
||||
self.sample_fifo.extend_from_slice(hop);
|
||||
let frame = if self.sample_fifo.len() >= WINDOW_16K {
|
||||
let start = self.sample_fifo.len() - WINDOW_16K;
|
||||
self.sample_fifo[start..].to_vec()
|
||||
} else {
|
||||
let mut padded = vec![0.0; WINDOW_16K - self.sample_fifo.len()];
|
||||
padded.extend_from_slice(&self.sample_fifo);
|
||||
padded
|
||||
};
|
||||
if self.sample_fifo.len() > WINDOW_16K {
|
||||
let excess = self.sample_fifo.len() - WINDOW_16K;
|
||||
self.sample_fifo.drain(..excess);
|
||||
}
|
||||
|
||||
let feature = compute_feature(
|
||||
&self.mel_filters,
|
||||
self.fft.as_ref(),
|
||||
&mut self.fft_buffer,
|
||||
&frame,
|
||||
);
|
||||
self.feature_stack.copy_within(1..CONTEXT, 0);
|
||||
self.feature_stack[CONTEXT - 1] = feature;
|
||||
self.run_onnx();
|
||||
}
|
||||
|
||||
fn run_onnx(&mut self) {
|
||||
use ndarray::{Array, IxDyn};
|
||||
use ort::value::Value;
|
||||
|
||||
let input: Vec<f32> = self.feature_stack.iter().flatten().copied().collect();
|
||||
let input_arr = match Array::from_shape_vec(IxDyn(&[1, CONTEXT, FEATURE_LEN]), input) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return,
|
||||
};
|
||||
let state_arrs = [0, 1, 2, 3]
|
||||
.map(|idx| Array::from_shape_vec(IxDyn(&[1, HIDDEN]), self.states[idx].to_vec()));
|
||||
let input_val = match Value::from_array(input_arr) {
|
||||
Ok(v) => v,
|
||||
Err(error) => {
|
||||
tracing::warn!(target: "chanora_audio", %error, "TEN VAD input tensor error");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let state_vals = match state_arrs {
|
||||
[Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d],
|
||||
_ => return,
|
||||
};
|
||||
let state_vals = match state_vals.map(Value::from_array) {
|
||||
[Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d],
|
||||
_ => return,
|
||||
};
|
||||
|
||||
let outputs = match self.session.run([
|
||||
(&input_val).into(),
|
||||
(&state_vals[0]).into(),
|
||||
(&state_vals[1]).into(),
|
||||
(&state_vals[2]).into(),
|
||||
(&state_vals[3]).into(),
|
||||
]) {
|
||||
Ok(outputs) => outputs,
|
||||
Err(error) => {
|
||||
tracing::warn!(target: "chanora_audio", %error, "TEN VAD ONNX inference failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if let Ok((_, prob)) = outputs["output_1"].try_extract_tensor::<f32>() {
|
||||
if let Some(&p) = prob.first() {
|
||||
self.last_probability = p.clamp(0.0, 1.0);
|
||||
self.last_speech = self.last_probability >= 0.5;
|
||||
}
|
||||
}
|
||||
for (idx, name) in ["output_2", "output_3", "output_6", "output_7"]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
if let Ok((_, state)) = outputs[*name].try_extract_tensor::<f32>() {
|
||||
let copy_len = state.len().min(HIDDEN);
|
||||
self.states[idx][..copy_len].copy_from_slice(&state[..copy_len]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn compute_feature(
|
||||
mel_filters: &[[f32; N_BINS]],
|
||||
fft: &dyn rustfft::Fft<f32>,
|
||||
fft_buffer: &mut [Complex32],
|
||||
frame: &[f32],
|
||||
) -> [f32; FEATURE_LEN] {
|
||||
let power = power_spectrum(fft, fft_buffer, frame);
|
||||
let mut feature = [0.0; FEATURE_LEN];
|
||||
for band in 0..MEL_BANDS {
|
||||
let energy = mel_filters[band]
|
||||
.iter()
|
||||
.zip(power.iter())
|
||||
.map(|(w, p)| w * p)
|
||||
.sum::<f32>()
|
||||
/ POWER_NORMALIZER;
|
||||
let log_energy = (energy + EPS).ln();
|
||||
feature[band] = (log_energy - FEATURE_MEANS[band]) / (FEATURE_STDS[band] + EPS);
|
||||
}
|
||||
let pitch_hz = estimate_pitch_hz(frame);
|
||||
feature[MEL_BANDS] = (pitch_hz - FEATURE_MEANS[MEL_BANDS]) / (FEATURE_STDS[MEL_BANDS] + EPS);
|
||||
feature
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for TenOnnxVad {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
debug_assert_eq!(samples.len(), INPUT_FRAME_10MS);
|
||||
let mut input = [0.0_f32; INPUT_FRAME_10MS];
|
||||
input.copy_from_slice(samples);
|
||||
let downsampled = self.downsampler.process_frame_10ms(&input);
|
||||
self.hop_accum.extend_from_slice(&downsampled);
|
||||
while self.hop_accum.len() >= HOP_16K {
|
||||
let hop: Vec<f32> = self.hop_accum[..HOP_16K].to_vec();
|
||||
self.hop_accum.drain(..HOP_16K);
|
||||
self.process_hop(&hop);
|
||||
}
|
||||
VadOutput {
|
||||
probability: self.last_probability,
|
||||
speech: self.last_speech,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hz_to_mel(hz: f32) -> f32 {
|
||||
2595.0 * (1.0 + hz / 700.0).log10()
|
||||
}
|
||||
|
||||
fn mel_to_hz(mel: f32) -> f32 {
|
||||
700.0 * (10.0_f32.powf(mel / 2595.0) - 1.0)
|
||||
}
|
||||
|
||||
fn build_mel_filters() -> Vec<[f32; N_BINS]> {
|
||||
let low_mel = hz_to_mel(0.0);
|
||||
let high_mel = hz_to_mel(8000.0);
|
||||
let mut bins = [0_usize; MEL_BANDS + 2];
|
||||
for idx in 0..bins.len() {
|
||||
let mel = idx as f32 * (high_mel - low_mel) / (MEL_BANDS as f32 + 1.0) + low_mel;
|
||||
let hz = mel_to_hz(mel);
|
||||
let mut bin = ((FFT_SIZE as f32 + 1.0) * hz / SAMPLE_RATE_16K).floor() as usize;
|
||||
bin = bin.min(N_BINS - 1);
|
||||
if idx > 0 && bin == bins[idx - 1] {
|
||||
bin = (bin + 1).min(N_BINS - 1);
|
||||
}
|
||||
bins[idx] = bin;
|
||||
}
|
||||
|
||||
let mut filters = vec![[0.0_f32; N_BINS]; MEL_BANDS];
|
||||
for band in 0..MEL_BANDS {
|
||||
let left = bins[band];
|
||||
let center = bins[band + 1].max(left + 1);
|
||||
let right = bins[band + 2].max(center + 1).min(N_BINS - 1);
|
||||
for (i, weight) in filters[band]
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
.take(center.min(N_BINS))
|
||||
.skip(left)
|
||||
{
|
||||
*weight = (i - left) as f32 / (center - left) as f32;
|
||||
}
|
||||
for (i, weight) in filters[band]
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
.take(right + 1)
|
||||
.skip(center)
|
||||
{
|
||||
*weight = (right - i) as f32 / (right - center).max(1) as f32;
|
||||
}
|
||||
}
|
||||
filters
|
||||
}
|
||||
|
||||
fn power_spectrum(
|
||||
fft: &dyn rustfft::Fft<f32>,
|
||||
fft_buffer: &mut [Complex32],
|
||||
frame: &[f32],
|
||||
) -> [f32; N_BINS] {
|
||||
debug_assert_eq!(fft_buffer.len(), FFT_SIZE);
|
||||
fft_buffer.fill(Complex32::ZERO);
|
||||
for (idx, sample) in frame.iter().take(WINDOW_16K).enumerate() {
|
||||
let hann = 0.5 - 0.5 * (2.0 * std::f32::consts::PI * idx as f32 / WINDOW_16K as f32).cos();
|
||||
fft_buffer[idx].re = f32_to_i16(*sample) as f32 * hann;
|
||||
}
|
||||
|
||||
fft.process(fft_buffer);
|
||||
|
||||
let mut out = [0.0_f32; N_BINS];
|
||||
for (dst, bin) in out.iter_mut().zip(fft_buffer.iter()) {
|
||||
*dst = bin.norm_sqr();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn estimate_pitch_hz(frame: &[f32]) -> f32 {
|
||||
let min_lag = (SAMPLE_RATE_16K / 400.0) as usize;
|
||||
let max_lag = (SAMPLE_RATE_16K / 60.0) as usize;
|
||||
let mut best_lag = 0_usize;
|
||||
let mut best_corr = 0.0_f32;
|
||||
for lag in min_lag..=max_lag.min(frame.len().saturating_sub(1)) {
|
||||
let mut corr = 0.0_f32;
|
||||
let mut energy = 0.0_f32;
|
||||
for i in lag..frame.len() {
|
||||
corr += frame[i] * frame[i - lag];
|
||||
energy += frame[i - lag] * frame[i - lag];
|
||||
}
|
||||
let norm = if energy > 1.0e-8 {
|
||||
corr / energy.sqrt()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
if norm > best_corr {
|
||||
best_corr = norm;
|
||||
best_lag = lag;
|
||||
}
|
||||
}
|
||||
if best_lag == 0 || best_corr < 0.01 {
|
||||
0.0
|
||||
} else {
|
||||
SAMPLE_RATE_16K / best_lag as f32
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Background worker — same pattern as SileroOnnxVadWorker so the realtime
|
||||
// callback never blocks on STFT / pitch / ONNX inference.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64};
|
||||
use std::sync::Arc;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// Maximum number of 10 ms frames the worker may lag before the callback
|
||||
/// treats its output as stale and uses WebRTC fallback instead.
|
||||
const TEN_MAX_STALE_FRAMES: u64 = 8;
|
||||
|
||||
struct TenFrameMessage {
|
||||
seq: u64,
|
||||
frame: [f32; INPUT_FRAME_10MS],
|
||||
}
|
||||
|
||||
/// Background TEN VAD worker. The realtime callback only enqueues 10 ms
|
||||
/// frames and reads the latest probability atomically.
|
||||
pub struct TenOnnxVadWorker {
|
||||
tx: Option<std::sync::mpsc::SyncSender<TenFrameMessage>>,
|
||||
latest_probability: Arc<AtomicU32>,
|
||||
latest_processed_seq: Arc<AtomicU64>,
|
||||
alive: Arc<AtomicBool>,
|
||||
handle: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl TenOnnxVadWorker {
|
||||
/// Start a background TEN worker if the model loads.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
let vad = TenOnnxVad::try_new(model_path)?;
|
||||
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
|
||||
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
|
||||
let alive = Arc::new(AtomicBool::new(true));
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<TenFrameMessage>(128);
|
||||
let prob_arc = latest_probability.clone();
|
||||
let seq_arc = latest_processed_seq.clone();
|
||||
let alive_arc = alive.clone();
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("chanora-ten-vad".to_string())
|
||||
.spawn(move || {
|
||||
let mut vad = vad;
|
||||
while alive_arc.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let msg = match rx.recv() {
|
||||
Ok(m) => m,
|
||||
Err(_) => break,
|
||||
};
|
||||
let mut frame_f32 = [0.0_f32; INPUT_FRAME_10MS];
|
||||
frame_f32.copy_from_slice(&msg.frame);
|
||||
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame_f32);
|
||||
prob_arc.store(
|
||||
out.probability.clamp(0.0, 1.0).to_bits(),
|
||||
std::sync::atomic::Ordering::Relaxed,
|
||||
);
|
||||
seq_arc.store(msg.seq, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
})
|
||||
.ok()?;
|
||||
|
||||
Some(Self {
|
||||
tx: Some(tx),
|
||||
latest_probability,
|
||||
latest_processed_seq,
|
||||
alive,
|
||||
handle: Some(handle),
|
||||
})
|
||||
}
|
||||
|
||||
/// Best-effort enqueue of a 10 ms frame for background inference.
|
||||
pub fn try_send(&self, seq: u64, frame: &[f32; INPUT_FRAME_10MS]) -> bool {
|
||||
let Some(tx) = &self.tx else {
|
||||
return false;
|
||||
};
|
||||
tx.try_send(TenFrameMessage { seq, frame: *frame }).is_ok()
|
||||
}
|
||||
|
||||
/// Latest probability published by the background worker.
|
||||
pub fn latest_probability(&self) -> f32 {
|
||||
f32::from_bits(
|
||||
self.latest_probability
|
||||
.load(std::sync::atomic::Ordering::Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
/// True when the worker is too far behind to trust its output.
|
||||
pub fn is_stale(&self, capture_seq: u64) -> bool {
|
||||
let latest = self
|
||||
.latest_processed_seq
|
||||
.load(std::sync::atomic::Ordering::Relaxed);
|
||||
latest == u64::MAX || capture_seq.saturating_sub(latest) > TEN_MAX_STALE_FRAMES
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TenOnnxVadWorker {
|
||||
fn drop(&mut self) {
|
||||
self.alive
|
||||
.store(false, std::sync::atomic::Ordering::Relaxed);
|
||||
drop(self.tx.take());
|
||||
if let Some(h) = self.handle.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn mel_filter_bank_has_expected_shape() {
|
||||
let filters = build_mel_filters();
|
||||
assert_eq!(filters.len(), MEL_BANDS);
|
||||
assert!(filters.iter().all(|f| f.iter().any(|&v| v > 0.0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preprocessing_produces_finite_features() {
|
||||
let filters = build_mel_filters();
|
||||
let mut planner = FftPlanner::<f32>::new();
|
||||
let fft = planner.plan_fft_forward(FFT_SIZE);
|
||||
let mut fft_buffer = vec![Complex32::ZERO; FFT_SIZE];
|
||||
let frame = vec![0.0_f32; WINDOW_16K];
|
||||
let feature = compute_feature(&filters, fft.as_ref(), &mut fft_buffer, &frame);
|
||||
assert!(feature.iter().all(|v| v.is_finite()));
|
||||
}
|
||||
}
|
||||
@@ -145,7 +145,7 @@ pub(crate) fn publish_permission_state(permission: String, state: PermissionStat
|
||||
/// flood the diagnostic export during transient packet loss;
|
||||
/// users can still raise verbosity via `RUST_LOG=info`.
|
||||
const DEFAULT_LOG_FILTER: &str =
|
||||
"info,tsproto::resend=error,tsproto::packet_codec=error,tsclientlib=error";
|
||||
"info,tsproto::resend=error,tsproto::packet_codec=error,tsclientlib=error,ts_bookkeeping::messages::s2c=error";
|
||||
|
||||
/// Initialise the bridge. Must be called once on Dart side before
|
||||
/// any other API call. Sets up panic logging.
|
||||
@@ -921,8 +921,6 @@ pub enum BridgeAudioBackend {
|
||||
pub enum BridgeVadBackend {
|
||||
/// Silero ONNX VAD.
|
||||
SileroOnnx,
|
||||
/// TEN VAD.
|
||||
TenVad,
|
||||
/// WebRTC fallback VAD.
|
||||
WebrtcVad,
|
||||
/// Debug energy VAD.
|
||||
@@ -1014,6 +1012,16 @@ pub struct BridgeAudioProcessingStats {
|
||||
pub callback_xruns: u64,
|
||||
/// Clipped samples.
|
||||
pub clipped_samples: u64,
|
||||
/// Number of effectively silent processed capture frames.
|
||||
pub zero_frames: u64,
|
||||
/// Number of processed capture frames.
|
||||
pub capture_frames: u64,
|
||||
/// Input callbacks carrying 10 ms of audio.
|
||||
pub callbacks_10ms: u64,
|
||||
/// Input callbacks carrying 20 ms of audio.
|
||||
pub callbacks_20ms: u64,
|
||||
/// Input callbacks carrying other sizes.
|
||||
pub callbacks_other: u64,
|
||||
/// Sonora enabled.
|
||||
pub sonora_enabled: bool,
|
||||
/// Platform voice processing enabled.
|
||||
@@ -1092,7 +1100,6 @@ impl From<BridgeVadBackend> for chanora_core::VadBackend {
|
||||
fn from(backend: BridgeVadBackend) -> Self {
|
||||
match backend {
|
||||
BridgeVadBackend::SileroOnnx => Self::SileroOnnx,
|
||||
BridgeVadBackend::TenVad => Self::TenVad,
|
||||
BridgeVadBackend::WebrtcVad => Self::WebrtcVad,
|
||||
BridgeVadBackend::EnergyDebug => Self::EnergyDebug,
|
||||
BridgeVadBackend::Disabled => Self::Disabled,
|
||||
@@ -1104,7 +1111,6 @@ impl From<chanora_core::VadBackend> for BridgeVadBackend {
|
||||
fn from(backend: chanora_core::VadBackend) -> Self {
|
||||
match backend {
|
||||
chanora_core::VadBackend::SileroOnnx => Self::SileroOnnx,
|
||||
chanora_core::VadBackend::TenVad => Self::TenVad,
|
||||
chanora_core::VadBackend::WebrtcVad => Self::WebrtcVad,
|
||||
chanora_core::VadBackend::EnergyDebug => Self::EnergyDebug,
|
||||
chanora_core::VadBackend::Disabled => Self::Disabled,
|
||||
@@ -1196,6 +1202,11 @@ impl From<chanora_core::AudioProcessingStats> for BridgeAudioProcessingStats {
|
||||
output_underruns: stats.output_underruns,
|
||||
callback_xruns: stats.callback_xruns,
|
||||
clipped_samples: stats.clipped_samples,
|
||||
zero_frames: stats.zero_frames,
|
||||
capture_frames: stats.capture_frames,
|
||||
callbacks_10ms: stats.callbacks_10ms,
|
||||
callbacks_20ms: stats.callbacks_20ms,
|
||||
callbacks_other: stats.callbacks_other,
|
||||
sonora_enabled: stats.sonora_enabled,
|
||||
platform_voice_processing_enabled: stats.platform_voice_processing_enabled,
|
||||
}
|
||||
@@ -1236,8 +1247,31 @@ pub fn export_diagnostics() -> String {
|
||||
let android_audio_yaml =
|
||||
chanora_audio::mobile_voice_backend::current_android_audio_diagnostics()
|
||||
.map(|d| d.to_yaml_fragment());
|
||||
let audio_health = runtime().block_on(async { session().audio_processing_stats().await.ok() });
|
||||
let network_info = runtime().block_on(async { session().network_diagnostics_summary().await });
|
||||
let protocol_events = runtime().block_on(async { session().drain_protocol_events().await });
|
||||
let android_audio_yaml = android_audio_yaml.map(|mut yaml| {
|
||||
if let Some(stats) = audio_health {
|
||||
yaml.push_str(&format!(
|
||||
" health:\n\
|
||||
\x20\x20\x20\x20capture_frames: {}\n\
|
||||
\x20\x20\x20\x20zero_frames: {}\n\
|
||||
\x20\x20\x20\x20callbacks_10ms: {}\n\
|
||||
\x20\x20\x20\x20callbacks_20ms: {}\n\
|
||||
\x20\x20\x20\x20callbacks_other: {}\n\
|
||||
\x20\x20\x20\x20callback_xruns: {}\n\
|
||||
\x20\x20\x20\x20clipped_samples: {}\n",
|
||||
stats.capture_frames,
|
||||
stats.zero_frames,
|
||||
stats.callbacks_10ms,
|
||||
stats.callbacks_20ms,
|
||||
stats.callbacks_other,
|
||||
stats.callback_xruns,
|
||||
stats.clipped_samples,
|
||||
));
|
||||
}
|
||||
yaml
|
||||
});
|
||||
match chanora_core::DiagnosticExport::from_sink(log_sink(), metadata) {
|
||||
Ok(exp) => exp
|
||||
.with_android_audio(android_audio_yaml)
|
||||
@@ -1506,6 +1540,11 @@ pub enum BridgeEvent {
|
||||
/// Target scope (server/channel/private/poke).
|
||||
target: BridgeMessageTarget,
|
||||
},
|
||||
/// Human-readable server activity surfaced from protocol bookkeeping events.
|
||||
ServerActivity {
|
||||
/// TeamSpeak-style activity line.
|
||||
message: String,
|
||||
},
|
||||
/// Audio route changed (speaker/earpiece/BT/wired).
|
||||
AudioRouteChanged {
|
||||
/// The new audio route.
|
||||
@@ -1692,6 +1731,9 @@ impl From<chanora_core::SessionEvent> for BridgeEvent {
|
||||
message,
|
||||
target: target.into(),
|
||||
},
|
||||
chanora_core::SessionEvent::ServerActivity { message } => {
|
||||
BridgeEvent::ServerActivity { message }
|
||||
}
|
||||
chanora_core::SessionEvent::AudioRouteChanged { route } => {
|
||||
BridgeEvent::AudioRouteChanged {
|
||||
route: route.into(),
|
||||
@@ -1848,10 +1890,16 @@ pub async fn audio_processing_stats() -> Result<BridgeAudioProcessingStats, Brid
|
||||
/// Audio device info from the platform.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BridgeAudioDevice {
|
||||
/// Stable platform-reported device identifier.
|
||||
pub id: String,
|
||||
/// Human-readable device name.
|
||||
pub name: String,
|
||||
/// Additional device details useful for disambiguation.
|
||||
pub details: String,
|
||||
/// True if the OS reports this as the default device.
|
||||
pub is_default: bool,
|
||||
/// True if Chanora currently has this device pinned.
|
||||
pub is_selected: bool,
|
||||
}
|
||||
|
||||
/// List of available audio devices.
|
||||
@@ -1864,14 +1912,26 @@ pub struct BridgeAudioDeviceList {
|
||||
}
|
||||
|
||||
/// List available audio input and output devices from the platform.
|
||||
pub fn list_audio_devices() -> BridgeAudioDeviceList {
|
||||
let list = chanora_audio::list_audio_devices();
|
||||
BridgeAudioDeviceList {
|
||||
pub async fn list_audio_devices() -> Result<BridgeAudioDeviceList, BridgeError> {
|
||||
let (list, selected_input, selected_output) = runtime()
|
||||
.spawn(async move {
|
||||
let selected_input = session().preferred_input_device().await;
|
||||
let selected_output = session().preferred_output_device().await;
|
||||
let list = chanora_audio::list_audio_devices();
|
||||
(list, selected_input, selected_output)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| task_join_error("list_audio_devices", e))?;
|
||||
|
||||
Ok(BridgeAudioDeviceList {
|
||||
input_devices: list
|
||||
.input_devices
|
||||
.into_iter()
|
||||
.map(|d| BridgeAudioDevice {
|
||||
is_selected: selected_input.as_ref().is_some_and(|id| id == &d.id),
|
||||
id: d.id,
|
||||
name: d.name,
|
||||
details: d.details,
|
||||
is_default: d.is_default,
|
||||
})
|
||||
.collect(),
|
||||
@@ -1879,27 +1939,30 @@ pub fn list_audio_devices() -> BridgeAudioDeviceList {
|
||||
.output_devices
|
||||
.into_iter()
|
||||
.map(|d| BridgeAudioDevice {
|
||||
is_selected: selected_output.as_ref().is_some_and(|id| id == &d.id),
|
||||
id: d.id,
|
||||
name: d.name,
|
||||
details: d.details,
|
||||
is_default: d.is_default,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Set the preferred input device by name. Takes effect on next
|
||||
/// Set the preferred input device by id. Takes effect on next
|
||||
/// `start_audio`.
|
||||
pub async fn set_input_device(name: Option<String>) -> Result<(), BridgeError> {
|
||||
pub async fn set_input_device(id: Option<String>) -> Result<(), BridgeError> {
|
||||
runtime()
|
||||
.spawn(async move { session().set_input_device(name).await })
|
||||
.spawn(async move { session().set_input_device(id).await })
|
||||
.await
|
||||
.map_err(|e| task_join_error("set_input_device", e))??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the preferred output device by name.
|
||||
pub async fn set_output_device(name: Option<String>) -> Result<(), BridgeError> {
|
||||
/// Set the preferred output device by id.
|
||||
pub async fn set_output_device(id: Option<String>) -> Result<(), BridgeError> {
|
||||
runtime()
|
||||
.spawn(async move { session().set_output_device(name).await })
|
||||
.spawn(async move { session().set_output_device(id).await })
|
||||
.await
|
||||
.map_err(|e| task_join_error("set_output_device", e))??;
|
||||
Ok(())
|
||||
@@ -1919,21 +1982,6 @@ pub async fn set_vad_model_path(path: String) -> Result<(), BridgeError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Configure the TEN VAD ONNX model path.
|
||||
pub async fn set_ten_vad_model_path(path: String) -> Result<(), BridgeError> {
|
||||
if path.trim().is_empty() {
|
||||
return Err(BridgeError::InvalidCommand(
|
||||
"ten vad model path must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
runtime()
|
||||
.spawn(async move { chanora_audio::vad::set_ten_model_path(&path) })
|
||||
.await
|
||||
.map_err(|e| task_join_error("set_ten_vad_model_path", e))?
|
||||
.map_err(|e| BridgeError::Unmapped(format!("set_ten_vad_model_path: {e}")))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Enable or disable audio debug WAV dumping.
|
||||
pub async fn enable_audio_debug_wav_dump(enabled: bool) -> Result<(), BridgeError> {
|
||||
runtime()
|
||||
|
||||
@@ -38,7 +38,7 @@ flutter_rust_bridge::frb_generated_boilerplate!(
|
||||
default_rust_auto_opaque = RustAutoOpaqueMoi,
|
||||
);
|
||||
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.12.0";
|
||||
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = 1433826599;
|
||||
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -560177922;
|
||||
|
||||
// Section: executor
|
||||
|
||||
@@ -743,7 +743,7 @@ fn wire__crate__api__list_audio_devices_impl(
|
||||
rust_vec_len_: i32,
|
||||
data_len_: i32,
|
||||
) {
|
||||
FLUTTER_RUST_BRIDGE_HANDLER.wrap_normal::<flutter_rust_bridge::for_generated::SseCodec, _, _>(
|
||||
FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
|
||||
flutter_rust_bridge::for_generated::TaskInfo {
|
||||
debug_name: "list_audio_devices",
|
||||
port: Some(port_),
|
||||
@@ -760,11 +760,14 @@ fn wire__crate__api__list_audio_devices_impl(
|
||||
let mut deserializer =
|
||||
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
|
||||
deserializer.end();
|
||||
move |context| {
|
||||
transform_result_sse::<_, ()>((move || {
|
||||
let output_ok = Result::<_, ()>::Ok(crate::api::list_audio_devices())?;
|
||||
Ok(output_ok)
|
||||
})())
|
||||
move |context| async move {
|
||||
transform_result_sse::<_, crate::BridgeError>(
|
||||
(move || async move {
|
||||
let output_ok = crate::api::list_audio_devices().await?;
|
||||
Ok(output_ok)
|
||||
})()
|
||||
.await,
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -1141,12 +1144,12 @@ fn wire__crate__api__set_input_device_impl(
|
||||
};
|
||||
let mut deserializer =
|
||||
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
|
||||
let api_name = <Option<String>>::sse_decode(&mut deserializer);
|
||||
let api_id = <Option<String>>::sse_decode(&mut deserializer);
|
||||
deserializer.end();
|
||||
move |context| async move {
|
||||
transform_result_sse::<_, crate::BridgeError>(
|
||||
(move || async move {
|
||||
let output_ok = crate::api::set_input_device(api_name).await?;
|
||||
let output_ok = crate::api::set_input_device(api_id).await?;
|
||||
Ok(output_ok)
|
||||
})()
|
||||
.await,
|
||||
@@ -1282,12 +1285,12 @@ fn wire__crate__api__set_output_device_impl(
|
||||
};
|
||||
let mut deserializer =
|
||||
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
|
||||
let api_name = <Option<String>>::sse_decode(&mut deserializer);
|
||||
let api_id = <Option<String>>::sse_decode(&mut deserializer);
|
||||
deserializer.end();
|
||||
move |context| async move {
|
||||
transform_result_sse::<_, crate::BridgeError>(
|
||||
(move || async move {
|
||||
let output_ok = crate::api::set_output_device(api_name).await?;
|
||||
let output_ok = crate::api::set_output_device(api_id).await?;
|
||||
Ok(output_ok)
|
||||
})()
|
||||
.await,
|
||||
@@ -1478,42 +1481,6 @@ fn wire__crate__api__set_release_tail_ms_impl(
|
||||
},
|
||||
)
|
||||
}
|
||||
fn wire__crate__api__set_ten_vad_model_path_impl(
|
||||
port_: flutter_rust_bridge::for_generated::MessagePort,
|
||||
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
|
||||
rust_vec_len_: i32,
|
||||
data_len_: i32,
|
||||
) {
|
||||
FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
|
||||
flutter_rust_bridge::for_generated::TaskInfo {
|
||||
debug_name: "set_ten_vad_model_path",
|
||||
port: Some(port_),
|
||||
mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal,
|
||||
},
|
||||
move || {
|
||||
let message = unsafe {
|
||||
flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(
|
||||
ptr_,
|
||||
rust_vec_len_,
|
||||
data_len_,
|
||||
)
|
||||
};
|
||||
let mut deserializer =
|
||||
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
|
||||
let api_path = <String>::sse_decode(&mut deserializer);
|
||||
deserializer.end();
|
||||
move |context| async move {
|
||||
transform_result_sse::<_, crate::BridgeError>(
|
||||
(move || async move {
|
||||
let output_ok = crate::api::set_ten_vad_model_path(api_path).await?;
|
||||
Ok(output_ok)
|
||||
})()
|
||||
.await,
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
fn wire__crate__api__set_transmit_mode_impl(
|
||||
port_: flutter_rust_bridge::for_generated::MessagePort,
|
||||
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
|
||||
@@ -1783,11 +1750,17 @@ impl SseDecode for crate::api::BridgeAudioBackend {
|
||||
impl SseDecode for crate::api::BridgeAudioDevice {
|
||||
// Codec=Sse (Serialization based), see doc to use other codecs
|
||||
fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self {
|
||||
let mut var_id = <String>::sse_decode(deserializer);
|
||||
let mut var_name = <String>::sse_decode(deserializer);
|
||||
let mut var_details = <String>::sse_decode(deserializer);
|
||||
let mut var_isDefault = <bool>::sse_decode(deserializer);
|
||||
let mut var_isSelected = <bool>::sse_decode(deserializer);
|
||||
return crate::api::BridgeAudioDevice {
|
||||
id: var_id,
|
||||
name: var_name,
|
||||
details: var_details,
|
||||
is_default: var_isDefault,
|
||||
is_selected: var_isSelected,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1859,6 +1832,11 @@ impl SseDecode for crate::api::BridgeAudioProcessingStats {
|
||||
let mut var_outputUnderruns = <u64>::sse_decode(deserializer);
|
||||
let mut var_callbackXruns = <u64>::sse_decode(deserializer);
|
||||
let mut var_clippedSamples = <u64>::sse_decode(deserializer);
|
||||
let mut var_zeroFrames = <u64>::sse_decode(deserializer);
|
||||
let mut var_captureFrames = <u64>::sse_decode(deserializer);
|
||||
let mut var_callbacks10Ms = <u64>::sse_decode(deserializer);
|
||||
let mut var_callbacks20Ms = <u64>::sse_decode(deserializer);
|
||||
let mut var_callbacksOther = <u64>::sse_decode(deserializer);
|
||||
let mut var_sonoraEnabled = <bool>::sse_decode(deserializer);
|
||||
let mut var_platformVoiceProcessingEnabled = <bool>::sse_decode(deserializer);
|
||||
return crate::api::BridgeAudioProcessingStats {
|
||||
@@ -1879,6 +1857,11 @@ impl SseDecode for crate::api::BridgeAudioProcessingStats {
|
||||
output_underruns: var_outputUnderruns,
|
||||
callback_xruns: var_callbackXruns,
|
||||
clipped_samples: var_clippedSamples,
|
||||
zero_frames: var_zeroFrames,
|
||||
capture_frames: var_captureFrames,
|
||||
callbacks_10ms: var_callbacks10Ms,
|
||||
callbacks_20ms: var_callbacks20Ms,
|
||||
callbacks_other: var_callbacksOther,
|
||||
sonora_enabled: var_sonoraEnabled,
|
||||
platform_voice_processing_enabled: var_platformVoiceProcessingEnabled,
|
||||
};
|
||||
@@ -2147,6 +2130,12 @@ impl SseDecode for crate::api::BridgeEvent {
|
||||
};
|
||||
}
|
||||
12 => {
|
||||
let mut var_message = <String>::sse_decode(deserializer);
|
||||
return crate::api::BridgeEvent::ServerActivity {
|
||||
message: var_message,
|
||||
};
|
||||
}
|
||||
13 => {
|
||||
let mut var_route = <crate::api::BridgeAudioRoute>::sse_decode(deserializer);
|
||||
return crate::api::BridgeEvent::AudioRouteChanged { route: var_route };
|
||||
}
|
||||
@@ -2291,10 +2280,9 @@ impl SseDecode for crate::api::BridgeVadBackend {
|
||||
let mut inner = <i32>::sse_decode(deserializer);
|
||||
return match inner {
|
||||
0 => crate::api::BridgeVadBackend::SileroOnnx,
|
||||
1 => crate::api::BridgeVadBackend::TenVad,
|
||||
2 => crate::api::BridgeVadBackend::WebrtcVad,
|
||||
3 => crate::api::BridgeVadBackend::EnergyDebug,
|
||||
4 => crate::api::BridgeVadBackend::Disabled,
|
||||
1 => crate::api::BridgeVadBackend::WebrtcVad,
|
||||
2 => crate::api::BridgeVadBackend::EnergyDebug,
|
||||
3 => crate::api::BridgeVadBackend::Disabled,
|
||||
_ => unreachable!("Invalid variant for BridgeVadBackend: {}", inner),
|
||||
};
|
||||
}
|
||||
@@ -2544,13 +2532,12 @@ fn pde_ffi_dispatcher_primary_impl(
|
||||
39 => wire__crate__api__set_ptt_impl(port, ptr, rust_vec_len, data_len),
|
||||
40 => wire__crate__api__set_ptt_binding_impl(port, ptr, rust_vec_len, data_len),
|
||||
41 => wire__crate__api__set_release_tail_ms_impl(port, ptr, rust_vec_len, data_len),
|
||||
42 => wire__crate__api__set_ten_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
|
||||
43 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
|
||||
44 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
|
||||
45 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len),
|
||||
46 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len),
|
||||
47 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len),
|
||||
48 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len),
|
||||
42 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
|
||||
43 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
|
||||
44 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len),
|
||||
45 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len),
|
||||
46 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len),
|
||||
47 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
@@ -2609,8 +2596,11 @@ impl flutter_rust_bridge::IntoIntoDart<crate::api::BridgeAudioBackend>
|
||||
impl flutter_rust_bridge::IntoDart for crate::api::BridgeAudioDevice {
|
||||
fn into_dart(self) -> flutter_rust_bridge::for_generated::DartAbi {
|
||||
[
|
||||
self.id.into_into_dart().into_dart(),
|
||||
self.name.into_into_dart().into_dart(),
|
||||
self.details.into_into_dart().into_dart(),
|
||||
self.is_default.into_into_dart().into_dart(),
|
||||
self.is_selected.into_into_dart().into_dart(),
|
||||
]
|
||||
.into_dart()
|
||||
}
|
||||
@@ -2697,6 +2687,11 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeAudioProcessingStats {
|
||||
self.output_underruns.into_into_dart().into_dart(),
|
||||
self.callback_xruns.into_into_dart().into_dart(),
|
||||
self.clipped_samples.into_into_dart().into_dart(),
|
||||
self.zero_frames.into_into_dart().into_dart(),
|
||||
self.capture_frames.into_into_dart().into_dart(),
|
||||
self.callbacks_10ms.into_into_dart().into_dart(),
|
||||
self.callbacks_20ms.into_into_dart().into_dart(),
|
||||
self.callbacks_other.into_into_dart().into_dart(),
|
||||
self.sonora_enabled.into_into_dart().into_dart(),
|
||||
self.platform_voice_processing_enabled
|
||||
.into_into_dart()
|
||||
@@ -2973,8 +2968,11 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeEvent {
|
||||
target.into_into_dart().into_dart(),
|
||||
]
|
||||
.into_dart(),
|
||||
crate::api::BridgeEvent::ServerActivity { message } => {
|
||||
[12.into_dart(), message.into_into_dart().into_dart()].into_dart()
|
||||
}
|
||||
crate::api::BridgeEvent::AudioRouteChanged { route } => {
|
||||
[12.into_dart(), route.into_into_dart().into_dart()].into_dart()
|
||||
[13.into_dart(), route.into_into_dart().into_dart()].into_dart()
|
||||
}
|
||||
_ => {
|
||||
unimplemented!("");
|
||||
@@ -3170,10 +3168,9 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeVadBackend {
|
||||
fn into_dart(self) -> flutter_rust_bridge::for_generated::DartAbi {
|
||||
match self {
|
||||
Self::SileroOnnx => 0.into_dart(),
|
||||
Self::TenVad => 1.into_dart(),
|
||||
Self::WebrtcVad => 2.into_dart(),
|
||||
Self::EnergyDebug => 3.into_dart(),
|
||||
Self::Disabled => 4.into_dart(),
|
||||
Self::WebrtcVad => 1.into_dart(),
|
||||
Self::EnergyDebug => 2.into_dart(),
|
||||
Self::Disabled => 3.into_dart(),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
@@ -3313,8 +3310,11 @@ impl SseEncode for crate::api::BridgeAudioBackend {
|
||||
impl SseEncode for crate::api::BridgeAudioDevice {
|
||||
// Codec=Sse (Serialization based), see doc to use other codecs
|
||||
fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) {
|
||||
<String>::sse_encode(self.id, serializer);
|
||||
<String>::sse_encode(self.name, serializer);
|
||||
<String>::sse_encode(self.details, serializer);
|
||||
<bool>::sse_encode(self.is_default, serializer);
|
||||
<bool>::sse_encode(self.is_selected, serializer);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3368,6 +3368,11 @@ impl SseEncode for crate::api::BridgeAudioProcessingStats {
|
||||
<u64>::sse_encode(self.output_underruns, serializer);
|
||||
<u64>::sse_encode(self.callback_xruns, serializer);
|
||||
<u64>::sse_encode(self.clipped_samples, serializer);
|
||||
<u64>::sse_encode(self.zero_frames, serializer);
|
||||
<u64>::sse_encode(self.capture_frames, serializer);
|
||||
<u64>::sse_encode(self.callbacks_10ms, serializer);
|
||||
<u64>::sse_encode(self.callbacks_20ms, serializer);
|
||||
<u64>::sse_encode(self.callbacks_other, serializer);
|
||||
<bool>::sse_encode(self.sonora_enabled, serializer);
|
||||
<bool>::sse_encode(self.platform_voice_processing_enabled, serializer);
|
||||
}
|
||||
@@ -3595,8 +3600,12 @@ impl SseEncode for crate::api::BridgeEvent {
|
||||
<String>::sse_encode(message, serializer);
|
||||
<crate::api::BridgeMessageTarget>::sse_encode(target, serializer);
|
||||
}
|
||||
crate::api::BridgeEvent::AudioRouteChanged { route } => {
|
||||
crate::api::BridgeEvent::ServerActivity { message } => {
|
||||
<i32>::sse_encode(12, serializer);
|
||||
<String>::sse_encode(message, serializer);
|
||||
}
|
||||
crate::api::BridgeEvent::AudioRouteChanged { route } => {
|
||||
<i32>::sse_encode(13, serializer);
|
||||
<crate::api::BridgeAudioRoute>::sse_encode(route, serializer);
|
||||
}
|
||||
_ => {
|
||||
@@ -3734,10 +3743,9 @@ impl SseEncode for crate::api::BridgeVadBackend {
|
||||
<i32>::sse_encode(
|
||||
match self {
|
||||
crate::api::BridgeVadBackend::SileroOnnx => 0,
|
||||
crate::api::BridgeVadBackend::TenVad => 1,
|
||||
crate::api::BridgeVadBackend::WebrtcVad => 2,
|
||||
crate::api::BridgeVadBackend::EnergyDebug => 3,
|
||||
crate::api::BridgeVadBackend::Disabled => 4,
|
||||
crate::api::BridgeVadBackend::WebrtcVad => 1,
|
||||
crate::api::BridgeVadBackend::EnergyDebug => 2,
|
||||
crate::api::BridgeVadBackend::Disabled => 3,
|
||||
_ => {
|
||||
unimplemented!("");
|
||||
}
|
||||
|
||||
@@ -659,17 +659,11 @@ async fn connection_task(
|
||||
..
|
||||
} = &ev
|
||||
{
|
||||
let own_client = con
|
||||
.get_state()
|
||||
.ok()
|
||||
.map(|state| state.own_client);
|
||||
let own_client = con.get_state().ok().map(|state| state.own_client);
|
||||
if own_client == Some(*client_id) {
|
||||
let current_channel = con
|
||||
.get_state()
|
||||
.ok()
|
||||
.and_then(|state| {
|
||||
state.clients.get(client_id).map(|client| client.channel.0)
|
||||
});
|
||||
let current_channel = con.get_state().ok().and_then(|state| {
|
||||
state.clients.get(client_id).map(|client| client.channel.0)
|
||||
});
|
||||
if let Some(current_channel) = current_channel {
|
||||
let matched: Vec<MessageHandle> = pending_moves
|
||||
.iter()
|
||||
@@ -682,7 +676,9 @@ async fn connection_task(
|
||||
})
|
||||
.collect();
|
||||
for handle in matched {
|
||||
if let Some((_, reply, _)) = pending_moves.remove(&handle) {
|
||||
if let Some((_, reply, _)) =
|
||||
pending_moves.remove(&handle)
|
||||
{
|
||||
info!(
|
||||
target: "chanora_protocol",
|
||||
channel_id = current_channel,
|
||||
@@ -727,7 +723,9 @@ async fn connection_task(
|
||||
}
|
||||
}
|
||||
StreamItem::MessageResult(handle, result) => {
|
||||
if let Some((_target_channel, reply, _deadline)) = pending_moves.remove(&handle) {
|
||||
if let Some((_target_channel, reply, _deadline)) =
|
||||
pending_moves.remove(&handle)
|
||||
{
|
||||
let mapped = match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err(cmd_err) => {
|
||||
@@ -814,36 +812,32 @@ async fn connection_task(
|
||||
channel_id,
|
||||
password,
|
||||
reply,
|
||||
}) => {
|
||||
match move_self_to(&mut con, channel_id, password.as_deref()) {
|
||||
Ok(handle) => {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(3);
|
||||
pending_moves.insert(handle, (channel_id, Some(reply), deadline));
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = reply.send(Err(e));
|
||||
}
|
||||
}) => match move_self_to(&mut con, channel_id, password.as_deref()) {
|
||||
Ok(handle) => {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(3);
|
||||
pending_moves.insert(handle, (channel_id, Some(reply), deadline));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = reply.send(Err(e));
|
||||
}
|
||||
},
|
||||
Ok(Request::MoveToChannelNoWait {
|
||||
channel_id,
|
||||
password,
|
||||
}) => {
|
||||
match move_self_to(&mut con, channel_id, password.as_deref()) {
|
||||
Ok(handle) => {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(3);
|
||||
pending_moves.insert(handle, (channel_id, None, deadline));
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "chanora_protocol",
|
||||
error = %e,
|
||||
channel_id,
|
||||
"fire-and-forget client_move could not be queued"
|
||||
);
|
||||
}
|
||||
}) => match move_self_to(&mut con, channel_id, password.as_deref()) {
|
||||
Ok(handle) => {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(3);
|
||||
pending_moves.insert(handle, (channel_id, None, deadline));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "chanora_protocol",
|
||||
error = %e,
|
||||
channel_id,
|
||||
"fire-and-forget client_move could not be queued"
|
||||
);
|
||||
}
|
||||
},
|
||||
Ok(Request::SetMuted {
|
||||
input,
|
||||
output,
|
||||
@@ -1261,10 +1255,7 @@ fn activity_channel_group_name(
|
||||
.map(|group| quoted(&group.name))
|
||||
}
|
||||
|
||||
fn activity_server_group_name(
|
||||
con: &Connection,
|
||||
id: tsclientlib::ServerGroupId,
|
||||
) -> Option<String> {
|
||||
fn activity_server_group_name(con: &Connection, id: tsclientlib::ServerGroupId) -> Option<String> {
|
||||
con.get_state()
|
||||
.ok()
|
||||
.and_then(|state| state.server_groups.get(&id))
|
||||
@@ -1282,7 +1273,11 @@ fn format_server_activity(con: &Connection, ev: &tsclientlib::events::Event) ->
|
||||
use tsproto_types::Reason;
|
||||
|
||||
match ev {
|
||||
Event::PropertyAdded { id: PropertyId::Client(client_id), extra, .. } => {
|
||||
Event::PropertyAdded {
|
||||
id: PropertyId::Client(client_id),
|
||||
extra,
|
||||
..
|
||||
} => {
|
||||
if extra.reason.is_none() {
|
||||
return None;
|
||||
}
|
||||
@@ -1294,7 +1289,12 @@ fn format_server_activity(con: &Connection, ev: &tsclientlib::events::Event) ->
|
||||
channel
|
||||
))
|
||||
}
|
||||
Event::PropertyRemoved { id: PropertyId::Client(_), old, extra, .. } => {
|
||||
Event::PropertyRemoved {
|
||||
id: PropertyId::Client(_),
|
||||
old,
|
||||
extra,
|
||||
..
|
||||
} => {
|
||||
let PropertyValue::Client(client) = old else {
|
||||
return None;
|
||||
};
|
||||
@@ -1305,12 +1305,9 @@ fn format_server_activity(con: &Connection, ev: &tsclientlib::events::Event) ->
|
||||
Some(Reason::Clientdisconnect) => {
|
||||
Some(format!("{} disconnected (Leaving)", quoted(&client.name)))
|
||||
}
|
||||
Some(Reason::ClientdisconnectServerShutdown) | Some(Reason::Serverstop) => {
|
||||
Some(format!(
|
||||
"{} disconnected (server shutdown)",
|
||||
quoted(&client.name)
|
||||
))
|
||||
}
|
||||
Some(Reason::ClientdisconnectServerShutdown) | Some(Reason::Serverstop) => Some(
|
||||
format!("{} disconnected (server shutdown)", quoted(&client.name)),
|
||||
),
|
||||
_ => Some(format!(
|
||||
"{} dropped (connection lost)",
|
||||
quoted(&client.name)
|
||||
@@ -1329,8 +1326,7 @@ fn format_server_activity(con: &Connection, ev: &tsclientlib::events::Event) ->
|
||||
let client = activity_client(con, *client_id)?;
|
||||
let from = activity_channel_name(con, *from_channel_id)?;
|
||||
let to = activity_channel_name(con, client.channel)?;
|
||||
if invoker.as_ref().map(|invoker| invoker.id) == Some(*client_id) || invoker.is_none()
|
||||
{
|
||||
if invoker.as_ref().map(|invoker| invoker.id) == Some(*client_id) || invoker.is_none() {
|
||||
Some(format!(
|
||||
"{} switched from channel {} to {}",
|
||||
quoted(&client.name),
|
||||
|
||||
Reference in New Issue
Block a user