edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate chat voice and diagnostics client

Browse Source
This commit is contained in:
Edison Jwa
2026-05-23 06:51:55 +09:00
parent 7e28791ec2
commit 7d5d8c2c90
93 changed files with 10273 additions and 3347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/ios_voice_unit.rs
+55 -59
View File
@@ -81,9 +81,8 @@ use coreaudio::audio_unit::IOType;
use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat};
use tokio::sync::mpsc;
use tracing::{debug, error, info, warn};
use tsclientlib::audio::AudioHandler;
use crate::engine::SessionAudioId;
use crate::mobile_voice_backend::VoiceAudioParams;
use crate::AudioError;
use chanora_protocol::OutPacket;
@@ -152,6 +151,8 @@ struct IosCaptureState {
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
/// Last observed configured TEN model epoch.
ten_model_epoch: u64,
fallback_warned_backend: Option<crate::VadBackend>,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
@@ -172,16 +173,8 @@ impl IosCaptureState {
/// Encoder configuration is the same as cpal-side
/// `try_open_capture` (engine.rs) so audio quality is platform-
/// neutral.
#[allow(clippy::too_many_arguments)]
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
params: &VoiceAudioParams,
wav_recorder: Arc<Mutex<Option<Arc<crate::debug_wav::WavDebugRecorder>>>>,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("ios VPIO")?;
@@ -190,22 +183,23 @@ impl IosCaptureState {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
output_muted,
frames_sent,
mic_gain,
voice_activity_selector,
voice_out_tx: params.voice_out_tx.clone(),
transmit_active: params.transmit_active.clone(),
output_muted: params.output_muted.clone(),
frames_sent: params.frames_sent.clone(),
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
audio_processing_config: params.audio_processing_config.clone(),
sonora_processor: crate::processor::SonoraProcessor::new(),
audio_processing_stats,
audio_processing_stats: params.audio_processing_stats.clone(),
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
pre_roll_buf: [[0_i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
@@ -222,11 +216,20 @@ impl IosCaptureState {
return;
}
self.fallback_warned_backend = Some(failed_backend);
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
if self.capture_frame_seq < 128 {
tracing::info!(
target: "chanora_audio",
backend = failed_backend.as_str(),
seq = self.capture_frame_seq,
"VAD backend warming up; using WebRTC fallback"
);
} else {
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
}
/// Consume the i16 mono buffer delivered by VPIO, accumulate
@@ -360,8 +363,11 @@ impl IosCaptureState {
// Switch VAD backend when the config changes.
let silero_model_epoch = crate::vad::silero_model_epoch();
let ten_model_epoch = crate::vad::ten_model_epoch();
let silero_model_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_model_epoch != self.silero_model_epoch;
let ten_model_changed =
vad_backend == crate::VadBackend::TenVad && ten_model_epoch != self.ten_model_epoch;
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
if debug_wav_dump_enabled {
@@ -376,10 +382,11 @@ impl IosCaptureState {
}
}
if vad_backend != self.current_vad_backend || silero_model_changed {
if vad_backend != self.current_vad_backend || silero_model_changed || ten_model_changed {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
self.silero_model_epoch = silero_model_epoch;
self.ten_model_epoch = ten_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
// Attempt to load Silero model from the well-known
@@ -467,12 +474,17 @@ impl IosCaptureState {
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
@@ -486,12 +498,17 @@ impl IosCaptureState {
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
@@ -617,19 +634,7 @@ impl IosVoiceUnit {
///
/// Capture wiring landed in commit 3; playback wiring landed
/// in commit 4. Route-change observation is commit 5.
#[allow(clippy::too_many_arguments)]
pub fn start(
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<Self, AudioError> {
pub(crate) fn start(params: VoiceAudioParams) -> Result<Self, AudioError> {
// Construct the VoiceProcessingIO AudioUnit. cpal exposes
// `Default::default()` which on iOS picks the inferior
// RemoteIO unit; we explicitly pick VPIO. `coreaudio-rs`
@@ -721,7 +726,7 @@ impl IosVoiceUnit {
// because the input callback is the sole writer/reader on
// the audio thread.
let wav_recorder = Arc::new(Mutex::new({
let cfg = audio_processing_config.lock().unwrap().clone();
let cfg = params.audio_processing_config.lock().unwrap().clone();
if cfg.debug_wav_dump_enabled {
Some(crate::debug_wav::WavDebugRecorder::start(
cfg.route,
@@ -731,17 +736,7 @@ impl IosVoiceUnit {
None
}
}));
let mut capture_state = IosCaptureState::new(
voice_out_tx,
transmit_active,
output_muted.clone(),
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats.clone(),
wav_recorder.clone(),
)?;
let mut capture_state = IosCaptureState::new(&params, wav_recorder.clone())?;
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
// VPIO with our pinned stream format delivers
@@ -806,9 +801,10 @@ impl IosVoiceUnit {
// buffer. Same as Linux/SDL, just stereo-f32 -> mono-i16
// converted at the boundary.
let mut scratch_stereo: Vec<f32> = Vec::with_capacity(2048);
let handler_for_render = handler.clone();
let output_gain_for_render = output_gain.clone();
let output_muted_for_render = output_muted.clone();
let handler_for_render = params.handler.clone();
let output_gain_for_render = params.output_gain.clone();
let output_muted_for_render = params.output_muted.clone();
let audio_processing_stats_for_render = params.audio_processing_stats.clone();
let wav_recorder_for_render = wav_recorder.clone();
// Diagnostic counters (sampled every 100 callbacks ~= 2 s).
let mut cb_count: u64 = 0;
@@ -841,7 +837,7 @@ impl IosVoiceUnit {
let _removed = h.fill_buffer(&mut scratch_stereo[..needed]);
}
Err(std::sync::TryLockError::WouldBlock) => {
audio_processing_stats.increment_callback_xrun();
audio_processing_stats_for_render.increment_callback_xrun();
// scratch_stereo is already zeroed above.
}
Err(std::sync::TryLockError::Poisoned(e)) => {
@@ -859,9 +855,9 @@ impl IosVoiceUnit {
muted,
);
if mix_stats.clipped_samples > 0 {
audio_processing_stats.add_clipped_samples(mix_stats.clipped_samples);
audio_processing_stats_for_render.add_clipped_samples(mix_stats.clipped_samples);
}
audio_processing_stats.update_render(
audio_processing_stats_for_render.update_render(
crate::frame::dbfs(&scratch_stereo[..needed]),
num_frames as u32,
);
@@ -895,7 +891,7 @@ impl IosVoiceUnit {
if mix_stats.peak_i16 > 0 {
callbacks_with_audio = callbacks_with_audio.wrapping_add(1);
} else {
audio_processing_stats.increment_output_underrun();
audio_processing_stats_for_render.increment_output_underrun();
callbacks_with_silence = callbacks_with_silence.wrapping_add(1);
}