edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): add iOS VAD runtime support

Browse Source
This commit is contained in:
Edison Jwa
2026-05-21 20:51:45 +09:00
parent 171baf6e41
commit 6af4ecab0f
73 changed files with 11529 additions and 1249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/ios_voice_unit.rs
+463 -122
View File
@@ -75,10 +75,6 @@ use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use audiopus::coder::Encoder as OpusEncoder;
use audiopus::{
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
SampleRate as OpusSampleRate,
};
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
use coreaudio::audio_unit::render_callback::{self, data};
use coreaudio::audio_unit::IOType;
@@ -89,22 +85,7 @@ use tsclientlib::audio::AudioHandler;
use crate::engine::SessionAudioId;
use crate::AudioError;
use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
/// 20 ms at 48 kHz mono — one Opus frame's worth of samples.
/// Aligning the AudioUnit IO buffer to this frame size keeps the
/// jitter-buffer / encoder handshake tight (no fractional-frame
/// reads inside fill_buffer or accumulator drift inside the
/// capture pipeline).
const FRAME_SAMPLES_MONO: usize = 960;
/// Maximum size of an encoded Opus frame in bytes (per RFC 6716
/// §3.2.1). Same constant the cpal-side `CaptureState` uses; we
/// duplicate it here instead of cross-importing from engine.rs
/// because engine.rs's copy is cfg-gated to non-iOS for cpal-only
/// reasons. Post-step-5 review may dedupe by promoting both to a
/// shared `crate::framing` module.
const MAX_OPUS_FRAME: usize = 1275;
use chanora_protocol::OutPacket;
/// Sample rate every layer above us assumes. Matches the Opus
/// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the
@@ -124,6 +105,11 @@ const OUTPUT_BUS: Element = Element::Output;
/// samples in.
const INPUT_BUS: Element = Element::Input;
/// Pre-roll buffer capacity: 160 ms / 10 ms = 16 frames.
/// Stores processed i16 frames so the first syllable is not lost
/// when the VAD gate opens (VAD_004 / pre_roll_ms=160).
const PRE_ROLL_FRAMES: usize = 16;
/// Capture pipeline state owned by the VPIO input callback. The
/// AudioUnit hands us 48 kHz signed-int16 mono PCM directly (no
/// downmix or resample needed — VPIO's hardware-side mix-down
@@ -146,17 +132,36 @@ const INPUT_BUS: Element = Element::Input;
/// shared with `AudioEngine`.
struct IosCaptureState {
encoder: OpusEncoder,
/// 48 kHz mono PCM scratch accumulating to FRAME_SAMPLES_MONO
/// 48 kHz mono PCM scratch accumulating to FRAME_20MS_SAMPLES
/// per encode. Capacity 2x to absorb cpal-style buffer-size
/// jitter without reallocating.
pcm_accum: Vec<i16>,
opus_out: [u8; MAX_OPUS_FRAME],
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
/// PTT transmission gate. Read once per outbound frame; this
/// struct never mutates the flag (SAD-075 / SDD-089).
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
/// Background Silero worker — enqueues frames off the realtime
/// callback and publishes the latest probability atomically.
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
sonora_processor: crate::processor::SonoraProcessor,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: usize,
pre_roll_buf: [[i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
pre_roll_head: usize,
pre_roll_count: usize,
pre_roll_flushed: bool,
capture_frame_seq: u64,
wav_recorder: Arc<Mutex<Option<Arc<crate::debug_wav::WavDebugRecorder>>>>,
}
impl IosCaptureState {
@@ -164,54 +169,55 @@ impl IosCaptureState {
/// Encoder configuration is the same as cpal-side
/// `try_open_capture` (engine.rs) so audio quality is platform-
/// neutral.
#[allow(clippy::too_many_arguments)]
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
wav_recorder: Arc<Mutex<Option<Arc<crate::debug_wav::WavDebugRecorder>>>>,
) -> Result<Self, AudioError> {
let mut encoder =
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.map_err(|e| AudioError::Opus(format!("encoder new (ios): {e}")))?;
// VoIP-tuned settings — bitrate 32 kbps, complexity 10,
// inband FEC on, packet-loss-perc 5. Soft-fail each setter
// with a warn log to match the cpal-side behaviour: an
// unusual libopus build that rejects one setter shouldn't
// tank the whole pipeline. Full rationale + RFC citations
// are in engine.rs::try_open_capture line ~640.
if let Err(e) = encoder.set_bitrate(OpusBitrate::BitsPerSecond(32_000)) {
warn!(target: "chanora_audio", error = %e, "opus(ios): set_bitrate(32000) failed");
}
if let Err(e) = encoder.set_complexity(10) {
warn!(target: "chanora_audio", error = %e, "opus(ios): set_complexity(10) failed");
}
if let Err(e) = encoder.set_inband_fec(true) {
warn!(target: "chanora_audio", error = %e, "opus(ios): set_inband_fec(true) failed");
}
if let Err(e) = encoder.set_packet_loss_perc(5) {
warn!(target: "chanora_audio", error = %e, "opus(ios): set_packet_loss_perc(5) failed");
}
info!(
target: "chanora_audio",
bitrate_bps = 32_000,
complexity = 10,
inband_fec = true,
packet_loss_perc = 5,
"ios VPIO opus encoder tuned for VoIP"
);
let encoder = crate::opus_voice::new_voip_encoder("ios VPIO")?;
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(FRAME_SAMPLES_MONO * 2),
opus_out: [0u8; MAX_OPUS_FRAME],
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
frames_sent,
mic_gain,
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
sonora_processor: crate::processor::SonoraProcessor::new(),
audio_processing_stats,
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
pre_roll_buf: [[0_i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
pre_roll_head: 0,
pre_roll_count: 0,
pre_roll_flushed: false,
capture_frame_seq: 0,
wav_recorder,
})
}
fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) {
if let Ok(mut cfg) = self.audio_processing_config.try_lock() {
if cfg.disable_failed_vad_backend(failed_backend) {
self.current_vad_backend = crate::VadBackend::WebrtcVad;
}
}
}
/// Consume the i16 mono buffer delivered by VPIO, accumulate
/// to a 20 ms frame boundary, encode + send when PTT is held.
///
@@ -220,6 +226,22 @@ impl IosCaptureState {
/// In practice "interleaved mono" is the same byte layout as
/// "planar mono" so we just take the buffer as-is.
fn ingest_i16(&mut self, samples: &[i16]) {
let mut offset = 0;
while offset < samples.len() {
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let take = remaining.min(samples.len() - offset);
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
.copy_from_slice(&samples[offset..offset + take]);
self.pending_10ms_len += take;
offset += take;
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
let frame = self.pending_10ms;
self.process_10ms_capture_frame(&frame);
self.pending_10ms_len = 0;
}
}
if !self.transmit_active.load(Ordering::Relaxed) {
// Drain accumulator while muted so we don't pop on the
// PTT release edge. Matches cpal-side behaviour.
@@ -227,63 +249,40 @@ impl IosCaptureState {
return;
}
// Mic-gain application. When gain==1.0 we skip the
// multiply + saturate loop entirely — that's the common
// case and the loop is the inner-most hot path of the
// realtime audio thread.
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
self.pcm_accum.extend_from_slice(samples);
} else {
let gain = self.mic_gain;
self.pcm_accum.extend(samples.iter().map(|&s| {
// Saturating mul-then-cast keeps the signal in
// the i16 envelope. Clipping in this branch is
// expected — if the user pushed mic_gain past 1.0
// and is shouting, the alternative is wrap-around
// distortion which sounds far worse.
let scaled = (s as f32) * gain;
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
}));
}
// Drain complete 20 ms frames out of the accumulator, encode
// each, send the resulting Opus packet on the protocol
// queue. The `while` covers the case where a single VPIO
// callback delivers more than one frame's worth (rare on
// iOS where the HW IO buffer duration aligns with the
// Opus frame, but always possible during route changes).
while self.pcm_accum.len() >= FRAME_SAMPLES_MONO {
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
// Use a stack-allocated frame buffer to avoid the
// per-callback allocation a `drain(..N).collect()`
// would incur. The encoder doesn't need ownership.
let mut frame = [0i16; FRAME_SAMPLES_MONO];
frame.copy_from_slice(&self.pcm_accum[..FRAME_SAMPLES_MONO]);
self.pcm_accum.drain(..FRAME_SAMPLES_MONO);
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &self.opus_out[..len],
});
match self.voice_out_tx.try_send(packet) {
Ok(()) => {
self.frames_sent.fetch_add(1, Ordering::Relaxed);
}
Err(mpsc::error::TrySendError::Full(_)) => {
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.frames_sent,
&self.opus_out,
len,
|| {
warn!(
target: "chanora_audio",
"ios VPIO: voice_out queue full; dropping frame"
);
}
Err(mpsc::error::TrySendError::Closed(_)) => {
},
|| {
debug!(
target: "chanora_audio",
"ios VPIO: voice_out closed; capture pipeline stopping"
);
}
}
},
);
}
Err(e) => {
error!(target: "chanora_audio", error = %e, "ios VPIO opus encode failed");
@@ -291,6 +290,260 @@ impl IosCaptureState {
}
}
}
fn process_10ms_capture_frame(&mut self, samples: &[i16; crate::frame::FRAME_10MS_SAMPLES]) {
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
*dst = crate::frame::i16_to_f32(src);
}
let input_dbfs = crate::frame::dbfs(&frame);
// WAV tap: raw mic (before processing, DIAG_002).
if let Ok(guard) = self.wav_recorder.try_lock() {
if let Some(rec) = guard.as_ref() {
rec.push_raw_mic(&frame);
}
}
// Read config once per frame (try_lock: non-blocking, falls back to
// last-known values if the lock is contended — safe to miss one frame).
let (
run_ns,
run_agc,
run_hpf,
vad_backend,
vad_hangover,
debug_wav_dump_enabled,
route,
processing_backend,
) = self
.audio_processing_config
.try_lock()
.map(|cfg| {
let ns =
cfg.ns != crate::EffectOwner::Off && cfg.ns != crate::EffectOwner::Platform;
let agc =
cfg.agc != crate::EffectOwner::Off && cfg.agc != crate::EffectOwner::Platform;
let hpf = cfg.hpf_enabled;
(
ns,
agc,
hpf,
cfg.vad_backend,
cfg.vad_hangover_ms,
cfg.debug_wav_dump_enabled,
cfg.route,
cfg.processing_backend,
)
})
.unwrap_or((
false,
false,
true,
crate::VadBackend::SileroOnnx,
crate::voice_activity::VAD_HANGOVER_MS,
false,
crate::AudioRoute::Unknown,
crate::AudioBackend::PlatformVoiceProcessing,
));
// Switch VAD backend when the config changes.
let silero_model_epoch = crate::vad::silero_model_epoch();
let silero_model_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_model_epoch != self.silero_model_epoch;
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
if debug_wav_dump_enabled {
if recorder_guard.is_none() {
*recorder_guard = Some(crate::debug_wav::WavDebugRecorder::start(
route,
processing_backend,
));
}
} else if let Some(recorder) = recorder_guard.take() {
recorder.stop();
}
}
if vad_backend != self.current_vad_backend || silero_model_changed {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
// Attempt to load Silero model from the well-known
// bundle path. The actual inference runs on a
// background worker; the callback only enqueues
// 10 ms frames and falls back to WebRTC if the
// worker is missing or stale.
let model_path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path);
if self.silero_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"Silero VAD model not found at {model_path}; falling back to WebRTC VAD"
);
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
}
self.audio_processing_stats
.set_vad_fallback_active(self.silero_vad_worker.is_none());
}
crate::VadBackend::TenVad => {
self.silero_vad_worker = None;
warn!(
target: "chanora_audio",
"TEN VAD selected but native TEN runtime is not bundled; falling back to WebRTC VAD"
);
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
self.audio_processing_stats.set_vad_fallback_active(true);
}
_ => {
self.silero_vad_worker = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
}
// Reset VAD state machine timers on backend switch.
self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
self.vad_state.reset();
}
// Keep the VAD state machine aligned with the active config.
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
// Apply the enabled stages through the SonoraProcessor.
// We reconfigure it on-the-fly to match the current settings.
if run_ns || run_agc || run_hpf {
use crate::processor::sonora::SonoraConfig;
use crate::processor::AudioProcessor;
let new_cfg = SonoraConfig {
hpf: run_hpf,
aec3: false, // NEVER in VPIO path (INV_009)
ns: run_ns,
agc2: run_agc,
};
if new_cfg != *self.sonora_processor.config() {
self.sonora_processor.apply_config(new_cfg);
}
self.sonora_processor.process_capture(&mut frame);
}
// VAD: use Silero if loaded, otherwise WebRTC fallback.
// Disabled backend → always open (Continuous-like for VAD mode).
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
if worker.try_send(capture_seq, &frame) && !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else {
used_fallback_vad = true;
self.silero_vad_worker = None;
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
used_fallback_vad = true;
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let gate_open = self.vad_state.update(vad.speech);
if let Some(selector) = &self.voice_activity_selector {
selector.set_voice_activity_open(gate_open);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
gate_open,
transmit_active,
);
// WAV tap: processed mic (after Rust DSP, DIAG_002).
if let Ok(guard) = self.wav_recorder.try_lock() {
if let Some(rec) = guard.as_ref() {
rec.push_processed_mic(&frame);
}
}
// Convert to i16 for accumulation.
let mut pcm_frame = [0_i16; crate::frame::FRAME_10MS_SAMPLES];
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) {
*dst = crate::frame::f32_to_i16(src);
}
} else {
let gain = self.mic_gain;
for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) {
let scaled = crate::frame::f32_to_i16(src) as f32 * gain;
*dst = scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16;
}
}
// Update pre-roll ring buffer (VAD_004: preserve first syllable).
let slot_idx = self.pre_roll_head % PRE_ROLL_FRAMES;
self.pre_roll_buf[slot_idx] = pcm_frame;
self.pre_roll_head = (self.pre_roll_head + 1) % PRE_ROLL_FRAMES;
if self.pre_roll_count < PRE_ROLL_FRAMES {
self.pre_roll_count += 1;
}
// If the transmit gate just opened and we haven't flushed the
// pre-roll yet, drain it into the accumulator.
if transmit_active && !self.pre_roll_flushed {
self.pre_roll_flushed = true;
// The oldest frame in the ring is at
// (pre_roll_head + PRE_ROLL_FRAMES - pre_roll_count) % PRE_ROLL_FRAMES.
// We emit frames in chronological order (oldest first), excluding
// the frame we just wrote (which goes into pcm_accum normally below).
let oldest =
(self.pre_roll_head + PRE_ROLL_FRAMES - self.pre_roll_count) % PRE_ROLL_FRAMES;
// Emit pre_roll_count - 1 frames (the -1 excludes the current frame
// which will be added below in the normal path).
let pre_roll_to_emit = self.pre_roll_count.saturating_sub(1);
for i in 0..pre_roll_to_emit {
let idx = (oldest + i) % PRE_ROLL_FRAMES;
self.pcm_accum.extend_from_slice(&self.pre_roll_buf[idx]);
}
} else if !transmit_active {
// Gate closed — reset the flush flag so pre-roll fires again
// on the next gate open.
self.pre_roll_flushed = false;
}
if !transmit_active {
return;
}
self.pcm_accum.extend_from_slice(&pcm_frame);
}
}
/// Live iOS audio unit wrapper. Construct + start = audio
@@ -339,6 +592,9 @@ impl IosVoiceUnit {
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<Self, AudioError> {
// Construct the VoiceProcessingIO AudioUnit. cpal exposes
// `Default::default()` which on iOS picks the inferior
@@ -430,8 +686,27 @@ impl IosVoiceUnit {
// scratch are owned by the closure — no Mutex needed
// because the input callback is the sole writer/reader on
// the audio thread.
let mut capture_state =
IosCaptureState::new(voice_out_tx, transmit_active, frames_sent, mic_gain)?;
let wav_recorder = Arc::new(Mutex::new({
let cfg = audio_processing_config.lock().unwrap().clone();
if cfg.debug_wav_dump_enabled {
Some(crate::debug_wav::WavDebugRecorder::start(
cfg.route,
cfg.processing_backend,
))
} else {
None
}
}));
let mut capture_state = IosCaptureState::new(
voice_out_tx,
transmit_active,
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats.clone(),
wav_recorder.clone(),
)?;
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
// VPIO with our pinned stream format delivers
@@ -499,12 +774,16 @@ impl IosVoiceUnit {
let handler_for_render = handler.clone();
let output_gain_for_render = output_gain.clone();
let output_muted_for_render = output_muted.clone();
let wav_recorder_for_render = wav_recorder.clone();
// Diagnostic counters (sampled every 100 callbacks ~= 2 s).
let mut cb_count: u64 = 0;
let mut last_num_frames: usize = 0;
let mut num_frames_changes: u32 = 0;
let mut callbacks_with_audio: u64 = 0;
let mut callbacks_with_silence: u64 = 0;
let mut render_ref_accum = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
let mut render_ref_len: usize = 0;
let mut render_recorder_active = false;
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
let out: &mut [i16] = args.data.buffer;
let num_frames = out.len();
@@ -527,6 +806,7 @@ impl IosVoiceUnit {
let _removed = h.fill_buffer(&mut scratch_stereo[..needed]);
}
Err(std::sync::TryLockError::WouldBlock) => {
audio_processing_stats.increment_callback_xrun();
// scratch_stereo is already zeroed above.
}
Err(std::sync::TryLockError::Poisoned(e)) => {
@@ -535,34 +815,52 @@ impl IosVoiceUnit {
}
}
// Downmix stereo f32 -> mono i16 with master gain.
// (l + r) * 0.5 preserves total signal energy with
// 3 dB headroom against sum-of-correlated-peaks
// clipping. Hard-clip i16 cast at the boundary.
let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed));
let muted = output_muted_for_render.load(Ordering::Relaxed);
let mut peak_out: i16 = 0;
for (i, dst) in out.iter_mut().enumerate() {
if muted {
*dst = 0;
continue;
}
let l = scratch_stereo[i * 2];
let r = scratch_stereo[i * 2 + 1];
let mono_f32 = (l + r) * 0.5 * gain;
let clamped = mono_f32.clamp(-1.0, 1.0);
let sample = (clamped * i16::MAX as f32) as i16;
*dst = sample;
let a = sample.unsigned_abs() as i16;
if a > peak_out {
peak_out = a;
let mix_stats = crate::voice_render::downmix_stereo_f32_to_mono_i16(
&scratch_stereo[..needed],
out,
gain,
muted,
);
if mix_stats.clipped_samples > 0 {
audio_processing_stats.add_clipped_samples(mix_stats.clipped_samples);
}
audio_processing_stats.update_render(
crate::frame::dbfs(&scratch_stereo[..needed]),
num_frames as u32,
);
if let Ok(guard) = wav_recorder_for_render.try_lock() {
if let Some(rec) = guard.as_ref() {
if !render_recorder_active {
render_ref_len = 0;
render_ref_accum.fill(0.0);
render_recorder_active = true;
}
let mut idx = 0;
while idx + 1 < needed {
let mono = (scratch_stereo[idx] + scratch_stereo[idx + 1]) * 0.5;
render_ref_accum[render_ref_len] = mono;
render_ref_len += 1;
idx += 2;
if render_ref_len == crate::frame::FRAME_10MS_SAMPLES {
rec.push_render_reference(&render_ref_accum);
render_ref_len = 0;
}
}
} else {
render_recorder_active = false;
}
} else {
render_recorder_active = false;
}
// Track audio-vs-silence for the diagnostic.
if peak_out > 0 {
if mix_stats.peak_i16 > 0 {
callbacks_with_audio = callbacks_with_audio.wrapping_add(1);
} else {
audio_processing_stats.increment_output_underrun();
callbacks_with_silence = callbacks_with_silence.wrapping_add(1);
}
@@ -580,7 +878,7 @@ impl IosVoiceUnit {
frames_changes = num_frames_changes,
callbacks_with_audio,
callbacks_with_silence,
peak_out_i16 = peak_out,
peak_out_i16 = mix_stats.peak_i16,
gain,
"ios audio unit render callback diagnostic sample (direct fill_buffer)"
);
@@ -593,13 +891,52 @@ impl IosVoiceUnit {
// stream formats we set above. After initialize() most
// property changes are rejected (you have to uninitialize +
// re-initialize), which is why the property set must come
// first. Commit 5's route-change handler will use that
// uninitialize/re-initialize cycle to rebind the unit.
unit.initialize()
.map_err(|e| AudioError::Backend(format!("vpio initialize: {e}")))?;
// first.
//
// AudioUnit::initialize() issues an RPC to the CoreAudio server.
// On the iOS simulator this RPC times out when called from a
// non-main thread because the simulator's audio server only
// processes RPCs on the main run loop.
//
// Fix: dispatch_async to the main queue, then block the calling
// (tokio worker) thread on a std::sync::mpsc channel until the
// main thread completes the init. This is safe because:
// 1. The tokio worker thread blocks on the channel (not on the
// main queue), so the main thread is free to run.
// 2. AudioUnit is Send (coreaudio-rs marks it unsafe impl Send).
// 3. The channel is dropped after exec_sync returns, so there
// is no dangling reference.
{
let (tx, rx) = std::sync::mpsc::sync_channel::<Result<(), String>>(1);
// Move unit into the Arc so it can cross thread boundaries.
let unit_arc = std::sync::Arc::new(std::sync::Mutex::new(Some(unit)));
let unit_arc2 = unit_arc.clone();
unit.start()
.map_err(|e| AudioError::Backend(format!("vpio start: {e}")))?;
dispatch2::DispatchQueue::main().exec_async(move || {
let mut guard = unit_arc2.lock().unwrap();
let u = guard.as_mut().unwrap();
let result = u
.initialize()
.map_err(|e| format!("vpio initialize: {e}"))
.and_then(|_| u.start().map_err(|e| format!("vpio start: {e}")));
let _ = tx.send(result);
});
// Block the tokio worker thread until the main thread finishes.
// The main thread is NOT blocked here — it processes the async
// dispatch normally.
match rx.recv() {
Ok(Ok(())) => {}
Ok(Err(msg)) => return Err(AudioError::Backend(msg)),
Err(_) => {
return Err(AudioError::Backend(
"vpio init: main thread channel closed unexpectedly".to_string(),
))
}
}
unit = unit_arc.lock().unwrap().take().unwrap();
}
info!(
target: "chanora_audio",
@@ -659,6 +996,10 @@ impl IosVoiceUnit {
/// Route rebinding on iOS is most reliable when we bounce the
/// VoiceProcessingIO unit through an uninitialize/reinitialize
/// cycle, then start again.
///
/// Called from the Flutter method channel handler which runs on
/// the main isolate — that runs on the main thread — so the
/// CoreAudio RPC is already on the correct thread here.
#[cfg(target_os = "ios")]
pub fn restart(&mut self) -> Result<(), AudioError> {
self.unit