Edison Jwa
b20e6b663a
Replace 42 .lock().unwrap() calls with .unwrap_or_else(|e| e.into_inner()) across 7 files. Poisoned mutex recovery prevents panics in realtime audio callbacks. Add SAFETY comment to WebRtcFallbackVad Send impl (TODO-007).
1215 lines
56 KiB
Rust
1215 lines
56 KiB
Rust
//! Apple output + capture stream via the **VoiceProcessingIO**
|
||
//! AudioUnit (`kAudioUnitSubType_VoiceProcessingIO`, a.k.a. VPIO).
|
||
//!
|
||
//! ## Why not cpal on Apple voice paths
|
||
//!
|
||
//! cpal's Apple backend does not expose the voice-processing unit controls
|
||
//! Chanora needs for a VoIP client. On iOS, cpal opens
|
||
//! `kAudioUnitSubType_RemoteIO` with no
|
||
//! control over the stream format, buffer size, or channel count;
|
||
//! on iPhone 16 Pro running iOS 18 it reports the output element as
|
||
//! **mono 48 kHz** even when the session category is `.playAndRecord`
|
||
//! with mode `.default`. More importantly, RemoteIO opened by cpal
|
||
//! stays bound to the route that was active at construction time:
|
||
//! a later `AVAudioSession.overrideOutputAudioPort(.speaker)` flips
|
||
//! the session route metadata (visible in
|
||
//! `AVAudioSession.currentRoute`) but the underlying AudioUnit
|
||
//! keeps writing to the original transducer. End-user symptom: the
|
||
//! Speaker / Receiver toggle in our picker shows the route change
|
||
//! in logs but produces no audible difference — the audio is still
|
||
//! coming out the earpiece.
|
||
//!
|
||
//! Production VoIP clients on Apple platforms drive VPIO directly instead
|
||
//! of treating CoreAudio as a generic music-playback device. VPIO is Apple's
|
||
//! native voice unit: it ships hardware AEC + AGC + NS and accepts explicit
|
||
//! stream-format requests on bus 0 (output) and bus 1 (input).
|
||
//!
|
||
//! This file replaces the cpal capture + playback streams on iOS and macOS.
|
||
//! Linux uses SDL2 for output (see `sdl_output.rs`) and cpal for capture;
|
||
//! Windows uses cpal's WASAPI backend.
|
||
//!
|
||
//! ## What VPIO gives us
|
||
//!
|
||
//! * Pinned **48 kHz Int16 mono** stream format on both bus 0
|
||
//! (output to hardware) and bus 1 (input from hardware). 48 kHz
|
||
//! matches the Opus encoder + `tsclientlib::AudioHandler` mix
|
||
//! rate exactly, so no resampling is needed inside the audio
|
||
//! callback. Int16 is Apple's documented canonical iOS sample
|
||
//! format for AudioUnits (see Audio Unit Hosting Guide for iOS
|
||
//! §"Canonical formats").
|
||
//! * Hardware **AEC** (acoustic echo cancellation), **AGC**
|
||
//! (automatic gain control), and **NS** (noise suppression) ran
|
||
//! in the secure-enclave-adjacent voice processor. Free DSP that
|
||
//! we'd otherwise need to ship as software (DEC-007/008/009).
|
||
//! * **Route-change-correct** physical binding: tapping Speaker or
|
||
//! Receiver in our picker now actually moves the audio.
|
||
//!
|
||
//! ## Threading & lifecycle
|
||
//!
|
||
//! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the
|
||
//! AudioUnit internally holds the C `AudioUnit` opaque pointer and
|
||
//! the wrapper's destructor calls `AudioComponentInstanceDispose`.
|
||
//! `IosVoiceUnit` stores the unit as `Option<AudioUnit>` so initial
|
||
//! setup and lifecycle operations can dispatch CoreAudio
|
||
//! initialize/start/stop calls to `DispatchQueue::main()` while the
|
||
//! wrapper keeps ownership and preserves the render/input callback
|
||
//! state in the surrounding `Arc`s. `restart`, `pause`, and
|
||
//! `resume` may temporarily move the unit through that helper, and
|
||
//! `Drop` stops it if still present before those callback `Arc`s are
|
||
//! released.
|
||
//!
|
||
//! ## What this file does NOT do
|
||
//!
|
||
//! * Route-change observation — that lives in Swift
|
||
//! (`AppDelegate.handleRouteChange`) and bounces the unit via a
|
||
//! future FRB call. Tracked as Commit 5 of the VPIO rollout.
|
||
//! * AVAudioSession category / mode configuration — Swift owns the
|
||
//! session (it must be set up before Flutter loads).
|
||
|
||
use std::sync::atomic::{AtomicBool, Ordering};
|
||
use std::sync::{Arc, Mutex};
|
||
|
||
use audiopus::coder::Encoder as OpusEncoder;
|
||
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
|
||
use coreaudio::audio_unit::render_callback::{self, data};
|
||
use coreaudio::audio_unit::IOType;
|
||
use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat};
|
||
use crossbeam::queue::ArrayQueue;
|
||
use tracing::{debug, error, info, warn};
|
||
|
||
use crate::mobile_voice_backend::VoiceAudioParams;
|
||
use crate::AudioError;
|
||
|
||
/// Sample rate every layer above us assumes. Matches the Opus
|
||
/// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the
|
||
/// sample rate we ask iOS to give us via VPIO's StreamFormat.
|
||
const SAMPLE_RATE_HZ: f64 = 48_000.0;
|
||
|
||
/// Output element (bus 0) of an `IOType::VoiceProcessingIO` unit
|
||
/// drives the hardware speaker / receiver / AirPods / BT. The
|
||
/// stream format we set on `Scope::Input` of this element is the
|
||
/// format **we** push samples in; VPIO converts internally to
|
||
/// whatever the hardware needs.
|
||
const OUTPUT_BUS: Element = Element::Output;
|
||
|
||
/// Input element (bus 1) of an `IOType::VoiceProcessingIO` unit
|
||
/// pulls from the hardware microphone. The stream format we set on
|
||
/// `Scope::Output` of this element is the format **we** receive
|
||
/// samples in.
|
||
const INPUT_BUS: Element = Element::Input;
|
||
|
||
/// Pre-roll buffer capacity: 160 ms / 10 ms = 16 frames.
|
||
/// Stores processed i16 frames so the first syllable is not lost
|
||
/// when the VAD gate opens (VAD_004 / pre_roll_ms=160).
|
||
const PRE_ROLL_FRAMES: usize = 16;
|
||
|
||
/// Enough room for the 160 ms VAD pre-roll plus a few jitter frames, without
|
||
/// growing inside the input callback.
|
||
const CAPTURE_ACCUM_CAPACITY_SAMPLES: usize = crate::frame::FRAME_10MS_SAMPLES * 20;
|
||
|
||
/// Fixed iOS render scratch capacity. Larger callback requests are truncated
|
||
/// to this capacity and the remaining output is silence.
|
||
#[cfg_attr(not(target_os = "ios"), allow(dead_code))]
|
||
const IOS_RENDER_SCRATCH_FRAMES: usize = 4096;
|
||
|
||
/// Capture pipeline state owned by the VPIO input callback. The
|
||
/// AudioUnit hands us 48 kHz signed-int16 mono PCM directly (no
|
||
/// downmix or resample needed — VPIO's hardware-side mix-down
|
||
/// from whatever the route's native format is happens before we
|
||
/// see the samples). All this struct does is gate on PTT, scale
|
||
/// by mic_gain, accumulate to a 20 ms / 960-sample frame, encode
|
||
/// to Opus, and try-send the resulting packet on the protocol
|
||
/// queue.
|
||
///
|
||
/// Mirrors the cpal-side `CaptureState` in engine.rs but is
|
||
/// type-specialised to i16 (the cpal version is generic over
|
||
/// `T: ToF32` to handle arbitrary HAL formats). Same Opus VoIP
|
||
/// tuning (32 kbps, complexity 10, inband FEC, 5% packet-loss
|
||
/// budget) lifted verbatim from `try_open_capture` so iOS audio
|
||
/// quality matches every other platform.
|
||
///
|
||
/// This struct is moved into the VPIO `set_input_callback` closure
|
||
/// and is therefore `'static + Send`. The OpusEncoder + Vec + arrays
|
||
/// are all owned; the two atomics + sender are `Arc<...>` clones
|
||
/// shared with `AudioEngine`.
|
||
struct IosCaptureState {
|
||
encoder: OpusEncoder,
|
||
/// 48 kHz mono PCM scratch accumulating to FRAME_20MS_SAMPLES
|
||
/// per encode. Capacity 2x to absorb cpal-style buffer-size
|
||
/// jitter without reallocating.
|
||
pcm_accum: Vec<i16>,
|
||
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||
voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender,
|
||
transmit_active: Arc<AtomicBool>,
|
||
output_muted: Arc<AtomicBool>,
|
||
mic_gain: f32,
|
||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||
vad_detector: crate::vad::WebRtcFallbackVad,
|
||
silero_coreml_worker: Option<crate::vad::apple_coreml::AppleCoreMlVadWorker>,
|
||
/// Last VAD backend we configured — used to detect backend changes.
|
||
current_vad_backend: crate::VadBackend,
|
||
fallback_warned_backend: Option<crate::VadBackend>,
|
||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||
sonora_processor: crate::processor::SonoraProcessor,
|
||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
|
||
pending_10ms_len: usize,
|
||
pre_roll_buf: [[i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
|
||
pre_roll_head: usize,
|
||
pre_roll_count: usize,
|
||
pre_roll_flushed: bool,
|
||
capture_frame_seq: u64,
|
||
}
|
||
|
||
impl IosCaptureState {
|
||
/// Build a VoIP-tuned Opus encoder + the capture-state wrapper.
|
||
/// Encoder configuration is the same as cpal-side
|
||
/// `try_open_capture` (engine.rs) so audio quality is platform-
|
||
/// neutral.
|
||
fn new(params: &VoiceAudioParams) -> Result<Self, AudioError> {
|
||
let encoder = crate::opus_voice::new_voip_encoder("ios VPIO")?;
|
||
|
||
Ok(Self {
|
||
encoder,
|
||
pcm_accum: Vec::with_capacity(CAPTURE_ACCUM_CAPACITY_SAMPLES),
|
||
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||
voice_out_tx: crate::opus_voice::start_out_packet_worker(
|
||
params.voice_out_tx.clone(),
|
||
params.frames_sent.clone(),
|
||
"ios-vpio",
|
||
)?,
|
||
transmit_active: params.transmit_active.clone(),
|
||
output_muted: params.output_muted.clone(),
|
||
mic_gain: params.mic_gain,
|
||
voice_activity_selector: params.voice_activity_selector.clone(),
|
||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||
silero_coreml_worker: None,
|
||
current_vad_backend: crate::VadBackend::WebrtcVad,
|
||
fallback_warned_backend: None,
|
||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||
audio_processing_config: params.audio_processing_config.clone(),
|
||
sonora_processor: crate::processor::SonoraProcessor::new(),
|
||
audio_processing_stats: params.audio_processing_stats.clone(),
|
||
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
|
||
pending_10ms_len: 0,
|
||
pre_roll_buf: [[0_i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
|
||
pre_roll_head: 0,
|
||
pre_roll_count: 0,
|
||
pre_roll_flushed: false,
|
||
capture_frame_seq: 0,
|
||
})
|
||
}
|
||
|
||
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
|
||
if self.fallback_warned_backend == Some(failed_backend) {
|
||
return;
|
||
}
|
||
self.fallback_warned_backend = Some(failed_backend);
|
||
if self.capture_frame_seq < 128 {
|
||
tracing::info!(
|
||
target: "chanora_audio",
|
||
backend = failed_backend.as_str(),
|
||
seq = self.capture_frame_seq,
|
||
"VAD backend warming up; using WebRTC fallback"
|
||
);
|
||
} else {
|
||
tracing::warn!(
|
||
target: "chanora_audio",
|
||
backend = failed_backend.as_str(),
|
||
"VAD backend unavailable; using WebRTC fallback for runtime detection"
|
||
);
|
||
}
|
||
}
|
||
|
||
/// Consume the i16 mono buffer delivered by VPIO, accumulate
|
||
/// to a 20 ms frame boundary, encode + send when PTT is held.
|
||
///
|
||
/// VPIO's input element delivers samples already at the
|
||
/// stream format we pinned (48 kHz Int16 mono interleaved).
|
||
/// In practice "interleaved mono" is the same byte layout as
|
||
/// "planar mono" so we just take the buffer as-is.
|
||
fn ingest_i16(&mut self, samples: &[i16]) {
|
||
let mut offset = 0;
|
||
while offset < samples.len() {
|
||
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
|
||
let take = remaining.min(samples.len() - offset);
|
||
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
|
||
.copy_from_slice(&samples[offset..offset + take]);
|
||
self.pending_10ms_len += take;
|
||
offset += take;
|
||
|
||
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
|
||
let frame = self.pending_10ms;
|
||
self.process_10ms_capture_frame(&frame);
|
||
self.pending_10ms_len = 0;
|
||
}
|
||
}
|
||
|
||
if !self.transmit_active.load(Ordering::Relaxed) {
|
||
// Drain accumulator while muted so we don't pop on the
|
||
// PTT release edge. Matches cpal-side behaviour.
|
||
self.pcm_accum.clear();
|
||
return;
|
||
}
|
||
|
||
// Drain complete 20 ms frames out of the accumulator, encode
|
||
// each, send the resulting Opus packet on the protocol
|
||
// queue. The `while` covers the case where a single VPIO
|
||
// callback delivers more than one frame's worth (rare on
|
||
// iOS where the HW IO buffer duration aligns with the
|
||
// Opus frame, but always possible during route changes).
|
||
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
|
||
// Use a stack-allocated frame buffer to avoid the
|
||
// per-callback allocation a `drain(..N).collect()`
|
||
// would incur. The encoder doesn't need ownership.
|
||
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
|
||
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
|
||
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
|
||
|
||
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
|
||
Ok(len) => {
|
||
crate::opus_voice::send_voip_frame(
|
||
&self.voice_out_tx,
|
||
&self.opus_out,
|
||
len,
|
||
|| {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
"ios VPIO: voice_out queue full; dropping frame"
|
||
);
|
||
},
|
||
|| {
|
||
debug!(
|
||
target: "chanora_audio",
|
||
"ios VPIO: voice_out closed; capture pipeline stopping"
|
||
);
|
||
},
|
||
);
|
||
}
|
||
Err(e) => {
|
||
error!(target: "chanora_audio", error = %e, "ios VPIO opus encode failed");
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
fn process_10ms_capture_frame(&mut self, samples: &[i16; crate::frame::FRAME_10MS_SAMPLES]) {
|
||
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
|
||
*dst = crate::frame::i16_to_f32(src);
|
||
}
|
||
let input_dbfs = crate::frame::dbfs(&frame);
|
||
|
||
// Read config once per frame (try_lock: non-blocking, falls back to
|
||
// last-known values if the lock is contended — safe to miss one frame).
|
||
let (run_ns, run_agc, run_hpf, vad_backend, vad_hangover, debug_wav_dump_enabled) = self
|
||
.audio_processing_config
|
||
.try_lock()
|
||
.map(|cfg| {
|
||
let ns =
|
||
cfg.ns != crate::EffectOwner::Off && cfg.ns != crate::EffectOwner::Platform;
|
||
let agc =
|
||
cfg.agc != crate::EffectOwner::Off && cfg.agc != crate::EffectOwner::Platform;
|
||
let hpf = cfg.hpf_enabled;
|
||
(
|
||
ns,
|
||
agc,
|
||
hpf,
|
||
cfg.vad_backend,
|
||
cfg.vad_hangover_ms,
|
||
cfg.debug_wav_dump_enabled,
|
||
)
|
||
})
|
||
.unwrap_or((
|
||
false,
|
||
false,
|
||
true,
|
||
crate::VadBackend::WebrtcVad,
|
||
crate::voice_activity::VAD_HANGOVER_MS,
|
||
false,
|
||
));
|
||
|
||
// VPIO realtime callbacks cannot use WavDebugRecorder today: its push
|
||
// path allocates per frame. Debug WAV capture is intentionally disabled
|
||
// here until the recorder can hand off preallocated frames.
|
||
let _ = debug_wav_dump_enabled;
|
||
|
||
let voice_activity_mode = self
|
||
.voice_activity_selector
|
||
.as_ref()
|
||
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
||
.unwrap_or(false);
|
||
if !voice_activity_mode {
|
||
self.silero_coreml_worker = None;
|
||
self.current_vad_backend = crate::VadBackend::Disabled;
|
||
self.fallback_warned_backend = None;
|
||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||
}
|
||
|
||
if voice_activity_mode && vad_backend != self.current_vad_backend {
|
||
self.current_vad_backend = vad_backend;
|
||
self.fallback_warned_backend = None;
|
||
if vad_backend == crate::VadBackend::SileroOnnx {
|
||
self.silero_coreml_worker = None;
|
||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||
self.audio_processing_stats.set_vad_fallback_active(true);
|
||
} else {
|
||
self.silero_coreml_worker = None;
|
||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||
}
|
||
// Reset VAD state machine timers on backend switch.
|
||
self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new(
|
||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||
vad_hangover,
|
||
crate::voice_activity::VAD_MIN_TX_MS,
|
||
);
|
||
self.vad_state.reset();
|
||
}
|
||
|
||
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
|
||
if voice_activity_mode {
|
||
// Keep the VAD state machine aligned with the active config only
|
||
// while VoiceActivity mode owns the transmit gate.
|
||
self.vad_state.configure(
|
||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||
vad_hangover,
|
||
crate::voice_activity::VAD_MIN_TX_MS,
|
||
);
|
||
}
|
||
|
||
// VPIO owns AEC. Keep the legacy non-AEC conditioning path here until
|
||
// the raw WebRTC APM path is explicitly selected.
|
||
if run_ns || run_agc || run_hpf {
|
||
use crate::processor::sonora::SonoraConfig;
|
||
use crate::processor::AudioProcessor;
|
||
let new_cfg = SonoraConfig {
|
||
hpf: run_hpf,
|
||
aec3: false,
|
||
ns: run_ns,
|
||
agc2: run_agc,
|
||
};
|
||
if new_cfg != *self.sonora_processor.config() {
|
||
self.sonora_processor.apply_config(new_cfg);
|
||
}
|
||
self.sonora_processor.process_capture(&mut frame);
|
||
}
|
||
|
||
// VAD: only evaluate while VoiceActivity mode is active.
|
||
let (vad_probability, gate_open) = if voice_activity_mode {
|
||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||
let capture_seq = self.capture_frame_seq;
|
||
let mut used_fallback_vad = false;
|
||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||
crate::vad::VadOutput {
|
||
probability: 1.0,
|
||
speech: true,
|
||
}
|
||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||
if let Some(worker) = self.silero_coreml_worker.as_ref() {
|
||
let enqueued = worker.try_send(capture_seq, &frame);
|
||
if !worker.is_stale(capture_seq) {
|
||
let p = worker.latest_probability();
|
||
crate::vad::VadOutput {
|
||
probability: p,
|
||
speech: p >= 0.5,
|
||
}
|
||
} else if enqueued {
|
||
crate::vad::VadOutput {
|
||
probability: 0.0,
|
||
speech: false,
|
||
}
|
||
} else {
|
||
used_fallback_vad = true;
|
||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||
crate::vad::VoiceActivityDetector::process_10ms(
|
||
&mut self.vad_detector,
|
||
&frame,
|
||
)
|
||
}
|
||
} else {
|
||
used_fallback_vad = true;
|
||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||
crate::vad::VoiceActivityDetector::process_10ms(
|
||
&mut self.vad_detector,
|
||
&frame,
|
||
)
|
||
}
|
||
} else {
|
||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||
};
|
||
self.audio_processing_stats
|
||
.set_vad_fallback_active(used_fallback_vad);
|
||
(vad.probability, self.vad_state.update(vad.speech))
|
||
} else {
|
||
(0.0, false)
|
||
};
|
||
let output_muted = self.output_muted.load(Ordering::Relaxed);
|
||
if let Some(selector) = &self.voice_activity_selector {
|
||
selector.set_voice_activity_open(voice_activity_mode && gate_open && !output_muted);
|
||
}
|
||
self.audio_processing_stats.update_capture(
|
||
input_dbfs,
|
||
crate::frame::dbfs(&frame),
|
||
vad_probability,
|
||
voice_activity_mode && gate_open && !output_muted,
|
||
transmit_active,
|
||
);
|
||
|
||
// Convert to i16 for accumulation.
|
||
let mut pcm_frame = [0_i16; crate::frame::FRAME_10MS_SAMPLES];
|
||
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
|
||
for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) {
|
||
*dst = crate::frame::f32_to_i16(src);
|
||
}
|
||
} else {
|
||
let gain = self.mic_gain;
|
||
for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) {
|
||
let scaled = crate::frame::f32_to_i16(src) as f32 * gain;
|
||
*dst = scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16;
|
||
}
|
||
}
|
||
|
||
// Update pre-roll ring buffer (VAD_004: preserve first syllable).
|
||
let slot_idx = self.pre_roll_head % PRE_ROLL_FRAMES;
|
||
self.pre_roll_buf[slot_idx] = pcm_frame;
|
||
self.pre_roll_head = (self.pre_roll_head + 1) % PRE_ROLL_FRAMES;
|
||
if self.pre_roll_count < PRE_ROLL_FRAMES {
|
||
self.pre_roll_count += 1;
|
||
}
|
||
|
||
// If the transmit gate just opened and we haven't flushed the
|
||
// pre-roll yet, drain it into the accumulator.
|
||
if transmit_active && !self.pre_roll_flushed {
|
||
self.pre_roll_flushed = true;
|
||
// The oldest frame in the ring is at
|
||
// (pre_roll_head + PRE_ROLL_FRAMES - pre_roll_count) % PRE_ROLL_FRAMES.
|
||
// We emit frames in chronological order (oldest first), excluding
|
||
// the frame we just wrote (which goes into pcm_accum normally below).
|
||
let oldest =
|
||
(self.pre_roll_head + PRE_ROLL_FRAMES - self.pre_roll_count) % PRE_ROLL_FRAMES;
|
||
// Emit pre_roll_count - 1 frames (the -1 excludes the current frame
|
||
// which will be added below in the normal path).
|
||
let pre_roll_to_emit = self.pre_roll_count.saturating_sub(1);
|
||
for i in 0..pre_roll_to_emit {
|
||
let idx = (oldest + i) % PRE_ROLL_FRAMES;
|
||
if crate::capture_accumulator::append_i16_bounded(
|
||
&mut self.pcm_accum,
|
||
&self.pre_roll_buf[idx],
|
||
) {
|
||
self.audio_processing_stats.increment_callback_xrun();
|
||
break;
|
||
}
|
||
}
|
||
} else if !transmit_active {
|
||
// Gate closed — reset the flush flag so pre-roll fires again
|
||
// on the next gate open.
|
||
self.pre_roll_flushed = false;
|
||
}
|
||
|
||
if !transmit_active {
|
||
return;
|
||
}
|
||
|
||
if crate::capture_accumulator::append_i16_bounded(&mut self.pcm_accum, &pcm_frame) {
|
||
self.audio_processing_stats.increment_callback_xrun();
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Live iOS audio unit wrapper. Construct + start = audio
|
||
/// flowing; drop = audio stopped.
|
||
pub struct IosVoiceUnit {
|
||
// Drop = stop the audio unit (severs render callback). The
|
||
// wrapper's own Drop calls AudioComponentInstanceDispose
|
||
// after stop returns.
|
||
unit: Option<AudioUnit>,
|
||
// macOS-only: producer task (spawned in start_macos) polls
|
||
// this on every 20 ms tick and exits when set. Without it
|
||
// the tokio task captures `Arc<Mutex<AudioHandler>>` +
|
||
// `Arc<ArrayQueue<f32>>` and runs forever, leaking on every
|
||
// engine stop/restart cycle.
|
||
#[cfg(target_os = "macos")]
|
||
producer_shutdown: Arc<AtomicBool>,
|
||
}
|
||
|
||
impl IosVoiceUnit {
|
||
#[cfg(target_os = "ios")]
|
||
fn exec_main_queue_lifecycle(
|
||
&mut self,
|
||
op: impl FnOnce(&mut AudioUnit) -> Result<(), String> + Send + 'static,
|
||
) -> Result<(), AudioError> {
|
||
let unit = self
|
||
.unit
|
||
.take()
|
||
.ok_or_else(|| AudioError::Backend("vpio lifecycle: audio unit missing".to_string()))?;
|
||
|
||
let (tx, rx) = std::sync::mpsc::sync_channel::<Result<(), String>>(1);
|
||
let unit_arc = Arc::new(Mutex::new(Some(unit)));
|
||
let unit_arc2 = Arc::clone(&unit_arc);
|
||
|
||
dispatch2::DispatchQueue::main().exec_async(move || {
|
||
let mut guard = unit_arc2.lock().unwrap_or_else(|e| e.into_inner());
|
||
let unit = guard.as_mut().unwrap();
|
||
let _ = tx.send(op(unit));
|
||
});
|
||
|
||
let result = match rx.recv() {
|
||
Ok(result) => result,
|
||
Err(_) => Err("vpio lifecycle: main thread channel closed unexpectedly".to_string()),
|
||
};
|
||
|
||
self.unit = unit_arc.lock().unwrap_or_else(|e| e.into_inner()).take();
|
||
result.map_err(AudioError::Backend)
|
||
}
|
||
|
||
/// Open a VoiceProcessingIO AudioUnit, pin its stream format
|
||
/// to 48 kHz Int16 mono on both buses, install render + input
|
||
/// callbacks, and start it. The unit begins pumping audio
|
||
/// immediately on return — input callback fires when the mic
|
||
/// captures samples, render callback fires when the hardware
|
||
/// needs samples to play.
|
||
///
|
||
/// Parameters mirror the capture + playback inputs the cpal
|
||
/// and SDL backends accept so engine.rs can swap backends with
|
||
/// a `cfg`.
|
||
///
|
||
/// * `handler` — shared AudioHandler the inbound forwarder
|
||
/// feeds Opus packets into. The output render callback pulls
|
||
/// decoded f32 frames from it (48 kHz stereo) and downmixes
|
||
/// to the i16 mono buffer VPIO expects.
|
||
/// * `output_gain` / `output_muted` — same atomics the cpal
|
||
/// and SDL output paths read on every callback so the master
|
||
/// volume + local-mute UI works identically across backends.
|
||
/// * `voice_out_tx` — channel the capture pipeline sends
|
||
/// encoded `OutPacket`s on.
|
||
/// * `transmit_active` — PTT gate flag the capture pipeline
|
||
/// consults before encoding.
|
||
/// * `frames_sent` — counter the bridge stats surface reads.
|
||
/// * `mic_gain` — pre-encode amplitude scale.
|
||
///
|
||
/// Capture wiring landed in commit 3; playback wiring landed
|
||
/// in commit 4. Route-change observation is commit 5.
|
||
pub(crate) fn start(params: VoiceAudioParams) -> Result<Self, AudioError> {
|
||
// Construct the VoiceProcessingIO AudioUnit. cpal exposes
|
||
// `Default::default()` which on iOS picks the inferior
|
||
// RemoteIO unit; we explicitly pick VPIO. `coreaudio-rs`
|
||
// returns an already-initialized unit from `new`, but we
|
||
// need to set properties before init so use
|
||
// `new_uninitialized` and call `initialize` ourselves
|
||
// after the property set is complete.
|
||
let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO)
|
||
.map_err(|e| AudioError::Backend(format!("vpio audio unit new: {e}")))?;
|
||
|
||
// Enable input I/O on element 1. VPIO's input element is
|
||
// OFF by default — without this `kAudioOutputUnitProperty_EnableIO`
|
||
// toggle no audio flows in and the input callback never
|
||
// fires. The constant is 2003 per Apple's headers
|
||
// (`AudioUnitProperties.h`). The value is a u32 with
|
||
// 1 = enabled. Element::Input on `Scope::Input` is the
|
||
// mic side of the unit (element 1 of bus 1; despite the
|
||
// confusing nomenclature, `Scope::Input` here means
|
||
// "input to the unit" i.e. mic samples coming IN).
|
||
//
|
||
// Output element 0 is enabled by default for any
|
||
// `kAudioUnitType_Output` subtype (which VPIO is), so we
|
||
// don't need to toggle anything for playback.
|
||
//
|
||
// Apple's documented sequence for VPIO setup:
|
||
// 1. AudioComponentInstanceNew (= AudioUnit::new_uninitialized)
|
||
// 2. EnableIO on element 1 (= this set_property call)
|
||
// 3. Set stream format on both elements
|
||
// 4. Install callbacks
|
||
// 5. AudioUnitInitialize (= unit.initialize)
|
||
// 6. AudioOutputUnitStart (= unit.start)
|
||
const K_AUDIO_OUTPUT_UNIT_PROPERTY_ENABLE_IO: u32 = 2003;
|
||
let enable: u32 = 1;
|
||
unit.set_property(
|
||
K_AUDIO_OUTPUT_UNIT_PROPERTY_ENABLE_IO,
|
||
Scope::Input,
|
||
Element::Input,
|
||
Some(&enable),
|
||
)
|
||
.map_err(|e| AudioError::Backend(format!("vpio enable input I/O: {e}")))?;
|
||
|
||
// VPIO defaults to maximum "duck others" — when our voice plays,
|
||
// every other app's audio (Music, Safari, Discord, ...) is heavily
|
||
// attenuated. That is correct for a phone call but wrong for a
|
||
// chat client running alongside music or game audio. There is no
|
||
// public "off" switch; the lowest publicly-exposed level is Min
|
||
// and disabling Advanced Ducking turns off the voice-activity-
|
||
// driven dynamic ducking. This matches what Hume, Moonshine, and
|
||
// similar open-source VoIP/voice apps configure.
|
||
// Apple ref: kAUVoiceIOProperty_OtherAudioDuckingConfiguration
|
||
// Property ID 2108, Global scope, Output element (= 0).
|
||
// Struct layout matches AUVoiceIOOtherAudioDuckingConfiguration
|
||
// from <AudioToolbox/AUVoiceIOOtherAudioDuckingConfiguration.h>:
|
||
// Boolean (u8) mEnableAdvancedDucking + AUVoiceIOOtherAudio-
|
||
// DuckingLevel (u32) mDuckingLevel, #[repr(C)] yields 8 bytes
|
||
// with the 3-byte natural alignment pad before the u32.
|
||
#[repr(C)]
|
||
struct AuVoiceIoOtherAudioDuckingConfiguration {
|
||
m_enable_advanced_ducking: u8,
|
||
m_ducking_level: u32,
|
||
}
|
||
const K_AU_VOICE_IO_PROPERTY_OTHER_AUDIO_DUCKING_CONFIGURATION: u32 = 2108;
|
||
const K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_LEVEL_MIN: u32 = 10;
|
||
let ducking_config = AuVoiceIoOtherAudioDuckingConfiguration {
|
||
m_enable_advanced_ducking: 0,
|
||
m_ducking_level: K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_LEVEL_MIN,
|
||
};
|
||
// Apple introduced this property in iOS 17 / macOS 14. On older
|
||
// OS versions VPIO returns kAudioUnitErr_InvalidProperty (-10879)
|
||
// — log it but never fail VPIO startup over a ducking knob.
|
||
if let Err(e) = unit.set_property(
|
||
K_AU_VOICE_IO_PROPERTY_OTHER_AUDIO_DUCKING_CONFIGURATION,
|
||
Scope::Global,
|
||
Element::Output,
|
||
Some(&ducking_config),
|
||
) {
|
||
tracing::debug!("vpio set OtherAudioDuckingConfiguration failed (older OS?): {e}");
|
||
}
|
||
|
||
// Note: we keep VPIO's voice processing chain ENABLED
|
||
// (AEC + AGC + NS on the mic path) because it gives us
|
||
// clean capture for free. The historical playback
|
||
// breakage we attributed to this chain (commit c16318c
|
||
// tried to bypass it) was actually caused by the
|
||
// AVAudioSession mode .voiceChat ducking output to the
|
||
// earpiece \u2014 fixed in AppDelegate.swift by switching
|
||
// to .default + .defaultToSpeaker. With the session mode
|
||
// correct, voice processing can stay on.
|
||
|
||
// Stream format. Apple's iOS canonical format for
|
||
// AudioUnits is Linear PCM, 16-bit signed integer samples
|
||
// (see "Canonical formats" in the Audio Unit Hosting Guide
|
||
// for iOS). Mono channel matches the Opus encoder + the
|
||
// mic input pipe. 48 kHz matches every layer above us.
|
||
//
|
||
// The StreamFormat is set on:
|
||
// * OUTPUT_BUS (bus 0), Scope::Input — the format WE
|
||
// push to the unit, i.e. what fill_buffer writes.
|
||
// * INPUT_BUS (bus 1), Scope::Output — the format we
|
||
// RECEIVE from the unit, i.e. what the mic input
|
||
// callback hands us.
|
||
// This pairing is documented in Audio Unit Hosting Guide
|
||
// for iOS §"Specifying the Audio Stream Format".
|
||
let stream_format = StreamFormat {
|
||
sample_rate: SAMPLE_RATE_HZ,
|
||
sample_format: SampleFormat::I16,
|
||
flags: LinearPcmFlags::IS_SIGNED_INTEGER | LinearPcmFlags::IS_PACKED,
|
||
channels: 1,
|
||
};
|
||
|
||
unit.set_stream_format(stream_format, Scope::Input, OUTPUT_BUS)
|
||
.map_err(|e| {
|
||
AudioError::StreamConfig(format!(
|
||
"vpio set output stream format (bus 0 input scope): {e}"
|
||
))
|
||
})?;
|
||
unit.set_stream_format(stream_format, Scope::Output, INPUT_BUS)
|
||
.map_err(|e| {
|
||
AudioError::StreamConfig(format!(
|
||
"vpio set input stream format (bus 1 output scope): {e}"
|
||
))
|
||
})?;
|
||
|
||
// Build the capture pipeline and move it into the input
|
||
// callback. The Opus encoder + accumulator + opus_out
|
||
// scratch are owned by the closure — no Mutex needed
|
||
// because the input callback is the sole writer/reader on
|
||
// the audio thread.
|
||
let mut capture_state = IosCaptureState::new(¶ms)?;
|
||
|
||
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
|
||
// VPIO with our pinned stream format delivers
|
||
// interleaved Int16 mono. `args.data.buffer` is a
|
||
// `&mut [i16]` of length num_frames * channels = N * 1.
|
||
// coreaudio-rs handles the AudioBufferList plumbing +
|
||
// the AudioUnitRender call internally before invoking
|
||
// this closure.
|
||
capture_state.ingest_i16(args.data.buffer);
|
||
Ok(())
|
||
})
|
||
.map_err(|e| AudioError::Backend(format!("vpio set input callback: {e}")))?;
|
||
|
||
// Install the render callback that drives playback. The
|
||
// buffer iOS hands us is uninitialised — we MUST fill it
|
||
// (writing silence if we have nothing, never leaving stale
|
||
// frames).
|
||
//
|
||
// Pipeline per callback:
|
||
// 1. Lock the AudioHandler, ask it to fill a scratch
|
||
// f32 stereo buffer (length = 2 * num_frames). The
|
||
// handler runs Opus decode + per-client jitter
|
||
// buffer + mix. Same primitive cpal + SDL output
|
||
// paths use; this is the platform-neutral playback
|
||
// contract from `tsclientlib::audio::AudioHandler`.
|
||
// 2. Downmix to mono i16 with master gain, then copy that
|
||
// mono sample across every output channel the callback
|
||
// exposes. We still request mono Int16 from VPIO, but
|
||
// the callback must respect the actual channel count it
|
||
// receives. The handler itself produces stereo f32, so
|
||
// we average L+R rather than dropping R — the cpal-side
|
||
// mono-output path made the same mistake briefly
|
||
// (commit 6a4dbad / fix) and lost half the spatial mix.
|
||
// 3. Local-mute zeroes the output but STILL drains
|
||
// AudioHandler in step 1 so its jitter buffer
|
||
// doesn't grow unbounded while muted. This is the
|
||
// contract every other backend follows (matches
|
||
// SdlOutput::callback and the cpal output stream).
|
||
//
|
||
// Build the playback pipeline (direct fill_buffer in
|
||
// render callback; matches tsclientlib's reference SDL
|
||
// example at
|
||
// tsclientlib/examples/audio_utils/ts_to_audio.rs).
|
||
//
|
||
// The earlier ring-buffer attempt (rc.8+73..+74) decoupled
|
||
// AudioHandler from the render callback via a 50 Hz
|
||
// producer task + SPSC ring buffer. The +74 diagnostics
|
||
// showed that approach was making things worse: the
|
||
// producer drained AudioHandler at 50 Hz, but iOS VPIO
|
||
// calls our render callback at ~43.5 Hz (consuming 1440
|
||
// mono samples per 23 ms call). With consumer slightly
|
||
// slower than producer in chunks-per-second but each
|
||
// consumer pull being larger, the ring averaged out empty
|
||
// — fill_buffer was returning silence 65-84% of ticks
|
||
// because we drained it too aggressively before packets
|
||
// arrived. Linux/SDL's same pattern works fine because
|
||
// SDL calls fill_buffer at exactly the device callback
|
||
// rate.
|
||
//
|
||
// Revert to direct call: render callback locks
|
||
// AudioHandler, asks for `num_frames` stereo frames, and
|
||
// immediately downmixes to mono i16 replicated across the
|
||
// callback's actual output channels. Same as Linux/SDL,
|
||
// just stereo-f32 -> interleaved-i16 converted at the
|
||
// boundary.
|
||
// macOS cadence-fix (commit-f):
|
||
//
|
||
// The render callback is the wrong place to call AudioHandler::fill_buffer.
|
||
// macOS VPIO invokes us with `num_frames=512` (= 10.67 ms @ 48 kHz) which
|
||
// is NOT a multiple of the 20 ms Opus frame size that AudioHandler expects
|
||
// inside fill_buffer. The mismatch (a) leaves fill_buffer unable to
|
||
// satisfy the request on most calls and (b) advances MAX_PACKET_LOSSES,
|
||
// which removes the talker and starts PLC silence/decay.
|
||
//
|
||
// Converged fix across Mumble (Speex jitter buffer + decoded PCM FIFO),
|
||
// WebRTC NetEQ (adaptive 80ms prebuffer + sync buffer), Songbird (Fill→
|
||
// Drain playout buffer), cpal/rodio (SPSC ring + zero-fill on underrun),
|
||
// tsclientlib's own SDL example, and the upstream tsclientlib::audio
|
||
// contract: separate ingress quantum (20 ms decoded PCM = 1920 stereo
|
||
// f32) from egress quantum (whatever VPIO asks for), connected by a
|
||
// lock-free ring of decoded PCM.
|
||
//
|
||
// Layout:
|
||
// * macOS: spawn a tokio producer task paced at 20 ms; each tick
|
||
// try-locks AudioHandler, calls fill_buffer(1920), pushes 1920
|
||
// f32 into the ring. The render callback only pops — no lock,
|
||
// no Opus decode, no allocator on the audio thread. 60 ms
|
||
// prebuffer (3 × 20 ms) is held before the callback starts
|
||
// draining, matching Mumble's playout margin and WebRTC's
|
||
// kStartDelayMs order of magnitude. Crossbeam ArrayQueue
|
||
// is used because we already depend on crossbeam-queue.
|
||
// * iOS: keep the existing direct fill_buffer path — VPIO on iOS
|
||
// requests 480-frame slices that ARE 20 ms aligned so the
|
||
// cadence mismatch does not arise there.
|
||
#[cfg(target_os = "macos")]
|
||
let producer_shutdown = Arc::new(AtomicBool::new(false));
|
||
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
// 100 ms capacity = 9600 stereo f32. Sized so that the 60 ms
|
||
// prebuffer plus a few jitter spikes fit without forcing the
|
||
// producer to drop frames. crossbeam ArrayQueue is fixed-cap
|
||
// and lock-free SPSC-ish (MPMC but wait-free per end); for
|
||
// single producer + single consumer it's effectively SPSC.
|
||
const RING_CAPACITY: usize = 12000;
|
||
const PULL_SAMPLES: usize = 1920; // one 20 ms Opus frame, stereo
|
||
const PREBUFFER_SAMPLES: usize = 9600; // 100 ms @ 48 kHz stereo
|
||
|
||
let pcm_ring: Arc<ArrayQueue<f32>> = Arc::new(ArrayQueue::new(RING_CAPACITY));
|
||
let pcm_ring_producer = pcm_ring.clone();
|
||
let pcm_ring_consumer = pcm_ring.clone();
|
||
let handler_for_producer = params.handler.clone();
|
||
let output_gain_for_render = params.output_gain.clone();
|
||
let output_muted_for_render = params.output_muted.clone();
|
||
|
||
let producer_shutdown_for_task = producer_shutdown.clone();
|
||
|
||
tokio::spawn(async move {
|
||
let mut pull_scratch: Vec<f32> = vec![0.0; PULL_SAMPLES];
|
||
let mut interval = tokio::time::interval(std::time::Duration::from_millis(20));
|
||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||
loop {
|
||
interval.tick().await;
|
||
if producer_shutdown_for_task.load(Ordering::Relaxed) {
|
||
break;
|
||
}
|
||
match handler_for_producer.try_lock() {
|
||
Ok(mut h) => {
|
||
pull_scratch.fill(0.0);
|
||
let _ = h.fill_buffer(&mut pull_scratch[..]);
|
||
for &s in &pull_scratch {
|
||
pcm_ring_producer.force_push(s);
|
||
}
|
||
}
|
||
Err(std::sync::TryLockError::WouldBlock) => {}
|
||
Err(std::sync::TryLockError::Poisoned(e)) => {
|
||
warn!(target: "chanora_audio",
|
||
"producer: AudioHandler mutex poisoned: {e}");
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
|
||
let render_callback::Args {
|
||
data, num_frames, ..
|
||
} = args;
|
||
let out: &mut [i16] = data.buffer;
|
||
let out_channels = data.channels;
|
||
let needed = num_frames * out_channels;
|
||
|
||
if pcm_ring_consumer.len() < PREBUFFER_SAMPLES {
|
||
for sample in &mut out[..needed] {
|
||
*sample = 0;
|
||
}
|
||
return Ok(());
|
||
}
|
||
|
||
// Pop L,R as a pair per frame. The pairing is
|
||
// load-bearing — `written += 2` half-buffer bug.
|
||
let mut written_frames: usize = 0;
|
||
while written_frames < num_frames {
|
||
let l = match pcm_ring_consumer.pop() {
|
||
Some(v) => v,
|
||
None => break,
|
||
};
|
||
let r = pcm_ring_consumer.pop().unwrap_or(0.0);
|
||
let mut frame = [l, r];
|
||
crate::voice_render::limit_peak_inplace(&mut frame, 0.99);
|
||
let (l_lim, r_lim) = (frame[0], frame[1]);
|
||
let base = written_frames * out_channels;
|
||
if out_channels == 1 {
|
||
let mono = (l_lim + r_lim) * 0.5;
|
||
out[base] = (mono.clamp(-1.0, 1.0) * i16::MAX as f32) as i16;
|
||
} else {
|
||
out[base] = (l_lim.clamp(-1.0, 1.0) * i16::MAX as f32) as i16;
|
||
out[base + 1] = (r_lim.clamp(-1.0, 1.0) * i16::MAX as f32) as i16;
|
||
}
|
||
written_frames += 1;
|
||
}
|
||
|
||
if written_frames < num_frames {
|
||
let remaining = num_frames - written_frames;
|
||
for f in 0..remaining {
|
||
let base = (written_frames + f) * out_channels;
|
||
for c in 0..out_channels {
|
||
out[base + c] = 0;
|
||
}
|
||
}
|
||
}
|
||
|
||
let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed));
|
||
let muted = output_muted_for_render.load(Ordering::Relaxed);
|
||
if muted {
|
||
for sample in &mut out[..needed] {
|
||
*sample = 0;
|
||
}
|
||
} else if gain != 1.0 {
|
||
for sample in &mut out[..needed] {
|
||
*sample = (((*sample as f32) * gain)
|
||
.clamp(i16::MIN as f32, i16::MAX as f32))
|
||
as i16;
|
||
}
|
||
}
|
||
|
||
Ok(())
|
||
})
|
||
.map_err(|e| AudioError::Backend(format!("audio unit set render callback: {e}")))?;
|
||
}
|
||
|
||
// iOS path: direct fill_buffer in callback. iOS VPIO
|
||
// requests 480-frame slices that align with tsclientlib's
|
||
// 20 ms Opus frame, so the cadence mismatch that macOS
|
||
// hits does not arise here. macOS has its own cfg-gated
|
||
// producer-task path above.
|
||
#[cfg(target_os = "ios")]
|
||
{
|
||
let mut scratch_stereo: Vec<f32> = vec![0.0; IOS_RENDER_SCRATCH_FRAMES * 2];
|
||
let handler_for_render = params.handler.clone();
|
||
let output_gain_for_render = params.output_gain.clone();
|
||
let output_muted_for_render = params.output_muted.clone();
|
||
let audio_processing_stats_for_render = params.audio_processing_stats.clone();
|
||
// Level meter decimation: the render callback fires ~93
|
||
// times/sec, but the bridge consumer reads at ~30 Hz.
|
||
let mut render_level_decimation: u32 = 0;
|
||
// Debug WAV render-reference capture is intentionally unavailable
|
||
// on iOS VPIO callbacks until WavDebugRecorder supports a
|
||
// preallocated handoff; its current push path allocates per frame.
|
||
// Diagnostic counters sampled every 100 callbacks.
|
||
let mut cb_count: u64 = 0;
|
||
let mut last_num_frames: usize = 0;
|
||
let mut num_frames_changes: u64 = 0;
|
||
let mut callbacks_with_audio: u64 = 0;
|
||
let mut callbacks_with_silence: u64 = 0;
|
||
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
|
||
let render_callback::Args {
|
||
data, num_frames, ..
|
||
} = args;
|
||
let out: &mut [i16] = data.buffer;
|
||
let out_channels = data.channels;
|
||
let process_frames = num_frames.min(IOS_RENDER_SCRATCH_FRAMES);
|
||
if process_frames < num_frames {
|
||
audio_processing_stats_for_render.increment_callback_xrun();
|
||
}
|
||
// AudioHandler produces 48 kHz stereo f32 (= frames * 2 floats).
|
||
let needed = process_frames * 2;
|
||
// Zero the live slice. AudioHandler::fill_buffer is
|
||
// additive (does NOT clear); residual values from
|
||
// earlier callbacks (when scratch was bigger) would
|
||
// leak through otherwise.
|
||
scratch_stereo[..needed].fill(0.0);
|
||
match handler_for_render.try_lock() {
|
||
Ok(mut h) => {
|
||
let _ = h.fill_buffer(&mut scratch_stereo[..needed]);
|
||
}
|
||
Err(std::sync::TryLockError::WouldBlock) => {
|
||
audio_processing_stats_for_render.increment_callback_xrun();
|
||
// scratch_stereo is already zeroed above.
|
||
}
|
||
Err(std::sync::TryLockError::Poisoned(e)) => {
|
||
// Never panic on the realtime IO thread.
|
||
warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {e}");
|
||
}
|
||
}
|
||
// Peak limiter — multi-client mixes can sum past 0 dBFS;
|
||
// without this the downmix helper would hard-clip to i16::MAX.
|
||
crate::voice_render::limit_peak_inplace(&mut scratch_stereo[..needed], 0.99);
|
||
let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed));
|
||
let muted = output_muted_for_render.load(Ordering::Relaxed);
|
||
let mix_stats = crate::voice_render::downmix_stereo_f32_to_interleaved_i16(
|
||
&scratch_stereo[..needed],
|
||
out,
|
||
out_channels,
|
||
gain,
|
||
muted,
|
||
);
|
||
if mix_stats.clipped_samples > 0 {
|
||
audio_processing_stats_for_render
|
||
.add_clipped_samples(mix_stats.clipped_samples);
|
||
}
|
||
render_level_decimation = render_level_decimation.wrapping_add(1);
|
||
if render_level_decimation % 3 == 0 {
|
||
audio_processing_stats_for_render.update_render(
|
||
crate::frame::dbfs(&scratch_stereo[..needed]),
|
||
num_frames as u32,
|
||
);
|
||
}
|
||
|
||
// Track audio-vs-silence for the diagnostic.
|
||
if mix_stats.peak_i16 > 0 {
|
||
callbacks_with_audio = callbacks_with_audio.wrapping_add(1);
|
||
} else {
|
||
callbacks_with_silence = callbacks_with_silence.wrapping_add(1);
|
||
// Muted output writes intentional silence (peak_i16 == 0 by
|
||
// design), not a starved render path. Gate on !muted to avoid
|
||
// counting deliberate silence as an output underrun.
|
||
if !muted {
|
||
audio_processing_stats_for_render.increment_output_underrun();
|
||
}
|
||
}
|
||
|
||
// Diagnostic sampling.
|
||
if last_num_frames != 0 && last_num_frames != num_frames {
|
||
num_frames_changes = num_frames_changes.wrapping_add(1);
|
||
}
|
||
last_num_frames = num_frames;
|
||
cb_count = cb_count.wrapping_add(1);
|
||
if cb_count.is_multiple_of(100) {
|
||
debug!(
|
||
target: "chanora_audio",
|
||
cb = cb_count,
|
||
num_frames,
|
||
frames_changes = num_frames_changes,
|
||
callbacks_with_audio,
|
||
callbacks_with_silence,
|
||
peak_out_i16 = mix_stats.peak_i16,
|
||
gain,
|
||
"ios audio unit render callback diagnostic sample (direct fill_buffer)"
|
||
);
|
||
}
|
||
Ok(())
|
||
})
|
||
.map_err(|e| AudioError::Backend(format!("audio unit set render callback: {e}")))?;
|
||
} // end #[cfg(target_os = "ios")] block
|
||
|
||
// Finalise the unit — allocates internal buffers per the
|
||
// stream formats we set above. After initialize() most
|
||
// property changes are rejected (you have to uninitialize +
|
||
// re-initialize), which is why the property set must come
|
||
// first.
|
||
//
|
||
// AudioUnit::initialize() issues an RPC to the CoreAudio server.
|
||
// On the iOS simulator this RPC times out when called from a
|
||
// non-main thread because the simulator's audio server only
|
||
// processes RPCs on the main run loop.
|
||
//
|
||
// Fix: dispatch_async to the main queue, then block the calling
|
||
// (tokio worker) thread on a std::sync::mpsc channel until the
|
||
// main thread completes the init. This is safe because:
|
||
// 1. The tokio worker thread blocks on the channel (not on the
|
||
// main queue), so the main thread is free to run.
|
||
// 2. AudioUnit is Send (coreaudio-rs marks it unsafe impl Send).
|
||
// 3. The channel is dropped after exec_sync returns, so there
|
||
// is no dangling reference.
|
||
{
|
||
let (tx, rx) = std::sync::mpsc::sync_channel::<Result<(), String>>(1);
|
||
// Move unit into the Arc so it can cross thread boundaries.
|
||
let unit_arc = std::sync::Arc::new(std::sync::Mutex::new(Some(unit)));
|
||
let unit_arc2 = unit_arc.clone();
|
||
|
||
dispatch2::DispatchQueue::main().exec_async(move || {
|
||
let mut guard = unit_arc2.lock().unwrap_or_else(|e| e.into_inner());
|
||
let u = guard.as_mut().unwrap();
|
||
let result = u
|
||
.initialize()
|
||
.map_err(|e| format!("vpio initialize: {e}"))
|
||
.and_then(|_| u.start().map_err(|e| format!("vpio start: {e}")));
|
||
let _ = tx.send(result);
|
||
});
|
||
|
||
// Block the tokio worker thread until the main thread finishes.
|
||
// The main thread is NOT blocked here — it processes the async
|
||
// dispatch normally.
|
||
match rx.recv() {
|
||
Ok(Ok(())) => {}
|
||
Ok(Err(msg)) => return Err(AudioError::Backend(msg)),
|
||
Err(_) => {
|
||
return Err(AudioError::Backend(
|
||
"vpio init: main thread channel closed unexpectedly".to_string(),
|
||
))
|
||
}
|
||
}
|
||
|
||
unit = unit_arc.lock().unwrap_or_else(|e| e.into_inner()).take().unwrap();
|
||
}
|
||
|
||
info!(
|
||
target: "chanora_audio",
|
||
sample_rate_hz = SAMPLE_RATE_HZ,
|
||
channels = stream_format.channels,
|
||
sample_format = ?stream_format.sample_format,
|
||
"ios VPIO audio unit started"
|
||
);
|
||
|
||
// Read back the ACTUAL stream format VPIO accepted on each
|
||
// bus (iOS sometimes substitutes its own format if the
|
||
// hardware can't satisfy our preference) and the actual
|
||
// AVAudioSession sample rate + IO buffer duration. Without
|
||
// these we can't tell whether our 48 kHz Int16 mono format
|
||
// was honoured or silently downgraded to e.g. 44.1 kHz
|
||
// Float32 (which would cause our render callback to write
|
||
// i16 values into a buffer iOS interprets as f32 = severe
|
||
// distortion). Diagnostic prompted by external review
|
||
// pointing out that 'preferredSampleRate' is a hint, not
|
||
// a guarantee \u2014 must verify post-init.
|
||
match unit.output_stream_format() {
|
||
Ok(fmt) => info!(
|
||
target: "chanora_audio",
|
||
sample_rate = fmt.sample_rate,
|
||
channels = fmt.channels,
|
||
sample_format = ?fmt.sample_format,
|
||
flags = ?fmt.flags,
|
||
"ios VPIO actual OUTPUT stream format (post-init)"
|
||
),
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"ios VPIO output_stream_format read failed"
|
||
),
|
||
}
|
||
match unit.input_stream_format() {
|
||
Ok(fmt) => info!(
|
||
target: "chanora_audio",
|
||
sample_rate = fmt.sample_rate,
|
||
channels = fmt.channels,
|
||
sample_format = ?fmt.sample_format,
|
||
flags = ?fmt.flags,
|
||
"ios VPIO actual INPUT stream format (post-init)"
|
||
),
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"ios VPIO input_stream_format read failed"
|
||
),
|
||
}
|
||
|
||
Ok(Self {
|
||
unit: Some(unit),
|
||
#[cfg(target_os = "macos")]
|
||
producer_shutdown,
|
||
})
|
||
}
|
||
|
||
/// Restart the audio unit after route change handling.
|
||
///
|
||
/// Route rebinding on iOS is most reliable when we bounce the
|
||
/// VoiceProcessingIO unit through an uninitialize/reinitialize
|
||
/// cycle, then start again.
|
||
///
|
||
#[cfg(target_os = "ios")]
|
||
pub fn restart(&mut self) -> Result<(), AudioError> {
|
||
self.exec_main_queue_lifecycle(|unit| {
|
||
unit.stop().map_err(|e| format!("vpio restart stop: {e}"))?;
|
||
unit.uninitialize()
|
||
.map_err(|e| format!("vpio restart uninit: {e}"))?;
|
||
unit.initialize()
|
||
.map_err(|e| format!("vpio restart init: {e}"))?;
|
||
unit.start()
|
||
.map_err(|e| format!("vpio restart start: {e}"))?;
|
||
Ok(())
|
||
})?;
|
||
info!(target: "chanora_audio", "ios VPIO audio unit restarted");
|
||
Ok(())
|
||
}
|
||
|
||
/// Pause the audio unit during an interruption.
|
||
#[cfg(target_os = "ios")]
|
||
pub fn pause(&mut self) -> Result<(), AudioError> {
|
||
self.exec_main_queue_lifecycle(|unit| {
|
||
unit.stop().map_err(|e| format!("vpio pause stop: {e}"))
|
||
})
|
||
}
|
||
|
||
/// Resume the audio unit after an interruption.
|
||
#[cfg(target_os = "ios")]
|
||
pub fn resume(&mut self) -> Result<(), AudioError> {
|
||
self.exec_main_queue_lifecycle(|unit| {
|
||
unit.start().map_err(|e| format!("vpio resume start: {e}"))
|
||
})
|
||
}
|
||
}
|
||
|
||
impl Drop for IosVoiceUnit {
|
||
fn drop(&mut self) {
|
||
// Signal the macOS producer task to exit on its next tick
|
||
// (up to 20 ms) so it releases its handler / ring clones.
|
||
#[cfg(target_os = "macos")]
|
||
self.producer_shutdown.store(true, Ordering::Relaxed);
|
||
|
||
// Stop the audio unit so the render callback no longer
|
||
// fires. The coreaudio-rs wrapper's own Drop calls
|
||
// AudioComponentInstanceDispose afterwards.
|
||
if let Some(unit) = self.unit.as_mut() {
|
||
if let Err(e) = unit.stop() {
|
||
warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed");
|
||
} else {
|
||
info!(target: "chanora_audio", "ios audio unit stopped");
|
||
}
|
||
}
|
||
}
|
||
}
|