//! Apple output + capture stream via the **VoiceProcessingIO** //! AudioUnit (`kAudioUnitSubType_VoiceProcessingIO`, a.k.a. VPIO). //! //! ## Why not cpal on Apple voice paths //! //! cpal's Apple backend does not expose the voice-processing unit controls //! Chanora needs for a VoIP client. On iOS, cpal opens //! `kAudioUnitSubType_RemoteIO` with no //! control over the stream format, buffer size, or channel count; //! on iPhone 16 Pro running iOS 18 it reports the output element as //! **mono 48 kHz** even when the session category is `.playAndRecord` //! with mode `.default`. More importantly, RemoteIO opened by cpal //! stays bound to the route that was active at construction time: //! a later `AVAudioSession.overrideOutputAudioPort(.speaker)` flips //! the session route metadata (visible in //! `AVAudioSession.currentRoute`) but the underlying AudioUnit //! keeps writing to the original transducer. End-user symptom: the //! Speaker / Receiver toggle in our picker shows the route change //! in logs but produces no audible difference — the audio is still //! coming out the earpiece. //! //! Production VoIP clients on Apple platforms drive VPIO directly instead //! of treating CoreAudio as a generic music-playback device. VPIO is Apple's //! native voice unit: it ships hardware AEC + AGC + NS and accepts explicit //! stream-format requests on bus 0 (output) and bus 1 (input). //! //! This file replaces the cpal capture + playback streams on iOS and macOS. //! Linux uses SDL2 for output (see `sdl_output.rs`) and cpal for capture; //! Windows uses cpal's WASAPI backend. //! //! ## What VPIO gives us //! //! * Pinned **48 kHz Int16 mono** stream format on both bus 0 //! (output to hardware) and bus 1 (input from hardware). 48 kHz //! matches the Opus encoder + `tsclientlib::AudioHandler` mix //! rate exactly, so no resampling is needed inside the audio //! callback. Int16 is Apple's documented canonical iOS sample //! format for AudioUnits (see Audio Unit Hosting Guide for iOS //! §"Canonical formats"). //! * Hardware **AEC** (acoustic echo cancellation), **AGC** //! (automatic gain control), and **NS** (noise suppression) ran //! in the secure-enclave-adjacent voice processor. Free DSP that //! we'd otherwise need to ship as software (DEC-007/008/009). //! * **Route-change-correct** physical binding: tapping Speaker or //! Receiver in our picker now actually moves the audio. //! //! ## Threading & lifecycle //! //! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the //! AudioUnit internally holds the C `AudioUnit` opaque pointer and //! the wrapper's destructor calls `AudioComponentInstanceDispose`. //! `IosVoiceUnit` stores the unit as `Option` so initial //! setup and lifecycle operations can dispatch CoreAudio //! initialize/start/stop calls to `DispatchQueue::main()` while the //! wrapper keeps ownership and preserves the render/input callback //! state in the surrounding `Arc`s. `restart`, `pause`, and //! `resume` may temporarily move the unit through that helper, and //! `Drop` stops it if still present before those callback `Arc`s are //! released. //! //! ## What this file does NOT do //! //! * Route-change observation — that lives in Swift //! (`AppDelegate.handleRouteChange`) and bounces the unit via a //! future FRB call. Tracked as Commit 5 of the VPIO rollout. //! * AVAudioSession category / mode configuration — Swift owns the //! session (it must be set up before Flutter loads). use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use audiopus::coder::Encoder as OpusEncoder; use coreaudio::audio_unit::audio_format::LinearPcmFlags; use coreaudio::audio_unit::render_callback::{self, data}; use coreaudio::audio_unit::IOType; use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat}; use crossbeam::queue::ArrayQueue; use tracing::{debug, error, info, warn}; use crate::mobile_voice_backend::VoiceAudioParams; use crate::AudioError; /// Sample rate every layer above us assumes. Matches the Opus /// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the /// sample rate we ask iOS to give us via VPIO's StreamFormat. const SAMPLE_RATE_HZ: f64 = 48_000.0; /// Output element (bus 0) of an `IOType::VoiceProcessingIO` unit /// drives the hardware speaker / receiver / AirPods / BT. The /// stream format we set on `Scope::Input` of this element is the /// format **we** push samples in; VPIO converts internally to /// whatever the hardware needs. const OUTPUT_BUS: Element = Element::Output; /// Input element (bus 1) of an `IOType::VoiceProcessingIO` unit /// pulls from the hardware microphone. The stream format we set on /// `Scope::Output` of this element is the format **we** receive /// samples in. const INPUT_BUS: Element = Element::Input; /// Pre-roll buffer capacity: 160 ms / 10 ms = 16 frames. /// Stores processed i16 frames so the first syllable is not lost /// when the VAD gate opens (VAD_004 / pre_roll_ms=160). const PRE_ROLL_FRAMES: usize = 16; /// Enough room for the 160 ms VAD pre-roll plus a few jitter frames, without /// growing inside the input callback. const CAPTURE_ACCUM_CAPACITY_SAMPLES: usize = crate::frame::FRAME_10MS_SAMPLES * 20; /// Fixed iOS render scratch capacity. Larger callback requests are truncated /// to this capacity and the remaining output is silence. #[cfg_attr(not(target_os = "ios"), allow(dead_code))] const IOS_RENDER_SCRATCH_FRAMES: usize = 4096; /// Capture pipeline state owned by the VPIO input callback. The /// AudioUnit hands us 48 kHz signed-int16 mono PCM directly (no /// downmix or resample needed — VPIO's hardware-side mix-down /// from whatever the route's native format is happens before we /// see the samples). All this struct does is gate on PTT, scale /// by mic_gain, accumulate to a 20 ms / 960-sample frame, encode /// to Opus, and try-send the resulting packet on the protocol /// queue. /// /// Mirrors the cpal-side `CaptureState` in engine.rs but is /// type-specialised to i16 (the cpal version is generic over /// `T: ToF32` to handle arbitrary HAL formats). Same Opus VoIP /// tuning (32 kbps, complexity 10, inband FEC, 5% packet-loss /// budget) lifted verbatim from `try_open_capture` so iOS audio /// quality matches every other platform. /// /// This struct is moved into the VPIO `set_input_callback` closure /// and is therefore `'static + Send`. The OpusEncoder + Vec + arrays /// are all owned; the two atomics + sender are `Arc<...>` clones /// shared with `AudioEngine`. struct IosCaptureState { encoder: OpusEncoder, /// 48 kHz mono PCM scratch accumulating to FRAME_20MS_SAMPLES /// per encode. Capacity 2x to absorb cpal-style buffer-size /// jitter without reallocating. pcm_accum: Vec, opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender, transmit_active: Arc, output_muted: Arc, mic_gain: f32, voice_activity_selector: Option>, vad_detector: crate::vad::WebRtcFallbackVad, silero_coreml_worker: Option, /// Last VAD backend we configured — used to detect backend changes. current_vad_backend: crate::VadBackend, fallback_warned_backend: Option, vad_state: crate::voice_activity::VoiceActivityStateMachine, audio_processing_config: Arc>, sonora_processor: crate::processor::SonoraProcessor, audio_processing_stats: Arc, pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES], pending_10ms_len: usize, pre_roll_buf: [[i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES], pre_roll_head: usize, pre_roll_count: usize, pre_roll_flushed: bool, capture_frame_seq: u64, } impl IosCaptureState { /// Build a VoIP-tuned Opus encoder + the capture-state wrapper. /// Encoder configuration is the same as cpal-side /// `try_open_capture` (engine.rs) so audio quality is platform- /// neutral. fn new(params: &VoiceAudioParams) -> Result { let encoder = crate::opus_voice::new_voip_encoder("ios VPIO")?; Ok(Self { encoder, pcm_accum: Vec::with_capacity(CAPTURE_ACCUM_CAPACITY_SAMPLES), opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx: crate::opus_voice::start_out_packet_worker( params.voice_out_tx.clone(), params.frames_sent.clone(), "ios-vpio", )?, transmit_active: params.transmit_active.clone(), output_muted: params.output_muted.clone(), mic_gain: params.mic_gain, voice_activity_selector: params.voice_activity_selector.clone(), vad_detector: crate::vad::WebRtcFallbackVad::default(), silero_coreml_worker: None, current_vad_backend: crate::VadBackend::WebrtcVad, fallback_warned_backend: None, vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), audio_processing_config: params.audio_processing_config.clone(), sonora_processor: crate::processor::SonoraProcessor::new(), audio_processing_stats: params.audio_processing_stats.clone(), pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES], pending_10ms_len: 0, pre_roll_buf: [[0_i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES], pre_roll_head: 0, pre_roll_count: 0, pre_roll_flushed: false, capture_frame_seq: 0, }) } fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) { if self.fallback_warned_backend == Some(failed_backend) { return; } self.fallback_warned_backend = Some(failed_backend); if self.capture_frame_seq < 128 { tracing::info!( target: "chanora_audio", backend = failed_backend.as_str(), seq = self.capture_frame_seq, "VAD backend warming up; using WebRTC fallback" ); } else { tracing::warn!( target: "chanora_audio", backend = failed_backend.as_str(), "VAD backend unavailable; using WebRTC fallback for runtime detection" ); } } /// Consume the i16 mono buffer delivered by VPIO, accumulate /// to a 20 ms frame boundary, encode + send when PTT is held. /// /// VPIO's input element delivers samples already at the /// stream format we pinned (48 kHz Int16 mono interleaved). /// In practice "interleaved mono" is the same byte layout as /// "planar mono" so we just take the buffer as-is. fn ingest_i16(&mut self, samples: &[i16]) { let mut offset = 0; while offset < samples.len() { let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len; let take = remaining.min(samples.len() - offset); self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take] .copy_from_slice(&samples[offset..offset + take]); self.pending_10ms_len += take; offset += take; if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES { let frame = self.pending_10ms; self.process_10ms_capture_frame(&frame); self.pending_10ms_len = 0; } } if !self.transmit_active.load(Ordering::Relaxed) { // Drain accumulator while muted so we don't pop on the // PTT release edge. Matches cpal-side behaviour. self.pcm_accum.clear(); return; } // Drain complete 20 ms frames out of the accumulator, encode // each, send the resulting Opus packet on the protocol // queue. The `while` covers the case where a single VPIO // callback delivers more than one frame's worth (rare on // iOS where the HW IO buffer duration aligns with the // Opus frame, but always possible during route changes). while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES { // Use a stack-allocated frame buffer to avoid the // per-callback allocation a `drain(..N).collect()` // would incur. The encoder doesn't need ownership. let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES]; frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]); self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES); match self.encoder.encode(&frame, &mut self.opus_out[..]) { Ok(len) => { crate::opus_voice::send_voip_frame( &self.voice_out_tx, &self.opus_out, len, || { warn!( target: "chanora_audio", "ios VPIO: voice_out queue full; dropping frame" ); }, || { debug!( target: "chanora_audio", "ios VPIO: voice_out closed; capture pipeline stopping" ); }, ); } Err(e) => { error!(target: "chanora_audio", error = %e, "ios VPIO opus encode failed"); } } } } fn process_10ms_capture_frame(&mut self, samples: &[i16; crate::frame::FRAME_10MS_SAMPLES]) { let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) { *dst = crate::frame::i16_to_f32(src); } let input_dbfs = crate::frame::dbfs(&frame); // Read config once per frame (try_lock: non-blocking, falls back to // last-known values if the lock is contended — safe to miss one frame). let (run_ns, run_agc, run_hpf, vad_backend, vad_hangover, debug_wav_dump_enabled) = self .audio_processing_config .try_lock() .map(|cfg| { let ns = cfg.ns != crate::EffectOwner::Off && cfg.ns != crate::EffectOwner::Platform; let agc = cfg.agc != crate::EffectOwner::Off && cfg.agc != crate::EffectOwner::Platform; let hpf = cfg.hpf_enabled; ( ns, agc, hpf, cfg.vad_backend, cfg.vad_hangover_ms, cfg.debug_wav_dump_enabled, ) }) .unwrap_or(( false, false, true, crate::VadBackend::WebrtcVad, crate::voice_activity::VAD_HANGOVER_MS, false, )); // VPIO realtime callbacks cannot use WavDebugRecorder today: its push // path allocates per frame. Debug WAV capture is intentionally disabled // here until the recorder can hand off preallocated frames. let _ = debug_wav_dump_enabled; let voice_activity_mode = self .voice_activity_selector .as_ref() .map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity) .unwrap_or(false); if !voice_activity_mode { self.silero_coreml_worker = None; self.current_vad_backend = crate::VadBackend::Disabled; self.fallback_warned_backend = None; self.audio_processing_stats.set_vad_fallback_active(false); } if voice_activity_mode && vad_backend != self.current_vad_backend { self.current_vad_backend = vad_backend; self.fallback_warned_backend = None; if vad_backend == crate::VadBackend::SileroOnnx { self.silero_coreml_worker = None; self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); self.audio_processing_stats.set_vad_fallback_active(true); } else { self.silero_coreml_worker = None; self.audio_processing_stats.set_vad_fallback_active(false); } // Reset VAD state machine timers on backend switch. self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new( crate::voice_activity::VAD_OPEN_AFTER_MS, vad_hangover, crate::voice_activity::VAD_MIN_TX_MS, ); self.vad_state.reset(); } let transmit_active = self.transmit_active.load(Ordering::Relaxed); if voice_activity_mode { // Keep the VAD state machine aligned with the active config only // while VoiceActivity mode owns the transmit gate. self.vad_state.configure( crate::voice_activity::VAD_OPEN_AFTER_MS, vad_hangover, crate::voice_activity::VAD_MIN_TX_MS, ); } // VPIO owns AEC. Keep the legacy non-AEC conditioning path here until // the raw WebRTC APM path is explicitly selected. if run_ns || run_agc || run_hpf { use crate::processor::sonora::SonoraConfig; use crate::processor::AudioProcessor; let new_cfg = SonoraConfig { hpf: run_hpf, aec3: false, ns: run_ns, agc2: run_agc, }; if new_cfg != *self.sonora_processor.config() { self.sonora_processor.apply_config(new_cfg); } self.sonora_processor.process_capture(&mut frame); } // VAD: only evaluate while VoiceActivity mode is active. let (vad_probability, gate_open) = if voice_activity_mode { self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1); let capture_seq = self.capture_frame_seq; let mut used_fallback_vad = false; let vad = if vad_backend == crate::VadBackend::Disabled { crate::vad::VadOutput { probability: 1.0, speech: true, } } else if vad_backend == crate::VadBackend::SileroOnnx { if let Some(worker) = self.silero_coreml_worker.as_ref() { let enqueued = worker.try_send(capture_seq, &frame); if !worker.is_stale(capture_seq) { let p = worker.latest_probability(); crate::vad::VadOutput { probability: p, speech: p >= 0.5, } } else if enqueued { crate::vad::VadOutput { probability: 0.0, speech: false, } } else { used_fallback_vad = true; self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); crate::vad::VoiceActivityDetector::process_10ms( &mut self.vad_detector, &frame, ) } } else { used_fallback_vad = true; self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); crate::vad::VoiceActivityDetector::process_10ms( &mut self.vad_detector, &frame, ) } } else { crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) }; self.audio_processing_stats .set_vad_fallback_active(used_fallback_vad); (vad.probability, self.vad_state.update(vad.speech)) } else { (0.0, false) }; let output_muted = self.output_muted.load(Ordering::Relaxed); if let Some(selector) = &self.voice_activity_selector { selector.set_voice_activity_open(voice_activity_mode && gate_open && !output_muted); } self.audio_processing_stats.update_capture( input_dbfs, crate::frame::dbfs(&frame), vad_probability, voice_activity_mode && gate_open && !output_muted, transmit_active, ); // Convert to i16 for accumulation. let mut pcm_frame = [0_i16; crate::frame::FRAME_10MS_SAMPLES]; if (self.mic_gain - 1.0).abs() < f32::EPSILON { for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) { *dst = crate::frame::f32_to_i16(src); } } else { let gain = self.mic_gain; for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) { let scaled = crate::frame::f32_to_i16(src) as f32 * gain; *dst = scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16; } } // Update pre-roll ring buffer (VAD_004: preserve first syllable). let slot_idx = self.pre_roll_head % PRE_ROLL_FRAMES; self.pre_roll_buf[slot_idx] = pcm_frame; self.pre_roll_head = (self.pre_roll_head + 1) % PRE_ROLL_FRAMES; if self.pre_roll_count < PRE_ROLL_FRAMES { self.pre_roll_count += 1; } // If the transmit gate just opened and we haven't flushed the // pre-roll yet, drain it into the accumulator. if transmit_active && !self.pre_roll_flushed { self.pre_roll_flushed = true; // The oldest frame in the ring is at // (pre_roll_head + PRE_ROLL_FRAMES - pre_roll_count) % PRE_ROLL_FRAMES. // We emit frames in chronological order (oldest first), excluding // the frame we just wrote (which goes into pcm_accum normally below). let oldest = (self.pre_roll_head + PRE_ROLL_FRAMES - self.pre_roll_count) % PRE_ROLL_FRAMES; // Emit pre_roll_count - 1 frames (the -1 excludes the current frame // which will be added below in the normal path). let pre_roll_to_emit = self.pre_roll_count.saturating_sub(1); for i in 0..pre_roll_to_emit { let idx = (oldest + i) % PRE_ROLL_FRAMES; if crate::capture_accumulator::append_i16_bounded( &mut self.pcm_accum, &self.pre_roll_buf[idx], ) { self.audio_processing_stats.increment_callback_xrun(); break; } } } else if !transmit_active { // Gate closed — reset the flush flag so pre-roll fires again // on the next gate open. self.pre_roll_flushed = false; } if !transmit_active { return; } if crate::capture_accumulator::append_i16_bounded(&mut self.pcm_accum, &pcm_frame) { self.audio_processing_stats.increment_callback_xrun(); } } } /// Live iOS audio unit wrapper. Construct + start = audio /// flowing; drop = audio stopped. pub struct IosVoiceUnit { // Drop = stop the audio unit (severs render callback). The // wrapper's own Drop calls AudioComponentInstanceDispose // after stop returns. unit: Option, // macOS-only: producer task (spawned in start_macos) polls // this on every 20 ms tick and exits when set. Without it // the tokio task captures `Arc>` + // `Arc>` and runs forever, leaking on every // engine stop/restart cycle. #[cfg(target_os = "macos")] producer_shutdown: Arc, } impl IosVoiceUnit { #[cfg(target_os = "ios")] fn exec_main_queue_lifecycle( &mut self, op: impl FnOnce(&mut AudioUnit) -> Result<(), String> + Send + 'static, ) -> Result<(), AudioError> { let unit = self .unit .take() .ok_or_else(|| AudioError::Backend("vpio lifecycle: audio unit missing".to_string()))?; let (tx, rx) = std::sync::mpsc::sync_channel::>(1); let unit_arc = Arc::new(Mutex::new(Some(unit))); let unit_arc2 = Arc::clone(&unit_arc); dispatch2::DispatchQueue::main().exec_async(move || { let mut guard = unit_arc2.lock().unwrap_or_else(|e| e.into_inner()); let unit = guard.as_mut().unwrap(); let _ = tx.send(op(unit)); }); let result = match rx.recv() { Ok(result) => result, Err(_) => Err("vpio lifecycle: main thread channel closed unexpectedly".to_string()), }; self.unit = unit_arc.lock().unwrap_or_else(|e| e.into_inner()).take(); result.map_err(AudioError::Backend) } /// Open a VoiceProcessingIO AudioUnit, pin its stream format /// to 48 kHz Int16 mono on both buses, install render + input /// callbacks, and start it. The unit begins pumping audio /// immediately on return — input callback fires when the mic /// captures samples, render callback fires when the hardware /// needs samples to play. /// /// Parameters mirror the capture + playback inputs the cpal /// and SDL backends accept so engine.rs can swap backends with /// a `cfg`. /// /// * `handler` — shared AudioHandler the inbound forwarder /// feeds Opus packets into. The output render callback pulls /// decoded f32 frames from it (48 kHz stereo) and downmixes /// to the i16 mono buffer VPIO expects. /// * `output_gain` / `output_muted` — same atomics the cpal /// and SDL output paths read on every callback so the master /// volume + local-mute UI works identically across backends. /// * `voice_out_tx` — channel the capture pipeline sends /// encoded `OutPacket`s on. /// * `transmit_active` — PTT gate flag the capture pipeline /// consults before encoding. /// * `frames_sent` — counter the bridge stats surface reads. /// * `mic_gain` — pre-encode amplitude scale. /// /// Capture wiring landed in commit 3; playback wiring landed /// in commit 4. Route-change observation is commit 5. pub(crate) fn start(params: VoiceAudioParams) -> Result { // Construct the VoiceProcessingIO AudioUnit. cpal exposes // `Default::default()` which on iOS picks the inferior // RemoteIO unit; we explicitly pick VPIO. `coreaudio-rs` // returns an already-initialized unit from `new`, but we // need to set properties before init so use // `new_uninitialized` and call `initialize` ourselves // after the property set is complete. let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO) .map_err(|e| AudioError::Backend(format!("vpio audio unit new: {e}")))?; // Enable input I/O on element 1. VPIO's input element is // OFF by default — without this `kAudioOutputUnitProperty_EnableIO` // toggle no audio flows in and the input callback never // fires. The constant is 2003 per Apple's headers // (`AudioUnitProperties.h`). The value is a u32 with // 1 = enabled. Element::Input on `Scope::Input` is the // mic side of the unit (element 1 of bus 1; despite the // confusing nomenclature, `Scope::Input` here means // "input to the unit" i.e. mic samples coming IN). // // Output element 0 is enabled by default for any // `kAudioUnitType_Output` subtype (which VPIO is), so we // don't need to toggle anything for playback. // // Apple's documented sequence for VPIO setup: // 1. AudioComponentInstanceNew (= AudioUnit::new_uninitialized) // 2. EnableIO on element 1 (= this set_property call) // 3. Set stream format on both elements // 4. Install callbacks // 5. AudioUnitInitialize (= unit.initialize) // 6. AudioOutputUnitStart (= unit.start) const K_AUDIO_OUTPUT_UNIT_PROPERTY_ENABLE_IO: u32 = 2003; let enable: u32 = 1; unit.set_property( K_AUDIO_OUTPUT_UNIT_PROPERTY_ENABLE_IO, Scope::Input, Element::Input, Some(&enable), ) .map_err(|e| AudioError::Backend(format!("vpio enable input I/O: {e}")))?; // VPIO defaults to maximum "duck others" — when our voice plays, // every other app's audio (Music, Safari, Discord, ...) is heavily // attenuated. That is correct for a phone call but wrong for a // chat client running alongside music or game audio. There is no // public "off" switch; the lowest publicly-exposed level is Min // and disabling Advanced Ducking turns off the voice-activity- // driven dynamic ducking. This matches what Hume, Moonshine, and // similar open-source VoIP/voice apps configure. // Apple ref: kAUVoiceIOProperty_OtherAudioDuckingConfiguration // Property ID 2108, Global scope, Output element (= 0). // Struct layout matches AUVoiceIOOtherAudioDuckingConfiguration // from : // Boolean (u8) mEnableAdvancedDucking + AUVoiceIOOtherAudio- // DuckingLevel (u32) mDuckingLevel, #[repr(C)] yields 8 bytes // with the 3-byte natural alignment pad before the u32. #[repr(C)] struct AuVoiceIoOtherAudioDuckingConfiguration { m_enable_advanced_ducking: u8, m_ducking_level: u32, } const K_AU_VOICE_IO_PROPERTY_OTHER_AUDIO_DUCKING_CONFIGURATION: u32 = 2108; const K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_LEVEL_MIN: u32 = 10; let ducking_config = AuVoiceIoOtherAudioDuckingConfiguration { m_enable_advanced_ducking: 0, m_ducking_level: K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_LEVEL_MIN, }; // Apple introduced this property in iOS 17 / macOS 14. On older // OS versions VPIO returns kAudioUnitErr_InvalidProperty (-10879) // — log it but never fail VPIO startup over a ducking knob. if let Err(e) = unit.set_property( K_AU_VOICE_IO_PROPERTY_OTHER_AUDIO_DUCKING_CONFIGURATION, Scope::Global, Element::Output, Some(&ducking_config), ) { tracing::debug!("vpio set OtherAudioDuckingConfiguration failed (older OS?): {e}"); } // Note: we keep VPIO's voice processing chain ENABLED // (AEC + AGC + NS on the mic path) because it gives us // clean capture for free. The historical playback // breakage we attributed to this chain (commit c16318c // tried to bypass it) was actually caused by the // AVAudioSession mode .voiceChat ducking output to the // earpiece \u2014 fixed in AppDelegate.swift by switching // to .default + .defaultToSpeaker. With the session mode // correct, voice processing can stay on. // Stream format. Apple's iOS canonical format for // AudioUnits is Linear PCM, 16-bit signed integer samples // (see "Canonical formats" in the Audio Unit Hosting Guide // for iOS). Mono channel matches the Opus encoder + the // mic input pipe. 48 kHz matches every layer above us. // // The StreamFormat is set on: // * OUTPUT_BUS (bus 0), Scope::Input — the format WE // push to the unit, i.e. what fill_buffer writes. // * INPUT_BUS (bus 1), Scope::Output — the format we // RECEIVE from the unit, i.e. what the mic input // callback hands us. // This pairing is documented in Audio Unit Hosting Guide // for iOS §"Specifying the Audio Stream Format". let stream_format = StreamFormat { sample_rate: SAMPLE_RATE_HZ, sample_format: SampleFormat::I16, flags: LinearPcmFlags::IS_SIGNED_INTEGER | LinearPcmFlags::IS_PACKED, channels: 1, }; unit.set_stream_format(stream_format, Scope::Input, OUTPUT_BUS) .map_err(|e| { AudioError::StreamConfig(format!( "vpio set output stream format (bus 0 input scope): {e}" )) })?; unit.set_stream_format(stream_format, Scope::Output, INPUT_BUS) .map_err(|e| { AudioError::StreamConfig(format!( "vpio set input stream format (bus 1 output scope): {e}" )) })?; // Build the capture pipeline and move it into the input // callback. The Opus encoder + accumulator + opus_out // scratch are owned by the closure — no Mutex needed // because the input callback is the sole writer/reader on // the audio thread. let mut capture_state = IosCaptureState::new(¶ms)?; unit.set_input_callback(move |args: render_callback::Args>| { // VPIO with our pinned stream format delivers // interleaved Int16 mono. `args.data.buffer` is a // `&mut [i16]` of length num_frames * channels = N * 1. // coreaudio-rs handles the AudioBufferList plumbing + // the AudioUnitRender call internally before invoking // this closure. capture_state.ingest_i16(args.data.buffer); Ok(()) }) .map_err(|e| AudioError::Backend(format!("vpio set input callback: {e}")))?; // Install the render callback that drives playback. The // buffer iOS hands us is uninitialised — we MUST fill it // (writing silence if we have nothing, never leaving stale // frames). // // Pipeline per callback: // 1. Lock the AudioHandler, ask it to fill a scratch // f32 stereo buffer (length = 2 * num_frames). The // handler runs Opus decode + per-client jitter // buffer + mix. Same primitive cpal + SDL output // paths use; this is the platform-neutral playback // contract from `tsclientlib::audio::AudioHandler`. // 2. Downmix to mono i16 with master gain, then copy that // mono sample across every output channel the callback // exposes. We still request mono Int16 from VPIO, but // the callback must respect the actual channel count it // receives. The handler itself produces stereo f32, so // we average L+R rather than dropping R — the cpal-side // mono-output path made the same mistake briefly // (commit 6a4dbad / fix) and lost half the spatial mix. // 3. Local-mute zeroes the output but STILL drains // AudioHandler in step 1 so its jitter buffer // doesn't grow unbounded while muted. This is the // contract every other backend follows (matches // SdlOutput::callback and the cpal output stream). // // Build the playback pipeline (direct fill_buffer in // render callback; matches tsclientlib's reference SDL // example at // tsclientlib/examples/audio_utils/ts_to_audio.rs). // // The earlier ring-buffer attempt (rc.8+73..+74) decoupled // AudioHandler from the render callback via a 50 Hz // producer task + SPSC ring buffer. The +74 diagnostics // showed that approach was making things worse: the // producer drained AudioHandler at 50 Hz, but iOS VPIO // calls our render callback at ~43.5 Hz (consuming 1440 // mono samples per 23 ms call). With consumer slightly // slower than producer in chunks-per-second but each // consumer pull being larger, the ring averaged out empty // — fill_buffer was returning silence 65-84% of ticks // because we drained it too aggressively before packets // arrived. Linux/SDL's same pattern works fine because // SDL calls fill_buffer at exactly the device callback // rate. // // Revert to direct call: render callback locks // AudioHandler, asks for `num_frames` stereo frames, and // immediately downmixes to mono i16 replicated across the // callback's actual output channels. Same as Linux/SDL, // just stereo-f32 -> interleaved-i16 converted at the // boundary. // macOS cadence-fix (commit-f): // // The render callback is the wrong place to call AudioHandler::fill_buffer. // macOS VPIO invokes us with `num_frames=512` (= 10.67 ms @ 48 kHz) which // is NOT a multiple of the 20 ms Opus frame size that AudioHandler expects // inside fill_buffer. The mismatch (a) leaves fill_buffer unable to // satisfy the request on most calls and (b) advances MAX_PACKET_LOSSES, // which removes the talker and starts PLC silence/decay. // // Converged fix across Mumble (Speex jitter buffer + decoded PCM FIFO), // WebRTC NetEQ (adaptive 80ms prebuffer + sync buffer), Songbird (Fill→ // Drain playout buffer), cpal/rodio (SPSC ring + zero-fill on underrun), // tsclientlib's own SDL example, and the upstream tsclientlib::audio // contract: separate ingress quantum (20 ms decoded PCM = 1920 stereo // f32) from egress quantum (whatever VPIO asks for), connected by a // lock-free ring of decoded PCM. // // Layout: // * macOS: spawn a tokio producer task paced at 20 ms; each tick // try-locks AudioHandler, calls fill_buffer(1920), pushes 1920 // f32 into the ring. The render callback only pops — no lock, // no Opus decode, no allocator on the audio thread. 60 ms // prebuffer (3 × 20 ms) is held before the callback starts // draining, matching Mumble's playout margin and WebRTC's // kStartDelayMs order of magnitude. Crossbeam ArrayQueue // is used because we already depend on crossbeam-queue. // * iOS: keep the existing direct fill_buffer path — VPIO on iOS // requests 480-frame slices that ARE 20 ms aligned so the // cadence mismatch does not arise there. #[cfg(target_os = "macos")] let producer_shutdown = Arc::new(AtomicBool::new(false)); #[cfg(target_os = "macos")] { // 100 ms capacity = 9600 stereo f32. Sized so that the 60 ms // prebuffer plus a few jitter spikes fit without forcing the // producer to drop frames. crossbeam ArrayQueue is fixed-cap // and lock-free SPSC-ish (MPMC but wait-free per end); for // single producer + single consumer it's effectively SPSC. const RING_CAPACITY: usize = 12000; const PULL_SAMPLES: usize = 1920; // one 20 ms Opus frame, stereo const PREBUFFER_SAMPLES: usize = 9600; // 100 ms @ 48 kHz stereo let pcm_ring: Arc> = Arc::new(ArrayQueue::new(RING_CAPACITY)); let pcm_ring_producer = pcm_ring.clone(); let pcm_ring_consumer = pcm_ring.clone(); let handler_for_producer = params.handler.clone(); let output_gain_for_render = params.output_gain.clone(); let output_muted_for_render = params.output_muted.clone(); let producer_shutdown_for_task = producer_shutdown.clone(); tokio::spawn(async move { let mut pull_scratch: Vec = vec![0.0; PULL_SAMPLES]; let mut interval = tokio::time::interval(std::time::Duration::from_millis(20)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { interval.tick().await; if producer_shutdown_for_task.load(Ordering::Relaxed) { break; } match handler_for_producer.try_lock() { Ok(mut h) => { pull_scratch.fill(0.0); let _ = h.fill_buffer(&mut pull_scratch[..]); for &s in &pull_scratch { pcm_ring_producer.force_push(s); } } Err(std::sync::TryLockError::WouldBlock) => {} Err(std::sync::TryLockError::Poisoned(e)) => { warn!(target: "chanora_audio", "producer: AudioHandler mutex poisoned: {e}"); break; } } } }); unit.set_render_callback(move |args: render_callback::Args>| { let render_callback::Args { data, num_frames, .. } = args; let out: &mut [i16] = data.buffer; let out_channels = data.channels; let needed = num_frames * out_channels; if pcm_ring_consumer.len() < PREBUFFER_SAMPLES { for sample in &mut out[..needed] { *sample = 0; } return Ok(()); } // Pop L,R as a pair per frame. The pairing is // load-bearing — `written += 2` half-buffer bug. let mut written_frames: usize = 0; while written_frames < num_frames { let l = match pcm_ring_consumer.pop() { Some(v) => v, None => break, }; let r = pcm_ring_consumer.pop().unwrap_or(0.0); let mut frame = [l, r]; crate::voice_render::limit_peak_inplace(&mut frame, 0.99); let (l_lim, r_lim) = (frame[0], frame[1]); let base = written_frames * out_channels; if out_channels == 1 { let mono = (l_lim + r_lim) * 0.5; out[base] = (mono.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; } else { out[base] = (l_lim.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; out[base + 1] = (r_lim.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; } written_frames += 1; } if written_frames < num_frames { let remaining = num_frames - written_frames; for f in 0..remaining { let base = (written_frames + f) * out_channels; for c in 0..out_channels { out[base + c] = 0; } } } let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed)); let muted = output_muted_for_render.load(Ordering::Relaxed); if muted { for sample in &mut out[..needed] { *sample = 0; } } else if gain != 1.0 { for sample in &mut out[..needed] { *sample = (((*sample as f32) * gain) .clamp(i16::MIN as f32, i16::MAX as f32)) as i16; } } Ok(()) }) .map_err(|e| AudioError::Backend(format!("audio unit set render callback: {e}")))?; } // iOS path: direct fill_buffer in callback. iOS VPIO // requests 480-frame slices that align with tsclientlib's // 20 ms Opus frame, so the cadence mismatch that macOS // hits does not arise here. macOS has its own cfg-gated // producer-task path above. #[cfg(target_os = "ios")] { let mut scratch_stereo: Vec = vec![0.0; IOS_RENDER_SCRATCH_FRAMES * 2]; let handler_for_render = params.handler.clone(); let output_gain_for_render = params.output_gain.clone(); let output_muted_for_render = params.output_muted.clone(); let audio_processing_stats_for_render = params.audio_processing_stats.clone(); // Level meter decimation: the render callback fires ~93 // times/sec, but the bridge consumer reads at ~30 Hz. let mut render_level_decimation: u32 = 0; // Debug WAV render-reference capture is intentionally unavailable // on iOS VPIO callbacks until WavDebugRecorder supports a // preallocated handoff; its current push path allocates per frame. // Diagnostic counters sampled every 100 callbacks. let mut cb_count: u64 = 0; let mut last_num_frames: usize = 0; let mut num_frames_changes: u64 = 0; let mut callbacks_with_audio: u64 = 0; let mut callbacks_with_silence: u64 = 0; unit.set_render_callback(move |args: render_callback::Args>| { let render_callback::Args { data, num_frames, .. } = args; let out: &mut [i16] = data.buffer; let out_channels = data.channels; let process_frames = num_frames.min(IOS_RENDER_SCRATCH_FRAMES); if process_frames < num_frames { audio_processing_stats_for_render.increment_callback_xrun(); } // AudioHandler produces 48 kHz stereo f32 (= frames * 2 floats). let needed = process_frames * 2; // Zero the live slice. AudioHandler::fill_buffer is // additive (does NOT clear); residual values from // earlier callbacks (when scratch was bigger) would // leak through otherwise. scratch_stereo[..needed].fill(0.0); match handler_for_render.try_lock() { Ok(mut h) => { let _ = h.fill_buffer(&mut scratch_stereo[..needed]); } Err(std::sync::TryLockError::WouldBlock) => { audio_processing_stats_for_render.increment_callback_xrun(); // scratch_stereo is already zeroed above. } Err(std::sync::TryLockError::Poisoned(e)) => { // Never panic on the realtime IO thread. warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {e}"); } } // Peak limiter — multi-client mixes can sum past 0 dBFS; // without this the downmix helper would hard-clip to i16::MAX. crate::voice_render::limit_peak_inplace(&mut scratch_stereo[..needed], 0.99); let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed)); let muted = output_muted_for_render.load(Ordering::Relaxed); let mix_stats = crate::voice_render::downmix_stereo_f32_to_interleaved_i16( &scratch_stereo[..needed], out, out_channels, gain, muted, ); if mix_stats.clipped_samples > 0 { audio_processing_stats_for_render .add_clipped_samples(mix_stats.clipped_samples); } render_level_decimation = render_level_decimation.wrapping_add(1); if render_level_decimation % 3 == 0 { audio_processing_stats_for_render.update_render( crate::frame::dbfs(&scratch_stereo[..needed]), num_frames as u32, ); } // Track audio-vs-silence for the diagnostic. if mix_stats.peak_i16 > 0 { callbacks_with_audio = callbacks_with_audio.wrapping_add(1); } else { callbacks_with_silence = callbacks_with_silence.wrapping_add(1); // Muted output writes intentional silence (peak_i16 == 0 by // design), not a starved render path. Gate on !muted to avoid // counting deliberate silence as an output underrun. if !muted { audio_processing_stats_for_render.increment_output_underrun(); } } // Diagnostic sampling. if last_num_frames != 0 && last_num_frames != num_frames { num_frames_changes = num_frames_changes.wrapping_add(1); } last_num_frames = num_frames; cb_count = cb_count.wrapping_add(1); if cb_count.is_multiple_of(100) { debug!( target: "chanora_audio", cb = cb_count, num_frames, frames_changes = num_frames_changes, callbacks_with_audio, callbacks_with_silence, peak_out_i16 = mix_stats.peak_i16, gain, "ios audio unit render callback diagnostic sample (direct fill_buffer)" ); } Ok(()) }) .map_err(|e| AudioError::Backend(format!("audio unit set render callback: {e}")))?; } // end #[cfg(target_os = "ios")] block // Finalise the unit — allocates internal buffers per the // stream formats we set above. After initialize() most // property changes are rejected (you have to uninitialize + // re-initialize), which is why the property set must come // first. // // AudioUnit::initialize() issues an RPC to the CoreAudio server. // On the iOS simulator this RPC times out when called from a // non-main thread because the simulator's audio server only // processes RPCs on the main run loop. // // Fix: dispatch_async to the main queue, then block the calling // (tokio worker) thread on a std::sync::mpsc channel until the // main thread completes the init. This is safe because: // 1. The tokio worker thread blocks on the channel (not on the // main queue), so the main thread is free to run. // 2. AudioUnit is Send (coreaudio-rs marks it unsafe impl Send). // 3. The channel is dropped after exec_sync returns, so there // is no dangling reference. { let (tx, rx) = std::sync::mpsc::sync_channel::>(1); // Move unit into the Arc so it can cross thread boundaries. let unit_arc = std::sync::Arc::new(std::sync::Mutex::new(Some(unit))); let unit_arc2 = unit_arc.clone(); dispatch2::DispatchQueue::main().exec_async(move || { let mut guard = unit_arc2.lock().unwrap_or_else(|e| e.into_inner()); let u = guard.as_mut().unwrap(); let result = u .initialize() .map_err(|e| format!("vpio initialize: {e}")) .and_then(|_| u.start().map_err(|e| format!("vpio start: {e}"))); let _ = tx.send(result); }); // Block the tokio worker thread until the main thread finishes. // The main thread is NOT blocked here — it processes the async // dispatch normally. match rx.recv() { Ok(Ok(())) => {} Ok(Err(msg)) => return Err(AudioError::Backend(msg)), Err(_) => { return Err(AudioError::Backend( "vpio init: main thread channel closed unexpectedly".to_string(), )) } } unit = unit_arc.lock().unwrap_or_else(|e| e.into_inner()).take().unwrap(); } info!( target: "chanora_audio", sample_rate_hz = SAMPLE_RATE_HZ, channels = stream_format.channels, sample_format = ?stream_format.sample_format, "ios VPIO audio unit started" ); // Read back the ACTUAL stream format VPIO accepted on each // bus (iOS sometimes substitutes its own format if the // hardware can't satisfy our preference) and the actual // AVAudioSession sample rate + IO buffer duration. Without // these we can't tell whether our 48 kHz Int16 mono format // was honoured or silently downgraded to e.g. 44.1 kHz // Float32 (which would cause our render callback to write // i16 values into a buffer iOS interprets as f32 = severe // distortion). Diagnostic prompted by external review // pointing out that 'preferredSampleRate' is a hint, not // a guarantee \u2014 must verify post-init. match unit.output_stream_format() { Ok(fmt) => info!( target: "chanora_audio", sample_rate = fmt.sample_rate, channels = fmt.channels, sample_format = ?fmt.sample_format, flags = ?fmt.flags, "ios VPIO actual OUTPUT stream format (post-init)" ), Err(e) => warn!( target: "chanora_audio", error = %e, "ios VPIO output_stream_format read failed" ), } match unit.input_stream_format() { Ok(fmt) => info!( target: "chanora_audio", sample_rate = fmt.sample_rate, channels = fmt.channels, sample_format = ?fmt.sample_format, flags = ?fmt.flags, "ios VPIO actual INPUT stream format (post-init)" ), Err(e) => warn!( target: "chanora_audio", error = %e, "ios VPIO input_stream_format read failed" ), } Ok(Self { unit: Some(unit), #[cfg(target_os = "macos")] producer_shutdown, }) } /// Restart the audio unit after route change handling. /// /// Route rebinding on iOS is most reliable when we bounce the /// VoiceProcessingIO unit through an uninitialize/reinitialize /// cycle, then start again. /// #[cfg(target_os = "ios")] pub fn restart(&mut self) -> Result<(), AudioError> { self.exec_main_queue_lifecycle(|unit| { unit.stop().map_err(|e| format!("vpio restart stop: {e}"))?; unit.uninitialize() .map_err(|e| format!("vpio restart uninit: {e}"))?; unit.initialize() .map_err(|e| format!("vpio restart init: {e}"))?; unit.start() .map_err(|e| format!("vpio restart start: {e}"))?; Ok(()) })?; info!(target: "chanora_audio", "ios VPIO audio unit restarted"); Ok(()) } /// Pause the audio unit during an interruption. #[cfg(target_os = "ios")] pub fn pause(&mut self) -> Result<(), AudioError> { self.exec_main_queue_lifecycle(|unit| { unit.stop().map_err(|e| format!("vpio pause stop: {e}")) }) } /// Resume the audio unit after an interruption. #[cfg(target_os = "ios")] pub fn resume(&mut self) -> Result<(), AudioError> { self.exec_main_queue_lifecycle(|unit| { unit.start().map_err(|e| format!("vpio resume start: {e}")) }) } } impl Drop for IosVoiceUnit { fn drop(&mut self) { // Signal the macOS producer task to exit on its next tick // (up to 20 ms) so it releases its handler / ring clones. #[cfg(target_os = "macos")] self.producer_shutdown.store(true, Ordering::Relaxed); // Stop the audio unit so the render callback no longer // fires. The coreaudio-rs wrapper's own Drop calls // AudioComponentInstanceDispose afterwards. if let Some(unit) = self.unit.as_mut() { if let Err(e) = unit.stop() { warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed"); } else { info!(target: "chanora_audio", "ios audio unit stopped"); } } } }