//! Apple output + capture stream via the **VoiceProcessingIO** //! AudioUnit (`kAudioUnitSubType_VoiceProcessingIO`, a.k.a. VPIO). //! //! ## Why not cpal on Apple voice paths //! //! cpal's Apple backend does not expose the voice-processing unit controls //! Chanora needs for a VoIP client. On iOS, cpal opens //! `kAudioUnitSubType_RemoteIO` with no //! control over the stream format, buffer size, or channel count; //! on iPhone 16 Pro running iOS 18 it reports the output element as //! **mono 48 kHz** even when the session category is `.playAndRecord` //! with mode `.default`. More importantly, RemoteIO opened by cpal //! stays bound to the route that was active at construction time: //! a later `AVAudioSession.overrideOutputAudioPort(.speaker)` flips //! the session route metadata (visible in //! `AVAudioSession.currentRoute`) but the underlying AudioUnit //! keeps writing to the original transducer. End-user symptom: the //! Speaker / Receiver toggle in our picker shows the route change //! in logs but produces no audible difference — the audio is still //! coming out the earpiece. //! //! Production VoIP clients on Apple platforms drive VPIO directly instead //! of treating CoreAudio as a generic music-playback device. VPIO is Apple's //! native voice unit: it ships hardware AEC + AGC + NS and accepts explicit //! stream-format requests on bus 0 (output) and bus 1 (input). //! //! This file replaces the cpal capture + playback streams on iOS and macOS. //! Linux uses SDL2 for output (see `sdl_output.rs`) and cpal for capture; //! Windows uses cpal's WASAPI backend. //! //! ## What VPIO gives us //! //! * Pinned **48 kHz Int16 mono** stream format on both bus 0 //! (output to hardware) and bus 1 (input from hardware). 48 kHz //! matches the Opus encoder + `tsclientlib::AudioHandler` mix //! rate exactly, so no resampling is needed inside the audio //! callback. Int16 is Apple's documented canonical iOS sample //! format for AudioUnits (see Audio Unit Hosting Guide for iOS //! §"Canonical formats"). //! * Hardware **AEC** (acoustic echo cancellation), **AGC** //! (automatic gain control), and **NS** (noise suppression) ran //! in the secure-enclave-adjacent voice processor. Free DSP that //! we'd otherwise need to ship as software (DEC-007/008/009). //! * **Route-change-correct** physical binding: tapping Speaker or //! Receiver in our picker now actually moves the audio. //! //! ## Threading & lifecycle //! //! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the //! AudioUnit internally holds the C `AudioUnit` opaque pointer and //! the wrapper's destructor calls `AudioComponentInstanceDispose`. //! `IosVoiceUnit` stores the unit as `Option` so initial //! setup and lifecycle operations can dispatch CoreAudio //! initialize/start/stop calls to `DispatchQueue::main()` while the //! wrapper keeps ownership and preserves the render/input callback //! state in the surrounding `Arc`s. `restart`, `pause`, and //! `resume` may temporarily move the unit through that helper, and //! `Drop` stops it if still present before those callback `Arc`s are //! released. //! //! ## What this file does NOT do //! //! * Route-change observation — that lives in Swift //! (`AppDelegate.handleRouteChange`) and bounces the unit via a //! future FRB call. Tracked as Commit 5 of the VPIO rollout. //! * AVAudioSession category / mode configuration — Swift owns the //! session (it must be set up before Flutter loads). use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; use audiopus::coder::Encoder as OpusEncoder; use coreaudio::audio_unit::audio_format::LinearPcmFlags; use coreaudio::audio_unit::render_callback::{self, data}; use coreaudio::audio_unit::IOType; use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat}; use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; use crate::mobile_voice_backend::VoiceAudioParams; use crate::AudioError; use chanora_protocol::OutPacket; /// Sample rate every layer above us assumes. Matches the Opus /// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the /// sample rate we ask iOS to give us via VPIO's StreamFormat. const SAMPLE_RATE_HZ: f64 = 48_000.0; /// Output element (bus 0) of an `IOType::VoiceProcessingIO` unit /// drives the hardware speaker / receiver / AirPods / BT. The /// stream format we set on `Scope::Input` of this element is the /// format **we** push samples in; VPIO converts internally to /// whatever the hardware needs. const OUTPUT_BUS: Element = Element::Output; /// Input element (bus 1) of an `IOType::VoiceProcessingIO` unit /// pulls from the hardware microphone. The stream format we set on /// `Scope::Output` of this element is the format **we** receive /// samples in. const INPUT_BUS: Element = Element::Input; /// Pre-roll buffer capacity: 160 ms / 10 ms = 16 frames. /// Stores processed i16 frames so the first syllable is not lost /// when the VAD gate opens (VAD_004 / pre_roll_ms=160). const PRE_ROLL_FRAMES: usize = 16; /// Capture pipeline state owned by the VPIO input callback. The /// AudioUnit hands us 48 kHz signed-int16 mono PCM directly (no /// downmix or resample needed — VPIO's hardware-side mix-down /// from whatever the route's native format is happens before we /// see the samples). All this struct does is gate on PTT, scale /// by mic_gain, accumulate to a 20 ms / 960-sample frame, encode /// to Opus, and try-send the resulting packet on the protocol /// queue. /// /// Mirrors the cpal-side `CaptureState` in engine.rs but is /// type-specialised to i16 (the cpal version is generic over /// `T: ToF32` to handle arbitrary HAL formats). Same Opus VoIP /// tuning (32 kbps, complexity 10, inband FEC, 5% packet-loss /// budget) lifted verbatim from `try_open_capture` so iOS audio /// quality matches every other platform. /// /// This struct is moved into the VPIO `set_input_callback` closure /// and is therefore `'static + Send`. The OpusEncoder + Vec + arrays /// are all owned; the two atomics + sender are `Arc<...>` clones /// shared with `AudioEngine`. struct IosCaptureState { encoder: OpusEncoder, /// 48 kHz mono PCM scratch accumulating to FRAME_20MS_SAMPLES /// per encode. Capacity 2x to absorb cpal-style buffer-size /// jitter without reallocating. pcm_accum: Vec, opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx: mpsc::Sender, transmit_active: Arc, output_muted: Arc, frames_sent: Arc, mic_gain: f32, voice_activity_selector: Option>, vad_detector: crate::vad::WebRtcFallbackVad, silero_coreml_worker: Option, /// Last VAD backend we configured — used to detect backend changes. current_vad_backend: crate::VadBackend, fallback_warned_backend: Option, vad_state: crate::voice_activity::VoiceActivityStateMachine, audio_processing_config: Arc>, sonora_processor: crate::processor::SonoraProcessor, audio_processing_stats: Arc, pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES], pending_10ms_len: usize, pre_roll_buf: [[i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES], pre_roll_head: usize, pre_roll_count: usize, pre_roll_flushed: bool, capture_frame_seq: u64, wav_recorder: Arc>>>, } impl IosCaptureState { /// Build a VoIP-tuned Opus encoder + the capture-state wrapper. /// Encoder configuration is the same as cpal-side /// `try_open_capture` (engine.rs) so audio quality is platform- /// neutral. fn new( params: &VoiceAudioParams, wav_recorder: Arc>>>, ) -> Result { let encoder = crate::opus_voice::new_voip_encoder("ios VPIO")?; Ok(Self { encoder, pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2), opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME], voice_out_tx: params.voice_out_tx.clone(), transmit_active: params.transmit_active.clone(), output_muted: params.output_muted.clone(), frames_sent: params.frames_sent.clone(), mic_gain: params.mic_gain, voice_activity_selector: params.voice_activity_selector.clone(), vad_detector: crate::vad::WebRtcFallbackVad::default(), silero_coreml_worker: None, current_vad_backend: crate::VadBackend::WebrtcVad, fallback_warned_backend: None, vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), audio_processing_config: params.audio_processing_config.clone(), sonora_processor: crate::processor::SonoraProcessor::new(), audio_processing_stats: params.audio_processing_stats.clone(), pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES], pending_10ms_len: 0, pre_roll_buf: [[0_i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES], pre_roll_head: 0, pre_roll_count: 0, pre_roll_flushed: false, capture_frame_seq: 0, wav_recorder, }) } fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) { if self.fallback_warned_backend == Some(failed_backend) { return; } self.fallback_warned_backend = Some(failed_backend); if self.capture_frame_seq < 128 { tracing::info!( target: "chanora_audio", backend = failed_backend.as_str(), seq = self.capture_frame_seq, "VAD backend warming up; using WebRTC fallback" ); } else { tracing::warn!( target: "chanora_audio", backend = failed_backend.as_str(), "VAD backend unavailable; using WebRTC fallback for runtime detection" ); } } /// Consume the i16 mono buffer delivered by VPIO, accumulate /// to a 20 ms frame boundary, encode + send when PTT is held. /// /// VPIO's input element delivers samples already at the /// stream format we pinned (48 kHz Int16 mono interleaved). /// In practice "interleaved mono" is the same byte layout as /// "planar mono" so we just take the buffer as-is. fn ingest_i16(&mut self, samples: &[i16]) { let mut offset = 0; while offset < samples.len() { let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len; let take = remaining.min(samples.len() - offset); self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take] .copy_from_slice(&samples[offset..offset + take]); self.pending_10ms_len += take; offset += take; if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES { let frame = self.pending_10ms; self.process_10ms_capture_frame(&frame); self.pending_10ms_len = 0; } } if !self.transmit_active.load(Ordering::Relaxed) { // Drain accumulator while muted so we don't pop on the // PTT release edge. Matches cpal-side behaviour. self.pcm_accum.clear(); return; } // Drain complete 20 ms frames out of the accumulator, encode // each, send the resulting Opus packet on the protocol // queue. The `while` covers the case where a single VPIO // callback delivers more than one frame's worth (rare on // iOS where the HW IO buffer duration aligns with the // Opus frame, but always possible during route changes). while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES { // Use a stack-allocated frame buffer to avoid the // per-callback allocation a `drain(..N).collect()` // would incur. The encoder doesn't need ownership. let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES]; frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]); self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES); match self.encoder.encode(&frame, &mut self.opus_out[..]) { Ok(len) => { crate::opus_voice::send_voip_frame( &self.voice_out_tx, &self.frames_sent, &self.opus_out, len, || { warn!( target: "chanora_audio", "ios VPIO: voice_out queue full; dropping frame" ); }, || { debug!( target: "chanora_audio", "ios VPIO: voice_out closed; capture pipeline stopping" ); }, ); } Err(e) => { error!(target: "chanora_audio", error = %e, "ios VPIO opus encode failed"); } } } } fn process_10ms_capture_frame(&mut self, samples: &[i16; crate::frame::FRAME_10MS_SAMPLES]) { let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) { *dst = crate::frame::i16_to_f32(src); } let input_dbfs = crate::frame::dbfs(&frame); // WAV tap: raw mic (before processing, DIAG_002). if let Ok(guard) = self.wav_recorder.try_lock() { if let Some(rec) = guard.as_ref() { rec.push_raw_mic(&frame); } } // Read config once per frame (try_lock: non-blocking, falls back to // last-known values if the lock is contended — safe to miss one frame). let ( run_ns, run_agc, run_hpf, vad_backend, vad_hangover, debug_wav_dump_enabled, route, processing_backend, ) = self .audio_processing_config .try_lock() .map(|cfg| { let ns = cfg.ns != crate::EffectOwner::Off && cfg.ns != crate::EffectOwner::Platform; let agc = cfg.agc != crate::EffectOwner::Off && cfg.agc != crate::EffectOwner::Platform; let hpf = cfg.hpf_enabled; ( ns, agc, hpf, cfg.vad_backend, cfg.vad_hangover_ms, cfg.debug_wav_dump_enabled, cfg.route, cfg.processing_backend, ) }) .unwrap_or(( false, false, true, crate::VadBackend::WebrtcVad, crate::voice_activity::VAD_HANGOVER_MS, false, crate::AudioRoute::Unknown, crate::AudioBackend::PlatformVoiceProcessing, )); let voice_activity_mode = self .voice_activity_selector .as_ref() .map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity) .unwrap_or(false); if !voice_activity_mode { self.silero_coreml_worker = None; self.current_vad_backend = crate::VadBackend::Disabled; self.fallback_warned_backend = None; self.audio_processing_stats.set_vad_fallback_active(false); } // Switch VAD backend only while VoiceActivity mode is active. if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() { if debug_wav_dump_enabled { if recorder_guard.is_none() { *recorder_guard = Some(crate::debug_wav::WavDebugRecorder::start( route, processing_backend, )); } } else if let Some(recorder) = recorder_guard.take() { recorder.stop(); } } if voice_activity_mode && vad_backend != self.current_vad_backend { self.current_vad_backend = vad_backend; self.fallback_warned_backend = None; if vad_backend == crate::VadBackend::SileroOnnx { self.silero_coreml_worker = crate::vad::apple_coreml::AppleCoreMlVadWorker::try_new(); if self.silero_coreml_worker.is_none() { self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); self.audio_processing_stats.set_vad_fallback_active(true); } else { self.audio_processing_stats.set_vad_fallback_active(false); } } else { self.silero_coreml_worker = None; self.audio_processing_stats.set_vad_fallback_active(false); } // Reset VAD state machine timers on backend switch. self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new( crate::voice_activity::VAD_OPEN_AFTER_MS, vad_hangover, crate::voice_activity::VAD_MIN_TX_MS, ); self.vad_state.reset(); } let transmit_active = self.transmit_active.load(Ordering::Relaxed); if voice_activity_mode { // Keep the VAD state machine aligned with the active config only // while VoiceActivity mode owns the transmit gate. self.vad_state.configure( crate::voice_activity::VAD_OPEN_AFTER_MS, vad_hangover, crate::voice_activity::VAD_MIN_TX_MS, ); } // VPIO owns AEC. Keep the legacy non-AEC conditioning path here until // the raw WebRTC APM path is explicitly selected. if run_ns || run_agc || run_hpf { use crate::processor::sonora::SonoraConfig; use crate::processor::AudioProcessor; let new_cfg = SonoraConfig { hpf: run_hpf, aec3: false, ns: run_ns, agc2: run_agc, }; if new_cfg != *self.sonora_processor.config() { self.sonora_processor.apply_config(new_cfg); } self.sonora_processor.process_capture(&mut frame); } // VAD: only evaluate while VoiceActivity mode is active. let (vad_probability, gate_open) = if voice_activity_mode { self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1); let capture_seq = self.capture_frame_seq; let mut used_fallback_vad = false; let vad = if vad_backend == crate::VadBackend::Disabled { crate::vad::VadOutput { probability: 1.0, speech: true, } } else if vad_backend == crate::VadBackend::SileroOnnx { if let Some(worker) = self.silero_coreml_worker.as_ref() { let enqueued = worker.try_send(capture_seq, &frame); if !worker.is_stale(capture_seq) { let p = worker.latest_probability(); crate::vad::VadOutput { probability: p, speech: p >= 0.5, } } else if enqueued { crate::vad::VadOutput { probability: 0.0, speech: false, } } else { used_fallback_vad = true; self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); crate::vad::VoiceActivityDetector::process_10ms( &mut self.vad_detector, &frame, ) } } else { used_fallback_vad = true; self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) } } else { crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame) }; self.audio_processing_stats .set_vad_fallback_active(used_fallback_vad); (vad.probability, self.vad_state.update(vad.speech)) } else { (0.0, false) }; let output_muted = self.output_muted.load(Ordering::Relaxed); if let Some(selector) = &self.voice_activity_selector { selector.set_voice_activity_open(voice_activity_mode && gate_open && !output_muted); } self.audio_processing_stats.update_capture( input_dbfs, crate::frame::dbfs(&frame), vad_probability, voice_activity_mode && gate_open && !output_muted, transmit_active, ); // WAV tap: processed mic (after Rust DSP, DIAG_002). if let Ok(guard) = self.wav_recorder.try_lock() { if let Some(rec) = guard.as_ref() { rec.push_processed_mic(&frame); } } // Convert to i16 for accumulation. let mut pcm_frame = [0_i16; crate::frame::FRAME_10MS_SAMPLES]; if (self.mic_gain - 1.0).abs() < f32::EPSILON { for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) { *dst = crate::frame::f32_to_i16(src); } } else { let gain = self.mic_gain; for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) { let scaled = crate::frame::f32_to_i16(src) as f32 * gain; *dst = scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16; } } // Update pre-roll ring buffer (VAD_004: preserve first syllable). let slot_idx = self.pre_roll_head % PRE_ROLL_FRAMES; self.pre_roll_buf[slot_idx] = pcm_frame; self.pre_roll_head = (self.pre_roll_head + 1) % PRE_ROLL_FRAMES; if self.pre_roll_count < PRE_ROLL_FRAMES { self.pre_roll_count += 1; } // If the transmit gate just opened and we haven't flushed the // pre-roll yet, drain it into the accumulator. if transmit_active && !self.pre_roll_flushed { self.pre_roll_flushed = true; // The oldest frame in the ring is at // (pre_roll_head + PRE_ROLL_FRAMES - pre_roll_count) % PRE_ROLL_FRAMES. // We emit frames in chronological order (oldest first), excluding // the frame we just wrote (which goes into pcm_accum normally below). let oldest = (self.pre_roll_head + PRE_ROLL_FRAMES - self.pre_roll_count) % PRE_ROLL_FRAMES; // Emit pre_roll_count - 1 frames (the -1 excludes the current frame // which will be added below in the normal path). let pre_roll_to_emit = self.pre_roll_count.saturating_sub(1); for i in 0..pre_roll_to_emit { let idx = (oldest + i) % PRE_ROLL_FRAMES; self.pcm_accum.extend_from_slice(&self.pre_roll_buf[idx]); } } else if !transmit_active { // Gate closed — reset the flush flag so pre-roll fires again // on the next gate open. self.pre_roll_flushed = false; } if !transmit_active { return; } self.pcm_accum.extend_from_slice(&pcm_frame); } } /// Live iOS audio unit wrapper. Construct + start = audio /// flowing; drop = audio stopped. pub struct IosVoiceUnit { // Drop = stop the audio unit (severs render callback). The // wrapper's own Drop calls AudioComponentInstanceDispose // after stop returns. unit: Option, } impl IosVoiceUnit { #[cfg(target_os = "ios")] fn exec_main_queue_lifecycle( &mut self, op: impl FnOnce(&mut AudioUnit) -> Result<(), String> + Send + 'static, ) -> Result<(), AudioError> { let unit = self .unit .take() .ok_or_else(|| AudioError::Backend("vpio lifecycle: audio unit missing".to_string()))?; let (tx, rx) = std::sync::mpsc::sync_channel::>(1); let unit_arc = Arc::new(Mutex::new(Some(unit))); let unit_arc2 = Arc::clone(&unit_arc); dispatch2::DispatchQueue::main().exec_async(move || { let mut guard = unit_arc2.lock().unwrap(); let unit = guard.as_mut().unwrap(); let _ = tx.send(op(unit)); }); let result = match rx.recv() { Ok(result) => result, Err(_) => Err("vpio lifecycle: main thread channel closed unexpectedly".to_string()), }; self.unit = unit_arc.lock().unwrap().take(); result.map_err(AudioError::Backend) } /// Open a VoiceProcessingIO AudioUnit, pin its stream format /// to 48 kHz Int16 mono on both buses, install render + input /// callbacks, and start it. The unit begins pumping audio /// immediately on return — input callback fires when the mic /// captures samples, render callback fires when the hardware /// needs samples to play. /// /// Parameters mirror the capture + playback inputs the cpal /// and SDL backends accept so engine.rs can swap backends with /// a `cfg`. /// /// * `handler` — shared AudioHandler the inbound forwarder /// feeds Opus packets into. The output render callback pulls /// decoded f32 frames from it (48 kHz stereo) and downmixes /// to the i16 mono buffer VPIO expects. /// * `output_gain` / `output_muted` — same atomics the cpal /// and SDL output paths read on every callback so the master /// volume + local-mute UI works identically across backends. /// * `voice_out_tx` — channel the capture pipeline sends /// encoded `OutPacket`s on. /// * `transmit_active` — PTT gate flag the capture pipeline /// consults before encoding. /// * `frames_sent` — counter the bridge stats surface reads. /// * `mic_gain` — pre-encode amplitude scale. /// /// Capture wiring landed in commit 3; playback wiring landed /// in commit 4. Route-change observation is commit 5. pub(crate) fn start(params: VoiceAudioParams) -> Result { // Construct the VoiceProcessingIO AudioUnit. cpal exposes // `Default::default()` which on iOS picks the inferior // RemoteIO unit; we explicitly pick VPIO. `coreaudio-rs` // returns an already-initialized unit from `new`, but we // need to set properties before init so use // `new_uninitialized` and call `initialize` ourselves // after the property set is complete. let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO) .map_err(|e| AudioError::Backend(format!("vpio audio unit new: {e}")))?; // Enable input I/O on element 1. VPIO's input element is // OFF by default — without this `kAudioOutputUnitProperty_EnableIO` // toggle no audio flows in and the input callback never // fires. The constant is 2003 per Apple's headers // (`AudioUnitProperties.h`). The value is a u32 with // 1 = enabled. Element::Input on `Scope::Input` is the // mic side of the unit (element 1 of bus 1; despite the // confusing nomenclature, `Scope::Input` here means // "input to the unit" i.e. mic samples coming IN). // // Output element 0 is enabled by default for any // `kAudioUnitType_Output` subtype (which VPIO is), so we // don't need to toggle anything for playback. // // Apple's documented sequence for VPIO setup: // 1. AudioComponentInstanceNew (= AudioUnit::new_uninitialized) // 2. EnableIO on element 1 (= this set_property call) // 3. Set stream format on both elements // 4. Install callbacks // 5. AudioUnitInitialize (= unit.initialize) // 6. AudioOutputUnitStart (= unit.start) const K_AUDIO_OUTPUT_UNIT_PROPERTY_ENABLE_IO: u32 = 2003; let enable: u32 = 1; unit.set_property( K_AUDIO_OUTPUT_UNIT_PROPERTY_ENABLE_IO, Scope::Input, Element::Input, Some(&enable), ) .map_err(|e| AudioError::Backend(format!("vpio enable input I/O: {e}")))?; // Note: we keep VPIO's voice processing chain ENABLED // (AEC + AGC + NS on the mic path) because it gives us // clean capture for free. The historical playback // breakage we attributed to this chain (commit c16318c // tried to bypass it) was actually caused by the // AVAudioSession mode .voiceChat ducking output to the // earpiece \u2014 fixed in AppDelegate.swift by switching // to .default + .defaultToSpeaker. With the session mode // correct, voice processing can stay on. // Stream format. Apple's iOS canonical format for // AudioUnits is Linear PCM, 16-bit signed integer samples // (see "Canonical formats" in the Audio Unit Hosting Guide // for iOS). Mono channel matches the Opus encoder + the // mic input pipe. 48 kHz matches every layer above us. // // The StreamFormat is set on: // * OUTPUT_BUS (bus 0), Scope::Input — the format WE // push to the unit, i.e. what fill_buffer writes. // * INPUT_BUS (bus 1), Scope::Output — the format we // RECEIVE from the unit, i.e. what the mic input // callback hands us. // This pairing is documented in Audio Unit Hosting Guide // for iOS §"Specifying the Audio Stream Format". let stream_format = StreamFormat { sample_rate: SAMPLE_RATE_HZ, sample_format: SampleFormat::I16, flags: LinearPcmFlags::IS_SIGNED_INTEGER | LinearPcmFlags::IS_PACKED, channels: 1, }; unit.set_stream_format(stream_format, Scope::Input, OUTPUT_BUS) .map_err(|e| { AudioError::StreamConfig(format!( "vpio set output stream format (bus 0 input scope): {e}" )) })?; unit.set_stream_format(stream_format, Scope::Output, INPUT_BUS) .map_err(|e| { AudioError::StreamConfig(format!( "vpio set input stream format (bus 1 output scope): {e}" )) })?; // Build the capture pipeline and move it into the input // callback. The Opus encoder + accumulator + opus_out // scratch are owned by the closure — no Mutex needed // because the input callback is the sole writer/reader on // the audio thread. let wav_recorder = Arc::new(Mutex::new({ let cfg = params.audio_processing_config.lock().unwrap().clone(); if cfg.debug_wav_dump_enabled { Some(crate::debug_wav::WavDebugRecorder::start( cfg.route, cfg.processing_backend, )) } else { None } })); let mut capture_state = IosCaptureState::new(¶ms, wav_recorder.clone())?; unit.set_input_callback(move |args: render_callback::Args>| { // VPIO with our pinned stream format delivers // interleaved Int16 mono. `args.data.buffer` is a // `&mut [i16]` of length num_frames * channels = N * 1. // coreaudio-rs handles the AudioBufferList plumbing + // the AudioUnitRender call internally before invoking // this closure. capture_state.ingest_i16(args.data.buffer); Ok(()) }) .map_err(|e| AudioError::Backend(format!("vpio set input callback: {e}")))?; // Install the render callback that drives playback. The // buffer iOS hands us is uninitialised — we MUST fill it // (writing silence if we have nothing, never leaving stale // frames). // // Pipeline per callback: // 1. Lock the AudioHandler, ask it to fill a scratch // f32 stereo buffer (length = 2 * num_frames). The // handler runs Opus decode + per-client jitter // buffer + mix. Same primitive cpal + SDL output // paths use; this is the platform-neutral playback // contract from `tsclientlib::audio::AudioHandler`. // 2. Downmix to i16 mono with master gain. VPIO expects // mono int16 (the stream format we pinned above); // the handler produces stereo f32. We average L+R // to a single mono channel rather than dropping R — // the cpal-side mono-output path made the same // mistake briefly (commit 6a4dbad / fix) and lost // half the spatial mix. // 3. Local-mute zeroes the output but STILL drains // AudioHandler in step 1 so its jitter buffer // doesn't grow unbounded while muted. This is the // contract every other backend follows (matches // SdlOutput::callback and the cpal output stream). // // Build the playback pipeline (direct fill_buffer in // render callback; matches tsclientlib's reference SDL // example at // tsclientlib/examples/audio_utils/ts_to_audio.rs). // // The earlier ring-buffer attempt (rc.8+73..+74) decoupled // AudioHandler from the render callback via a 50 Hz // producer task + SPSC ring buffer. The +74 diagnostics // showed that approach was making things worse: the // producer drained AudioHandler at 50 Hz, but iOS VPIO // calls our render callback at ~43.5 Hz (consuming 1440 // mono samples per 23 ms call). With consumer slightly // slower than producer in chunks-per-second but each // consumer pull being larger, the ring averaged out empty // — fill_buffer was returning silence 65-84% of ticks // because we drained it too aggressively before packets // arrived. Linux/SDL's same pattern works fine because // SDL calls fill_buffer at exactly the device callback // rate. // // Revert to direct call: render callback locks // AudioHandler, asks for `num_frames` stereo frames, and // immediately downmixes to i16 mono into the output // buffer. Same as Linux/SDL, just stereo-f32 -> mono-i16 // converted at the boundary. let mut scratch_stereo: Vec = Vec::with_capacity(2048); let handler_for_render = params.handler.clone(); let output_gain_for_render = params.output_gain.clone(); let output_muted_for_render = params.output_muted.clone(); let audio_processing_stats_for_render = params.audio_processing_stats.clone(); let wav_recorder_for_render = wav_recorder.clone(); // Diagnostic counters (sampled every 100 callbacks ~= 2 s). let mut cb_count: u64 = 0; let mut last_num_frames: usize = 0; let mut num_frames_changes: u32 = 0; let mut callbacks_with_audio: u64 = 0; let mut callbacks_with_silence: u64 = 0; let mut render_ref_accum = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; let mut render_ref_len: usize = 0; let mut render_recorder_active = false; unit.set_render_callback(move |args: render_callback::Args>| { let out: &mut [i16] = args.data.buffer; let num_frames = out.len(); // AudioHandler produces 48 kHz stereo f32 (= num_frames * 2 floats). let needed = num_frames * 2; if scratch_stereo.len() < needed { scratch_stereo.resize(needed, 0.0); } // Zero the live slice. AudioHandler::fill_buffer is // additive (does NOT clear); residual values from // earlier callbacks (when scratch was bigger) would // leak through otherwise. scratch_stereo[..needed].fill(0.0); // Non-blocking fill on the realtime callback thread. // If the inbound forwarder currently owns this mutex, // emit this period as silence instead of blocking and // risking an AudioUnit underrun pop/click. match handler_for_render.try_lock() { Ok(mut h) => { let _removed = h.fill_buffer(&mut scratch_stereo[..needed]); } Err(std::sync::TryLockError::WouldBlock) => { audio_processing_stats_for_render.increment_callback_xrun(); // scratch_stereo is already zeroed above. } Err(std::sync::TryLockError::Poisoned(e)) => { // Never panic on the realtime IO thread. warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {e}"); } } let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed)); let muted = output_muted_for_render.load(Ordering::Relaxed); let mix_stats = crate::voice_render::downmix_stereo_f32_to_mono_i16( &scratch_stereo[..needed], out, gain, muted, ); if mix_stats.clipped_samples > 0 { audio_processing_stats_for_render.add_clipped_samples(mix_stats.clipped_samples); } audio_processing_stats_for_render.update_render( crate::frame::dbfs(&scratch_stereo[..needed]), num_frames as u32, ); if let Ok(guard) = wav_recorder_for_render.try_lock() { if let Some(rec) = guard.as_ref() { if !render_recorder_active { render_ref_len = 0; render_ref_accum.fill(0.0); render_recorder_active = true; } let mut idx = 0; while idx + 1 < needed { let mono = (scratch_stereo[idx] + scratch_stereo[idx + 1]) * 0.5; render_ref_accum[render_ref_len] = mono; render_ref_len += 1; idx += 2; if render_ref_len == crate::frame::FRAME_10MS_SAMPLES { rec.push_render_reference(&render_ref_accum); render_ref_len = 0; } } } else { render_recorder_active = false; } } else { render_recorder_active = false; } // Track audio-vs-silence for the diagnostic. if mix_stats.peak_i16 > 0 { callbacks_with_audio = callbacks_with_audio.wrapping_add(1); } else { audio_processing_stats_for_render.increment_output_underrun(); callbacks_with_silence = callbacks_with_silence.wrapping_add(1); } // Diagnostic sampling. if last_num_frames != 0 && last_num_frames != num_frames { num_frames_changes = num_frames_changes.wrapping_add(1); } last_num_frames = num_frames; cb_count = cb_count.wrapping_add(1); if cb_count.is_multiple_of(100) { debug!( target: "chanora_audio", cb = cb_count, num_frames, frames_changes = num_frames_changes, callbacks_with_audio, callbacks_with_silence, peak_out_i16 = mix_stats.peak_i16, gain, "ios audio unit render callback diagnostic sample (direct fill_buffer)" ); } Ok(()) }) .map_err(|e| AudioError::Backend(format!("audio unit set render callback: {e}")))?; // Finalise the unit — allocates internal buffers per the // stream formats we set above. After initialize() most // property changes are rejected (you have to uninitialize + // re-initialize), which is why the property set must come // first. // // AudioUnit::initialize() issues an RPC to the CoreAudio server. // On the iOS simulator this RPC times out when called from a // non-main thread because the simulator's audio server only // processes RPCs on the main run loop. // // Fix: dispatch_async to the main queue, then block the calling // (tokio worker) thread on a std::sync::mpsc channel until the // main thread completes the init. This is safe because: // 1. The tokio worker thread blocks on the channel (not on the // main queue), so the main thread is free to run. // 2. AudioUnit is Send (coreaudio-rs marks it unsafe impl Send). // 3. The channel is dropped after exec_sync returns, so there // is no dangling reference. { let (tx, rx) = std::sync::mpsc::sync_channel::>(1); // Move unit into the Arc so it can cross thread boundaries. let unit_arc = std::sync::Arc::new(std::sync::Mutex::new(Some(unit))); let unit_arc2 = unit_arc.clone(); dispatch2::DispatchQueue::main().exec_async(move || { let mut guard = unit_arc2.lock().unwrap(); let u = guard.as_mut().unwrap(); let result = u .initialize() .map_err(|e| format!("vpio initialize: {e}")) .and_then(|_| u.start().map_err(|e| format!("vpio start: {e}"))); let _ = tx.send(result); }); // Block the tokio worker thread until the main thread finishes. // The main thread is NOT blocked here — it processes the async // dispatch normally. match rx.recv() { Ok(Ok(())) => {} Ok(Err(msg)) => return Err(AudioError::Backend(msg)), Err(_) => { return Err(AudioError::Backend( "vpio init: main thread channel closed unexpectedly".to_string(), )) } } unit = unit_arc.lock().unwrap().take().unwrap(); } info!( target: "chanora_audio", sample_rate_hz = SAMPLE_RATE_HZ, channels = stream_format.channels, sample_format = ?stream_format.sample_format, "ios VPIO audio unit started" ); // Read back the ACTUAL stream format VPIO accepted on each // bus (iOS sometimes substitutes its own format if the // hardware can't satisfy our preference) and the actual // AVAudioSession sample rate + IO buffer duration. Without // these we can't tell whether our 48 kHz Int16 mono format // was honoured or silently downgraded to e.g. 44.1 kHz // Float32 (which would cause our render callback to write // i16 values into a buffer iOS interprets as f32 = severe // distortion). Diagnostic prompted by external review // pointing out that 'preferredSampleRate' is a hint, not // a guarantee \u2014 must verify post-init. match unit.output_stream_format() { Ok(fmt) => info!( target: "chanora_audio", sample_rate = fmt.sample_rate, channels = fmt.channels, sample_format = ?fmt.sample_format, flags = ?fmt.flags, "ios VPIO actual OUTPUT stream format (post-init)" ), Err(e) => warn!( target: "chanora_audio", error = %e, "ios VPIO output_stream_format read failed" ), } match unit.input_stream_format() { Ok(fmt) => info!( target: "chanora_audio", sample_rate = fmt.sample_rate, channels = fmt.channels, sample_format = ?fmt.sample_format, flags = ?fmt.flags, "ios VPIO actual INPUT stream format (post-init)" ), Err(e) => warn!( target: "chanora_audio", error = %e, "ios VPIO input_stream_format read failed" ), } Ok(Self { unit: Some(unit) }) } /// Restart the audio unit after route change handling. /// /// Route rebinding on iOS is most reliable when we bounce the /// VoiceProcessingIO unit through an uninitialize/reinitialize /// cycle, then start again. /// #[cfg(target_os = "ios")] pub fn restart(&mut self) -> Result<(), AudioError> { self.exec_main_queue_lifecycle(|unit| { unit.stop().map_err(|e| format!("vpio restart stop: {e}"))?; unit.uninitialize() .map_err(|e| format!("vpio restart uninit: {e}"))?; unit.initialize() .map_err(|e| format!("vpio restart init: {e}"))?; unit.start() .map_err(|e| format!("vpio restart start: {e}"))?; Ok(()) })?; info!(target: "chanora_audio", "ios VPIO audio unit restarted"); Ok(()) } /// Pause the audio unit during an interruption. #[cfg(target_os = "ios")] pub fn pause(&mut self) -> Result<(), AudioError> { self.exec_main_queue_lifecycle(|unit| { unit.stop().map_err(|e| format!("vpio pause stop: {e}")) }) } /// Resume the audio unit after an interruption. #[cfg(target_os = "ios")] pub fn resume(&mut self) -> Result<(), AudioError> { self.exec_main_queue_lifecycle(|unit| { unit.start().map_err(|e| format!("vpio resume start: {e}")) }) } } impl Drop for IosVoiceUnit { fn drop(&mut self) { // Stop the audio unit so the render callback no longer // fires. The coreaudio-rs wrapper's own Drop calls // AudioComponentInstanceDispose afterwards. if let Some(unit) = self.unit.as_mut() { if let Err(e) = unit.stop() { warn!(target: "chanora_audio", error = %e, "ios audio unit stop on drop failed"); } else { info!(target: "chanora_audio", "ios audio unit stopped"); } } } }