From 3b1724e970889bdfc6b393bcff9e7a4584aa5ceb Mon Sep 17 00:00:00 2001 From: EdisonJwa Date: Sun, 17 May 2026 00:45:50 +0800 Subject: [PATCH] feat(audio,ios): wire IosVoiceUnit into AudioEngine, cfg-gate cpal away on iOS (commit 2/5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split AudioEngine::start_with_gate into two backends: * start_with_gate_cpal — non-iOS path, the existing cpal + (SDL on Linux) flow, renamed verbatim, no behavioural change. * start_with_gate_ios — iOS path, constructs a single IosVoiceUnit (VoiceProcessingIO via coreaudio-rs) for combined mic + speaker. Spawns the same inbound forwarder task that pumps Opus packets into AudioHandler. The public entry point start_with_gate dispatches at the top via cfg(target_os = "ios") so callers stay backend-agnostic. Struct field changes: * _input_stream : cfg-gated to not(ios) * _output_stream : cfg-gated to not(ios), keeps the Linux=SdlOutput / else=cpal::Stream split * _ios_voice_unit: new field, cfg-gated to ios, owns the VPIO AudioUnit for the engine's lifetime. Module-level cfg-gating: * All cpal-only helpers (try_open_capture, build_input_stream, build_output_stream, CaptureState + impl, ToF32 / FromF32 traits and impls, PlaybackResampleState) are now wrapped with #[cfg(not(target_os = "ios"))]. Same for the audiopus encoder + cpal trait imports — iOS doesn't pull libopus into the engine yet (commit 3 will, once the VPIO input callback wires into CaptureState). Behaviour on iOS for THIS commit: * AudioEngine starts cleanly, IosVoiceUnit::start succeeds (VPIO unit allocates + initialises + starts). * Mic capture is dropped (the input callback is a no-op stub). * Output emits silence (the render callback fills the buffer with zeros). * Inbound forwarder still runs and pushes Opus packets into AudioHandler — they accumulate in the jitter buffer but no fill_buffer drain happens (commit 4 fixes that), so the buffer will grow up to MAX_BUFFER_TIME (~0.5 s) and then tsclientlib starts dropping the oldest frames. This is fine for now — the point of this commit is verifying the AudioUnit constructs + starts cleanly on the device. Audible silence is the expected state until commits 3/4 land. Build verify (Linux host): cargo check -p chanora_audio clean in 0.53s. iOS-side compile happens on the Mac via the Xcode build the user will trigger next. --- crates/chanora_audio/src/engine.rs | 176 ++++++++++++++++++++++++++++- 1 file changed, 173 insertions(+), 3 deletions(-) diff --git a/crates/chanora_audio/src/engine.rs b/crates/chanora_audio/src/engine.rs index 0bd6b98..ed6e7ca 100644 --- a/crates/chanora_audio/src/engine.rs +++ b/crates/chanora_audio/src/engine.rs @@ -8,12 +8,17 @@ use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; +#[cfg(not(target_os = "ios"))] use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +#[cfg(not(target_os = "ios"))] use cpal::{SampleFormat, SizedSample}; use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; +#[cfg(not(target_os = "ios"))] +#[cfg(not(target_os = "ios"))] use audiopus::coder::Encoder as OpusEncoder; +#[cfg(not(target_os = "ios"))] use audiopus::{ Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels, SampleRate as OpusSampleRate, @@ -103,13 +108,20 @@ pub struct AudioEngine { // cpal's Stream isn't Send on some backends; we keep them in an // Option wrapped by Mutex so stop() can move them out. On Linux // the output side is `crate::sdl_output::SdlOutput` instead of a - // cpal Stream (see the SDD note inside `sdl_output.rs`); the same - // unsafe Send/Sync impl below covers both. + // cpal Stream (see the SDD note inside `sdl_output.rs`); the + // same unsafe Send/Sync impl below covers both. On iOS both + // sides collapse into a single `IosVoiceUnit` (one + // VoiceProcessingIO AudioUnit hosts mic + speaker) — cpal is + // unused on iOS for the reasons documented in + // `ios_voice_unit.rs`. + #[cfg(not(target_os = "ios"))] _input_stream: Mutex>, #[cfg(target_os = "linux")] _output_stream: Mutex>, - #[cfg(not(target_os = "linux"))] + #[cfg(all(not(target_os = "linux"), not(target_os = "ios")))] _output_stream: Mutex>, + #[cfg(target_os = "ios")] + _ios_voice_unit: Mutex>, // Hand the inbound-voice forwarder task a shutdown signal. shutdown_tx: Option>, /// True if the capture stream actually opened. If false (typical @@ -170,6 +182,38 @@ impl AudioEngine { /// upstream (typically [`crate::TransmitModeSelector`]) is the /// authoritative writer of `transmit_active`. See SAD-083. pub fn start_with_gate( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + // iOS routes to a separate backend (VoiceProcessingIO via + // coreaudio-rs) because cpal's iOS RemoteIO path produces + // mono-only output bound to a stale physical transducer + // (see `ios_voice_unit.rs` for the long version). Every + // other platform stays on the cpal / SDL flow below. + #[cfg(target_os = "ios")] + { + return Self::start_with_gate_ios( + cfg, + voice_out_tx, + voice_in_rx, + transmit_gate, + ); + } + #[cfg(not(target_os = "ios"))] + { + Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate) + } + } + + /// Non-iOS implementation: cpal capture + (cpal | SDL2) output. + /// Kept as a separate function so the iOS path can short-circuit + /// at the top of `start_with_gate` without dragging a 200-line + /// cfg-gated block. Body is the pre-iOS-port code, unchanged + /// except for the new function name + signature. + #[cfg(not(target_os = "ios"))] + fn start_with_gate_cpal( cfg: AudioEngineConfig, voice_out_tx: mpsc::Sender, mut voice_in_rx: mpsc::Receiver, @@ -477,6 +521,118 @@ impl AudioEngine { }) } + /// iOS implementation: a single VoiceProcessingIO AudioUnit + /// drives mic capture + speaker playback (see + /// `ios_voice_unit.rs` for why cpal is unsuitable on iOS). + /// This mirrors `start_with_gate_cpal` in scaffolding — + /// atomics, audio handler, inbound forwarder task — but + /// replaces the two cpal stream constructions with a single + /// `IosVoiceUnit::start` call. + #[cfg(target_os = "ios")] + fn start_with_gate_ios( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + mut voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + info!( + target: "chanora_audio", + "starting audio engine: iOS VoiceProcessingIO backend" + ); + + // iOS AVAudioSession configuration is performed Swift-side + // in `apps/chanora_flutter/ios/Runner/AppDelegate.swift` + // BEFORE Flutter starts its audio pipeline. The category + + // mode pair set there (`.playAndRecord` + `.default`) is + // what VPIO binds against. Logging here just records that + // the engine-start path acknowledges the request; the + // actual session mutation lives in Swift because it must + // happen before Dart loads. + if cfg.mobile_voice_preset { + info!( + target: "chanora_audio", + "ios: voice-chat session mode requested — AVAudioSession configured in AppDelegate" + ); + } + + let transmit_flag_for_capture = transmit_gate.flag_arc(); + let frames_sent = Arc::new(AtomicU32::new(0)); + let frames_received = Arc::new(AtomicU32::new(0)); + let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits())); + let output_muted = Arc::new(AtomicBool::new(false)); + + let audio_handler: Arc>> = + Arc::new(Mutex::new(AudioHandler::new())); + + // Construct the VPIO unit. Commit 1 ships a no-op callback + // pair; commits 3 + 4 land the real capture + playback + // wiring. Construction failure here is fatal (mirrors how + // the cpal output-stream construction failure is fatal in + // the non-iOS path). + let ios_voice_unit = crate::ios_voice_unit::IosVoiceUnit::start( + audio_handler.clone(), + output_gain.clone(), + output_muted.clone(), + voice_out_tx, + transmit_flag_for_capture, + frames_sent.clone(), + cfg.mic_gain, + )?; + + // Capture is always considered active on iOS — VPIO's + // input element is wired up by the AudioUnit itself, no + // separate "did the capture stream open" question to + // answer. If the user denied mic permission VPIO will + // simply hand us silence buffers. + let capture_active = true; + + // Inbound forwarder: same shape as the non-iOS path. Pumps + // Opus packets from `voice_in_rx` into AudioHandler so the + // VPIO render callback (commit 4) finds decoded frames + // waiting. + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + let handler_for_task = audio_handler.clone(); + let frames_received_for_task = frames_received.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = &mut shutdown_rx => { + debug!(target: "chanora_audio", "inbound forwarder shutting down"); + break; + } + item = voice_in_rx.recv() => { + match item { + Some(v) => { + let id = SessionAudioId(v.from_client); + let mut h = handler_for_task.lock().unwrap(); + if let Err(e) = h.handle_packet(id, v.packet) { + debug!(target: "chanora_audio", error = %e, "decode failed"); + } else { + frames_received_for_task.fetch_add(1, Ordering::Relaxed); + } + } + None => break, + } + } + } + } + }); + + let ptt_watchdog: Option = None; + + Ok(Self { + transmit_gate, + frames_sent, + frames_received, + output_gain, + output_muted, + _ios_voice_unit: Mutex::new(Some(ios_voice_unit)), + shutdown_tx: Some(shutdown_tx), + capture_active, + ptt_watchdog, + }) + } + /// Stop the engine. Idempotent. pub fn stop(&mut self) { if let Some(tx) = self.shutdown_tx.take() { @@ -596,6 +752,7 @@ impl Drop for AudioEngine { // ---------- Capture pipeline ---------- +#[cfg(not(target_os = "ios"))] fn try_open_capture( in_dev: &cpal::Device, voice_out_tx: mpsc::Sender, @@ -707,6 +864,7 @@ fn try_open_capture( Ok(stream) } +#[cfg(not(target_os = "ios"))] struct CaptureState { encoder: OpusEncoder, in_sample_rate: u32, @@ -731,6 +889,7 @@ struct CaptureState { frames_sent: Arc, } +#[cfg(not(target_os = "ios"))] impl CaptureState { fn new( encoder: OpusEncoder, @@ -878,25 +1037,30 @@ impl CaptureState { } /// Per-sample format conversion to f32 in the range [-1.0, 1.0]. +#[cfg(not(target_os = "ios"))] trait ToF32 { fn to_f32_sample(self) -> f32; } +#[cfg(not(target_os = "ios"))] impl ToF32 for f32 { fn to_f32_sample(self) -> f32 { self } } +#[cfg(not(target_os = "ios"))] impl ToF32 for i16 { fn to_f32_sample(self) -> f32 { f32::from(self) / f32::from(i16::MAX) } } +#[cfg(not(target_os = "ios"))] impl ToF32 for u16 { fn to_f32_sample(self) -> f32 { (f32::from(self) - f32::from(i16::MAX) - 1.0) / f32::from(i16::MAX) } } +#[cfg(not(target_os = "ios"))] fn build_input_stream( device: &cpal::Device, config: &cpal::StreamConfig, @@ -924,6 +1088,7 @@ where // ---------- Playback pipeline ---------- #[cfg(not(target_os = "linux"))] +#[cfg(not(target_os = "ios"))] fn build_output_stream( device: &cpal::Device, config: &cpal::StreamConfig, @@ -1113,6 +1278,7 @@ where /// Resampler state carried across output cpal callbacks. See /// `build_output_stream` for the rationale. #[cfg(not(target_os = "linux"))] +#[cfg(not(target_os = "ios"))] struct PlaybackResampleState { pos: f64, last_l: f32, @@ -1120,22 +1286,26 @@ struct PlaybackResampleState { } #[cfg(not(target_os = "linux"))] +#[cfg(not(target_os = "ios"))] trait FromF32 { fn from_f32_sample(v: f32) -> Self; } #[cfg(not(target_os = "linux"))] +#[cfg(not(target_os = "ios"))] impl FromF32 for f32 { fn from_f32_sample(v: f32) -> Self { v } } #[cfg(not(target_os = "linux"))] +#[cfg(not(target_os = "ios"))] impl FromF32 for i16 { fn from_f32_sample(v: f32) -> Self { (v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i16 } } #[cfg(not(target_os = "linux"))] +#[cfg(not(target_os = "ios"))] impl FromF32 for u16 { fn from_f32_sample(v: f32) -> Self { let s = (v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i32;