diff --git a/Cargo.lock b/Cargo.lock index 41f0227..90d3158 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -349,6 +349,7 @@ version = "0.0.1-pre" dependencies = [ "audiopus", "chanora_protocol", + "coreaudio-rs 0.14.2", "cpal", "futures-util", "jni 0.21.1", @@ -553,6 +554,20 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "coreaudio-rs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" +dependencies = [ + "bitflags 2.11.1", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + [[package]] name = "cpal" version = "0.16.0" @@ -560,7 +575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbd307f43cc2a697e2d1f8bc7a1d824b5269e052209e28883e5bc04d095aaa3f" dependencies = [ "alsa", - "coreaudio-rs", + "coreaudio-rs 0.13.0", "dasp_sample", "jni 0.21.1", "js-sys", diff --git a/crates/chanora_audio/Cargo.toml b/crates/chanora_audio/Cargo.toml index 06c8520..64eff23 100644 --- a/crates/chanora_audio/Cargo.toml +++ b/crates/chanora_audio/Cargo.toml @@ -37,6 +37,27 @@ reqwest = { version = "0.13", default-features = false, features = ["charset", " tokio = { version = "1", features = ["sync", "rt", "macros", "time"] } +[target.'cfg(target_os = "ios")'.dependencies] +# Direct CoreAudio AudioUnit access on iOS (DEC-011.x follow-up). +# cpal's iOS backend is unsuitable for VoIP: it opens +# kAudioUnitSubType_RemoteIO with a mono-only output element and no +# control over buffer size / sample rate, AND its AudioUnit stays +# bound to the route present at construction time so user-driven +# `overrideOutputAudioPort` flips do not actually move audio to the +# new transducer. Every production iOS VoIP client (Linphone, Mumble +# iOS, Signal, Jitsi, WebRTC reference) instead drives +# `kAudioUnitSubType_VoiceProcessingIO` (a.k.a. VPIO) directly. VPIO +# is Apple's recommended voice unit; it ships hardware AEC + AGC + NS +# and honours route changes natively because it IS the canonical +# voice unit on iOS. `coreaudio-rs` (RustAudio org, same maintainers +# as `cpal`, 8.6M downloads) gives us a safe wrapper around the +# AudioUnit C API. We use it on iOS only; cpal stays on macOS where +# its CoreAudio backend works well against HAL units. +# +# Default features keep `audio_toolbox` + `core_audio`, both required +# for AudioUnit construction + property access. +coreaudio-rs = "0.14" + [target.'cfg(target_os = "android")'.dependencies] # JNI bindings to flip Android's AudioManager into MODE_IN_COMMUNICATION # when the voice-comm preset is requested. ndk_context is initialised diff --git a/crates/chanora_audio/src/ios_voice_unit.rs b/crates/chanora_audio/src/ios_voice_unit.rs new file mode 100644 index 0000000..0a7a867 --- /dev/null +++ b/crates/chanora_audio/src/ios_voice_unit.rs @@ -0,0 +1,276 @@ +//! iOS output + capture stream via the **VoiceProcessingIO** +//! AudioUnit (`kAudioUnitSubType_VoiceProcessingIO`, a.k.a. VPIO). +//! +//! ## Why not cpal on iOS +//! +//! cpal's iOS backend opens `kAudioUnitSubType_RemoteIO` with no +//! control over the stream format, buffer size, or channel count; +//! on iPhone 16 Pro running iOS 18 it reports the output element as +//! **mono 48 kHz** even when the session category is `.playAndRecord` +//! with mode `.default`. More importantly, RemoteIO opened by cpal +//! stays bound to the route that was active at construction time: +//! a later `AVAudioSession.overrideOutputAudioPort(.speaker)` flips +//! the session route metadata (visible in +//! `AVAudioSession.currentRoute`) but the underlying AudioUnit +//! keeps writing to the original transducer. End-user symptom: the +//! Speaker / Receiver toggle in our picker shows the route change +//! in logs but produces no audible difference — the audio is still +//! coming out the earpiece. +//! +//! Every production iOS VoIP client (Mumble iOS, Linphone / +//! mediastreamer2, Signal-iOS, Jitsi, the WebRTC reference impl) +//! avoids RemoteIO and drives VPIO directly instead. VPIO is +//! Apple's recommended voice unit: it ships hardware AEC + AGC + +//! NS, it accepts arbitrary stream-format requests on bus 0 +//! (output) and bus 1 (input), and it re-binds its underlying HAL +//! transducer correctly when the AVAudioSession route changes, +//! because it IS the canonical voice unit on iOS — Apple's own +//! FaceTime audio path runs through it. +//! +//! This file replaces the cpal capture + playback streams on iOS +//! only. macOS continues to use cpal's CoreAudio HAL backend (which +//! works correctly for desktop audio). Linux uses SDL2 (see +//! `sdl_output.rs`). Windows uses cpal's WASAPI backend. +//! +//! ## What VPIO gives us +//! +//! * Pinned **48 kHz Int16 mono** stream format on both bus 0 +//! (output to hardware) and bus 1 (input from hardware). 48 kHz +//! matches the Opus encoder + `tsclientlib::AudioHandler` mix +//! rate exactly, so no resampling is needed inside the audio +//! callback. Int16 is Apple's documented canonical iOS sample +//! format for AudioUnits (see Audio Unit Hosting Guide for iOS +//! §"Canonical formats"). +//! * Hardware **AEC** (acoustic echo cancellation), **AGC** +//! (automatic gain control), and **NS** (noise suppression) ran +//! in the secure-enclave-adjacent voice processor. Free DSP that +//! we'd otherwise need to ship as software (DEC-007/008/009). +//! * **Route-change-correct** physical binding: tapping Speaker or +//! Receiver in our picker now actually moves the audio. +//! +//! ## Threading & lifecycle +//! +//! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the +//! AudioUnit internally holds the C `AudioUnit` opaque pointer and +//! the wrapper's destructor calls `AudioComponentInstanceDispose`, +//! which (per Apple's threading rules) must be called from the +//! thread that owns the unit. We open the unit on the same thread +//! that calls `Self::start` (the tokio worker that runs +//! `chanora_core::ChanoraSession::start_audio`, the same pattern +//! cpal + SDL use) and never move it. The outer `AudioEngine` +//! already carries an `unsafe impl Send` to satisfy the same +//! constraint for cpal's `!Send` Stream type; that impl covers +//! VPIO too. +//! +//! Dropping `IosVoiceUnit` calls `audio_unit.stop()` via the +//! wrapper's `Drop`, which detaches the render + input callbacks +//! and stops the unit. The AudioHandler + CaptureState `Arc`s the +//! callbacks held are then released. +//! +//! ## What this file does NOT do +//! +//! * Route-change observation — that lives in Swift +//! (`AppDelegate.handleRouteChange`) and bounces the unit via a +//! future FRB call. Tracked as Commit 5 of the VPIO rollout. +//! * AVAudioSession category / mode configuration — Swift owns the +//! session (it must be set up before Flutter loads). + +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; + +use coreaudio::audio_unit::render_callback::{self, data}; +use coreaudio::audio_unit::{ + AudioUnit, Element, SampleFormat, Scope, StreamFormat, +}; +use coreaudio::audio_unit::stream_format::LinearPcmFlags; +use coreaudio::audio_unit::IOType; +use tokio::sync::mpsc; +use tracing::{info, warn}; +use tsclientlib::audio::AudioHandler; + +use crate::engine::SessionAudioId; +use crate::AudioError; +use chanora_protocol::voice::OutPacket; + +/// 20 ms at 48 kHz mono — one Opus frame's worth of samples. +/// Aligning the AudioUnit IO buffer to this frame size keeps the +/// jitter-buffer / encoder handshake tight (no fractional-frame +/// reads inside fill_buffer or accumulator drift inside the +/// capture pipeline). +#[allow(dead_code)] // Used in commits 3/4 when callbacks land. +const FRAME_SAMPLES_MONO: u32 = 960; + +/// Sample rate every layer above us assumes. Matches the Opus +/// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the +/// sample rate we ask iOS to give us via VPIO's StreamFormat. +const SAMPLE_RATE_HZ: f64 = 48_000.0; + +/// Output element (bus 0) of an `IOType::VoiceProcessingIO` unit +/// drives the hardware speaker / receiver / AirPods / BT. The +/// stream format we set on `Scope::Input` of this element is the +/// format **we** push samples in; VPIO converts internally to +/// whatever the hardware needs. +const OUTPUT_BUS: Element = Element::Output; + +/// Input element (bus 1) of an `IOType::VoiceProcessingIO` unit +/// pulls from the hardware microphone. The stream format we set on +/// `Scope::Output` of this element is the format **we** receive +/// samples in. +const INPUT_BUS: Element = Element::Input; + +/// Live iOS VPIO AudioUnit wrapper. Construct + start = audio +/// flowing; drop = audio stopped. +pub struct IosVoiceUnit { + // Drop order: stop the unit first (severs callbacks), then + // drop the wrapper so `AudioComponentInstanceDispose` runs. + unit: AudioUnit, +} + +impl IosVoiceUnit { + /// Open a VoiceProcessingIO AudioUnit, pin its stream format + /// to 48 kHz Int16 mono on both buses, install render + input + /// callbacks, and start it. The unit begins pumping audio + /// immediately on return — input callback fires when the mic + /// captures samples, render callback fires when the hardware + /// needs samples to play. + /// + /// Parameters mirror the capture + playback inputs the cpal + /// and SDL backends accept so engine.rs can swap backends with + /// a `cfg`. + /// + /// * `_handler` — shared AudioHandler the inbound forwarder + /// feeds Opus packets into. The output render callback pulls + /// decoded f32 frames from it. + /// * `_output_gain` / `_output_muted` — same atomics the cpal + /// and SDL output paths read on every callback so the master + /// volume + local-mute UI works identically across backends. + /// * `_voice_out_tx` — channel the capture pipeline sends + /// encoded `OutPacket`s on. + /// * `_transmit_active` — PTT gate flag the capture pipeline + /// consults before encoding. + /// * `_frames_sent` — counter the bridge stats surface reads. + /// * `_mic_gain` — pre-encode amplitude scale. + /// + /// In Commit 1 the parameters are accepted but the callbacks + /// emit silence / drop input. Commits 3 + 4 wire them up. + #[allow(clippy::too_many_arguments)] + pub fn start( + _handler: Arc>>, + _output_gain: Arc, + _output_muted: Arc, + _voice_out_tx: mpsc::Sender, + _transmit_active: Arc, + _frames_sent: Arc, + _mic_gain: f32, + ) -> Result { + // Construct the VoiceProcessingIO AudioUnit. cpal exposes + // `Default::default()` which on iOS picks the inferior + // RemoteIO unit; we explicitly pick VPIO. `coreaudio-rs` + // returns an already-initialized unit from `new`, but we + // need to set properties before init so use + // `new_uninitialized` and call `initialize` ourselves + // after the property set is complete. + let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO) + .map_err(|e| AudioError::Backend(format!("vpio audio unit new: {e}")))?; + + // Stream format. Apple's iOS canonical format for + // AudioUnits is Linear PCM, 16-bit signed integer samples + // (see "Canonical formats" in the Audio Unit Hosting Guide + // for iOS). Mono channel matches the Opus encoder + the + // mic input pipe. 48 kHz matches every layer above us. + // + // The StreamFormat is set on: + // * OUTPUT_BUS (bus 0), Scope::Input — the format WE + // push to the unit, i.e. what fill_buffer writes. + // * INPUT_BUS (bus 1), Scope::Output — the format we + // RECEIVE from the unit, i.e. what the mic input + // callback hands us. + // This pairing is documented in Audio Unit Hosting Guide + // for iOS §"Specifying the Audio Stream Format". + let stream_format = StreamFormat { + sample_rate: SAMPLE_RATE_HZ, + sample_format: SampleFormat::I16, + flags: LinearPcmFlags::IS_SIGNED_INTEGER | LinearPcmFlags::IS_PACKED, + channels: 1, + }; + + unit.set_stream_format(stream_format, Scope::Input, OUTPUT_BUS) + .map_err(|e| { + AudioError::StreamConfig(format!( + "vpio set output stream format (bus 0 input scope): {e}" + )) + })?; + unit.set_stream_format(stream_format, Scope::Output, INPUT_BUS) + .map_err(|e| { + AudioError::StreamConfig(format!( + "vpio set input stream format (bus 1 output scope): {e}" + )) + })?; + + // Enable I/O on the input bus (off by default for VPIO). + // The output bus is enabled by default. We can't use cpal's + // set_input_callback abstraction here because the underlying + // property toggle (kAudioOutputUnitProperty_EnableIO with + // value 1 on input scope, element 1) is what coreaudio-rs's + // `set_input_callback` already does internally as the first + // step of installing the callback. Set the callback now + // (Commit 1 = drop input) and the enable bit comes with it. + unit.set_input_callback(|args: render_callback::Args>| { + // Commit 1: drop captured samples. The buffer is filled + // by the framework; we read nothing. Commit 3 wires + // this into CaptureState::ingest_i16. + let _ = args; + Ok(()) + }) + .map_err(|e| AudioError::Backend(format!("vpio set input callback: {e}")))?; + + // Install the render callback that emits playback samples + // to the hardware. The buffer iOS hands us is uninitialised + // — we MUST fill it (writing silence if we have nothing, + // never leaving stale frames). Commit 4 replaces the silence + // loop with an AudioHandler::fill_buffer + downmix path. + unit.set_render_callback(|args: render_callback::Args>| { + for s in args.data.buffer.iter_mut() { + *s = 0; + } + Ok(()) + }) + .map_err(|e| AudioError::Backend(format!("vpio set render callback: {e}")))?; + + // Finalise the unit — allocates internal buffers per the + // stream formats we set above. After initialize() most + // property changes are rejected (you have to uninitialize + + // re-initialize), which is why the property set must come + // first. Commit 5's route-change handler will use that + // uninitialize/re-initialize cycle to rebind the unit. + unit.initialize() + .map_err(|e| AudioError::Backend(format!("vpio initialize: {e}")))?; + + unit.start() + .map_err(|e| AudioError::Backend(format!("vpio start: {e}")))?; + + info!( + target: "chanora_audio", + sample_rate_hz = SAMPLE_RATE_HZ, + channels = stream_format.channels, + sample_format = ?stream_format.sample_format, + "ios VPIO audio unit started" + ); + + Ok(Self { unit }) + } +} + +impl Drop for IosVoiceUnit { + fn drop(&mut self) { + // `AudioUnit::stop` returns Result but we can't surface it + // from Drop. Log on failure so the chanora.log timeline + // matches engine shutdown. The wrapper's own Drop calls + // AudioComponentInstanceDispose after stop returns. + if let Err(e) = self.unit.stop() { + warn!(target: "chanora_audio", error = %e, "vpio stop on drop failed"); + } else { + info!(target: "chanora_audio", "ios VPIO audio unit stopped"); + } + } +} diff --git a/crates/chanora_audio/src/lib.rs b/crates/chanora_audio/src/lib.rs index 8aa4457..a42876e 100644 --- a/crates/chanora_audio/src/lib.rs +++ b/crates/chanora_audio/src/lib.rs @@ -38,6 +38,9 @@ pub mod transmit_selector; #[cfg(target_os = "linux")] mod sdl_output; +#[cfg(target_os = "ios")] +mod ios_voice_unit; + pub use engine::{AudioEngine, AudioEngineConfig}; pub use ptt::{ AudioTransmitGate, MissedKeyUpWatchdog, PttBackendDescriptor, PttCapabilityLevel,