feat(audio,ios): add coreaudio-rs dep + IosVoiceUnit skeleton (commit 1/5)
Skeleton scaffolding for the iOS VoiceProcessingIO backend that
will replace cpal on iOS. This commit lands the dependency + the
module + a constructable AudioUnit that emits silence and drops
input; nothing in engine.rs is wired up yet (that is commit 2).
Compilation contract for this commit:
* Linux / Windows / macOS / Android builds unaffected (the new
module is target_os='ios' gated, the new dep is in
'[target."cfg(target_os = \"ios\")"]').
* iOS build pulls in coreaudio-rs 0.14, constructs a VPIO unit,
pins stream format to 48 kHz Int16 mono on both buses, installs
no-op input + silence-emitting render callbacks, initializes,
and starts. No audio is actually moved until commits 3/4.
Why VPIO and not RemoteIO via cpal: cpal's iOS backend opens
RemoteIO with no control over stream format / buffer size /
channels and produces a mono-only output element that stays bound
to the route present at construction time. End-user symptom on
iPhone 16 Pro iOS 18.7.8: tapping Speaker in the picker flips
AVAudioSession.currentRoute.outputs to Speaker (confirmed in our
diagnostic logs from commit da631a2) but audio keeps coming out
the receiver because the AudioUnit's output binding is stale.
Every production iOS VoIP client (Mumble iOS, Linphone /
mediastreamer2, Signal-iOS, Jitsi Meet iOS, the WebRTC reference
impl) avoids RemoteIO and uses VoiceProcessingIO instead. VPIO is
Apple's recommended voice unit; it ships hardware AEC + AGC + NS
and re-binds the physical transducer correctly on route changes
because it IS the canonical voice unit on iOS — FaceTime's audio
path runs through it.
coreaudio-rs 0.14 (RustAudio org, 8.6M downloads, same maintainers
as cpal) gives us a safe wrapper around the AudioUnit C API on
iOS. Uses objc2-* crates underneath so links cleanly into iOS
builds. ios_voice_unit.rs sits next to sdl_output.rs as the iOS
sibling of the Linux SDL2 output path.
Build verify (Linux host): `cargo check -p chanora_audio`
finished clean in 16.09s. iOS build verification happens in
commit 2 when the module is exercised; for this commit the module
compiles in isolation but is dead code on iOS too (no caller).
This commit is contained in:
EdisonJwa
Generated
+16
-1
@@ -349,6 +349,7 @@ version = "0.0.1-pre"
|
||||
dependencies = [
|
||||
"audiopus",
|
||||
"chanora_protocol",
|
||||
"coreaudio-rs 0.14.2",
|
||||
"cpal",
|
||||
"futures-util",
|
||||
"jni 0.21.1",
|
||||
@@ -553,6 +554,20 @@ dependencies = [
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "coreaudio-rs"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"libc",
|
||||
"objc2-audio-toolbox",
|
||||
"objc2-core-audio",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpal"
|
||||
version = "0.16.0"
|
||||
@@ -560,7 +575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbd307f43cc2a697e2d1f8bc7a1d824b5269e052209e28883e5bc04d095aaa3f"
|
||||
dependencies = [
|
||||
"alsa",
|
||||
"coreaudio-rs",
|
||||
"coreaudio-rs 0.13.0",
|
||||
"dasp_sample",
|
||||
"jni 0.21.1",
|
||||
"js-sys",
|
||||
|
||||
@@ -37,6 +37,27 @@ reqwest = { version = "0.13", default-features = false, features = ["charset", "
|
||||
|
||||
tokio = { version = "1", features = ["sync", "rt", "macros", "time"] }
|
||||
|
||||
[target.'cfg(target_os = "ios")'.dependencies]
|
||||
# Direct CoreAudio AudioUnit access on iOS (DEC-011.x follow-up).
|
||||
# cpal's iOS backend is unsuitable for VoIP: it opens
|
||||
# kAudioUnitSubType_RemoteIO with a mono-only output element and no
|
||||
# control over buffer size / sample rate, AND its AudioUnit stays
|
||||
# bound to the route present at construction time so user-driven
|
||||
# `overrideOutputAudioPort` flips do not actually move audio to the
|
||||
# new transducer. Every production iOS VoIP client (Linphone, Mumble
|
||||
# iOS, Signal, Jitsi, WebRTC reference) instead drives
|
||||
# `kAudioUnitSubType_VoiceProcessingIO` (a.k.a. VPIO) directly. VPIO
|
||||
# is Apple's recommended voice unit; it ships hardware AEC + AGC + NS
|
||||
# and honours route changes natively because it IS the canonical
|
||||
# voice unit on iOS. `coreaudio-rs` (RustAudio org, same maintainers
|
||||
# as `cpal`, 8.6M downloads) gives us a safe wrapper around the
|
||||
# AudioUnit C API. We use it on iOS only; cpal stays on macOS where
|
||||
# its CoreAudio backend works well against HAL units.
|
||||
#
|
||||
# Default features keep `audio_toolbox` + `core_audio`, both required
|
||||
# for AudioUnit construction + property access.
|
||||
coreaudio-rs = "0.14"
|
||||
|
||||
[target.'cfg(target_os = "android")'.dependencies]
|
||||
# JNI bindings to flip Android's AudioManager into MODE_IN_COMMUNICATION
|
||||
# when the voice-comm preset is requested. ndk_context is initialised
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
//! iOS output + capture stream via the **VoiceProcessingIO**
|
||||
//! AudioUnit (`kAudioUnitSubType_VoiceProcessingIO`, a.k.a. VPIO).
|
||||
//!
|
||||
//! ## Why not cpal on iOS
|
||||
//!
|
||||
//! cpal's iOS backend opens `kAudioUnitSubType_RemoteIO` with no
|
||||
//! control over the stream format, buffer size, or channel count;
|
||||
//! on iPhone 16 Pro running iOS 18 it reports the output element as
|
||||
//! **mono 48 kHz** even when the session category is `.playAndRecord`
|
||||
//! with mode `.default`. More importantly, RemoteIO opened by cpal
|
||||
//! stays bound to the route that was active at construction time:
|
||||
//! a later `AVAudioSession.overrideOutputAudioPort(.speaker)` flips
|
||||
//! the session route metadata (visible in
|
||||
//! `AVAudioSession.currentRoute`) but the underlying AudioUnit
|
||||
//! keeps writing to the original transducer. End-user symptom: the
|
||||
//! Speaker / Receiver toggle in our picker shows the route change
|
||||
//! in logs but produces no audible difference — the audio is still
|
||||
//! coming out the earpiece.
|
||||
//!
|
||||
//! Every production iOS VoIP client (Mumble iOS, Linphone /
|
||||
//! mediastreamer2, Signal-iOS, Jitsi, the WebRTC reference impl)
|
||||
//! avoids RemoteIO and drives VPIO directly instead. VPIO is
|
||||
//! Apple's recommended voice unit: it ships hardware AEC + AGC +
|
||||
//! NS, it accepts arbitrary stream-format requests on bus 0
|
||||
//! (output) and bus 1 (input), and it re-binds its underlying HAL
|
||||
//! transducer correctly when the AVAudioSession route changes,
|
||||
//! because it IS the canonical voice unit on iOS — Apple's own
|
||||
//! FaceTime audio path runs through it.
|
||||
//!
|
||||
//! This file replaces the cpal capture + playback streams on iOS
|
||||
//! only. macOS continues to use cpal's CoreAudio HAL backend (which
|
||||
//! works correctly for desktop audio). Linux uses SDL2 (see
|
||||
//! `sdl_output.rs`). Windows uses cpal's WASAPI backend.
|
||||
//!
|
||||
//! ## What VPIO gives us
|
||||
//!
|
||||
//! * Pinned **48 kHz Int16 mono** stream format on both bus 0
|
||||
//! (output to hardware) and bus 1 (input from hardware). 48 kHz
|
||||
//! matches the Opus encoder + `tsclientlib::AudioHandler` mix
|
||||
//! rate exactly, so no resampling is needed inside the audio
|
||||
//! callback. Int16 is Apple's documented canonical iOS sample
|
||||
//! format for AudioUnits (see Audio Unit Hosting Guide for iOS
|
||||
//! §"Canonical formats").
|
||||
//! * Hardware **AEC** (acoustic echo cancellation), **AGC**
|
||||
//! (automatic gain control), and **NS** (noise suppression) ran
|
||||
//! in the secure-enclave-adjacent voice processor. Free DSP that
|
||||
//! we'd otherwise need to ship as software (DEC-007/008/009).
|
||||
//! * **Route-change-correct** physical binding: tapping Speaker or
|
||||
//! Receiver in our picker now actually moves the audio.
|
||||
//!
|
||||
//! ## Threading & lifecycle
|
||||
//!
|
||||
//! `coreaudio::audio_unit::AudioUnit` is `Send` but `!Sync` — the
|
||||
//! AudioUnit internally holds the C `AudioUnit` opaque pointer and
|
||||
//! the wrapper's destructor calls `AudioComponentInstanceDispose`,
|
||||
//! which (per Apple's threading rules) must be called from the
|
||||
//! thread that owns the unit. We open the unit on the same thread
|
||||
//! that calls `Self::start` (the tokio worker that runs
|
||||
//! `chanora_core::ChanoraSession::start_audio`, the same pattern
|
||||
//! cpal + SDL use) and never move it. The outer `AudioEngine`
|
||||
//! already carries an `unsafe impl Send` to satisfy the same
|
||||
//! constraint for cpal's `!Send` Stream type; that impl covers
|
||||
//! VPIO too.
|
||||
//!
|
||||
//! Dropping `IosVoiceUnit` calls `audio_unit.stop()` via the
|
||||
//! wrapper's `Drop`, which detaches the render + input callbacks
|
||||
//! and stops the unit. The AudioHandler + CaptureState `Arc`s the
|
||||
//! callbacks held are then released.
|
||||
//!
|
||||
//! ## What this file does NOT do
|
||||
//!
|
||||
//! * Route-change observation — that lives in Swift
|
||||
//! (`AppDelegate.handleRouteChange`) and bounces the unit via a
|
||||
//! future FRB call. Tracked as Commit 5 of the VPIO rollout.
|
||||
//! * AVAudioSession category / mode configuration — Swift owns the
|
||||
//! session (it must be set up before Flutter loads).
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use coreaudio::audio_unit::render_callback::{self, data};
|
||||
use coreaudio::audio_unit::{
|
||||
AudioUnit, Element, SampleFormat, Scope, StreamFormat,
|
||||
};
|
||||
use coreaudio::audio_unit::stream_format::LinearPcmFlags;
|
||||
use coreaudio::audio_unit::IOType;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{info, warn};
|
||||
use tsclientlib::audio::AudioHandler;
|
||||
|
||||
use crate::engine::SessionAudioId;
|
||||
use crate::AudioError;
|
||||
use chanora_protocol::voice::OutPacket;
|
||||
|
||||
/// 20 ms at 48 kHz mono — one Opus frame's worth of samples.
|
||||
/// Aligning the AudioUnit IO buffer to this frame size keeps the
|
||||
/// jitter-buffer / encoder handshake tight (no fractional-frame
|
||||
/// reads inside fill_buffer or accumulator drift inside the
|
||||
/// capture pipeline).
|
||||
#[allow(dead_code)] // Used in commits 3/4 when callbacks land.
|
||||
const FRAME_SAMPLES_MONO: u32 = 960;
|
||||
|
||||
/// Sample rate every layer above us assumes. Matches the Opus
|
||||
/// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the
|
||||
/// sample rate we ask iOS to give us via VPIO's StreamFormat.
|
||||
const SAMPLE_RATE_HZ: f64 = 48_000.0;
|
||||
|
||||
/// Output element (bus 0) of an `IOType::VoiceProcessingIO` unit
|
||||
/// drives the hardware speaker / receiver / AirPods / BT. The
|
||||
/// stream format we set on `Scope::Input` of this element is the
|
||||
/// format **we** push samples in; VPIO converts internally to
|
||||
/// whatever the hardware needs.
|
||||
const OUTPUT_BUS: Element = Element::Output;
|
||||
|
||||
/// Input element (bus 1) of an `IOType::VoiceProcessingIO` unit
|
||||
/// pulls from the hardware microphone. The stream format we set on
|
||||
/// `Scope::Output` of this element is the format **we** receive
|
||||
/// samples in.
|
||||
const INPUT_BUS: Element = Element::Input;
|
||||
|
||||
/// Live iOS VPIO AudioUnit wrapper. Construct + start = audio
|
||||
/// flowing; drop = audio stopped.
|
||||
pub struct IosVoiceUnit {
|
||||
// Drop order: stop the unit first (severs callbacks), then
|
||||
// drop the wrapper so `AudioComponentInstanceDispose` runs.
|
||||
unit: AudioUnit,
|
||||
}
|
||||
|
||||
impl IosVoiceUnit {
|
||||
/// Open a VoiceProcessingIO AudioUnit, pin its stream format
|
||||
/// to 48 kHz Int16 mono on both buses, install render + input
|
||||
/// callbacks, and start it. The unit begins pumping audio
|
||||
/// immediately on return — input callback fires when the mic
|
||||
/// captures samples, render callback fires when the hardware
|
||||
/// needs samples to play.
|
||||
///
|
||||
/// Parameters mirror the capture + playback inputs the cpal
|
||||
/// and SDL backends accept so engine.rs can swap backends with
|
||||
/// a `cfg`.
|
||||
///
|
||||
/// * `_handler` — shared AudioHandler the inbound forwarder
|
||||
/// feeds Opus packets into. The output render callback pulls
|
||||
/// decoded f32 frames from it.
|
||||
/// * `_output_gain` / `_output_muted` — same atomics the cpal
|
||||
/// and SDL output paths read on every callback so the master
|
||||
/// volume + local-mute UI works identically across backends.
|
||||
/// * `_voice_out_tx` — channel the capture pipeline sends
|
||||
/// encoded `OutPacket`s on.
|
||||
/// * `_transmit_active` — PTT gate flag the capture pipeline
|
||||
/// consults before encoding.
|
||||
/// * `_frames_sent` — counter the bridge stats surface reads.
|
||||
/// * `_mic_gain` — pre-encode amplitude scale.
|
||||
///
|
||||
/// In Commit 1 the parameters are accepted but the callbacks
|
||||
/// emit silence / drop input. Commits 3 + 4 wire them up.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn start(
|
||||
_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||||
_output_gain: Arc<AtomicU32>,
|
||||
_output_muted: Arc<AtomicBool>,
|
||||
_voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
_transmit_active: Arc<AtomicBool>,
|
||||
_frames_sent: Arc<AtomicU32>,
|
||||
_mic_gain: f32,
|
||||
) -> Result<Self, AudioError> {
|
||||
// Construct the VoiceProcessingIO AudioUnit. cpal exposes
|
||||
// `Default::default()` which on iOS picks the inferior
|
||||
// RemoteIO unit; we explicitly pick VPIO. `coreaudio-rs`
|
||||
// returns an already-initialized unit from `new`, but we
|
||||
// need to set properties before init so use
|
||||
// `new_uninitialized` and call `initialize` ourselves
|
||||
// after the property set is complete.
|
||||
let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO)
|
||||
.map_err(|e| AudioError::Backend(format!("vpio audio unit new: {e}")))?;
|
||||
|
||||
// Stream format. Apple's iOS canonical format for
|
||||
// AudioUnits is Linear PCM, 16-bit signed integer samples
|
||||
// (see "Canonical formats" in the Audio Unit Hosting Guide
|
||||
// for iOS). Mono channel matches the Opus encoder + the
|
||||
// mic input pipe. 48 kHz matches every layer above us.
|
||||
//
|
||||
// The StreamFormat is set on:
|
||||
// * OUTPUT_BUS (bus 0), Scope::Input — the format WE
|
||||
// push to the unit, i.e. what fill_buffer writes.
|
||||
// * INPUT_BUS (bus 1), Scope::Output — the format we
|
||||
// RECEIVE from the unit, i.e. what the mic input
|
||||
// callback hands us.
|
||||
// This pairing is documented in Audio Unit Hosting Guide
|
||||
// for iOS §"Specifying the Audio Stream Format".
|
||||
let stream_format = StreamFormat {
|
||||
sample_rate: SAMPLE_RATE_HZ,
|
||||
sample_format: SampleFormat::I16,
|
||||
flags: LinearPcmFlags::IS_SIGNED_INTEGER | LinearPcmFlags::IS_PACKED,
|
||||
channels: 1,
|
||||
};
|
||||
|
||||
unit.set_stream_format(stream_format, Scope::Input, OUTPUT_BUS)
|
||||
.map_err(|e| {
|
||||
AudioError::StreamConfig(format!(
|
||||
"vpio set output stream format (bus 0 input scope): {e}"
|
||||
))
|
||||
})?;
|
||||
unit.set_stream_format(stream_format, Scope::Output, INPUT_BUS)
|
||||
.map_err(|e| {
|
||||
AudioError::StreamConfig(format!(
|
||||
"vpio set input stream format (bus 1 output scope): {e}"
|
||||
))
|
||||
})?;
|
||||
|
||||
// Enable I/O on the input bus (off by default for VPIO).
|
||||
// The output bus is enabled by default. We can't use cpal's
|
||||
// set_input_callback abstraction here because the underlying
|
||||
// property toggle (kAudioOutputUnitProperty_EnableIO with
|
||||
// value 1 on input scope, element 1) is what coreaudio-rs's
|
||||
// `set_input_callback` already does internally as the first
|
||||
// step of installing the callback. Set the callback now
|
||||
// (Commit 1 = drop input) and the enable bit comes with it.
|
||||
unit.set_input_callback(|args: render_callback::Args<data::Interleaved<i16>>| {
|
||||
// Commit 1: drop captured samples. The buffer is filled
|
||||
// by the framework; we read nothing. Commit 3 wires
|
||||
// this into CaptureState::ingest_i16.
|
||||
let _ = args;
|
||||
Ok(())
|
||||
})
|
||||
.map_err(|e| AudioError::Backend(format!("vpio set input callback: {e}")))?;
|
||||
|
||||
// Install the render callback that emits playback samples
|
||||
// to the hardware. The buffer iOS hands us is uninitialised
|
||||
// — we MUST fill it (writing silence if we have nothing,
|
||||
// never leaving stale frames). Commit 4 replaces the silence
|
||||
// loop with an AudioHandler::fill_buffer + downmix path.
|
||||
unit.set_render_callback(|args: render_callback::Args<data::Interleaved<i16>>| {
|
||||
for s in args.data.buffer.iter_mut() {
|
||||
*s = 0;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.map_err(|e| AudioError::Backend(format!("vpio set render callback: {e}")))?;
|
||||
|
||||
// Finalise the unit — allocates internal buffers per the
|
||||
// stream formats we set above. After initialize() most
|
||||
// property changes are rejected (you have to uninitialize +
|
||||
// re-initialize), which is why the property set must come
|
||||
// first. Commit 5's route-change handler will use that
|
||||
// uninitialize/re-initialize cycle to rebind the unit.
|
||||
unit.initialize()
|
||||
.map_err(|e| AudioError::Backend(format!("vpio initialize: {e}")))?;
|
||||
|
||||
unit.start()
|
||||
.map_err(|e| AudioError::Backend(format!("vpio start: {e}")))?;
|
||||
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
sample_rate_hz = SAMPLE_RATE_HZ,
|
||||
channels = stream_format.channels,
|
||||
sample_format = ?stream_format.sample_format,
|
||||
"ios VPIO audio unit started"
|
||||
);
|
||||
|
||||
Ok(Self { unit })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for IosVoiceUnit {
|
||||
fn drop(&mut self) {
|
||||
// `AudioUnit::stop` returns Result but we can't surface it
|
||||
// from Drop. Log on failure so the chanora.log timeline
|
||||
// matches engine shutdown. The wrapper's own Drop calls
|
||||
// AudioComponentInstanceDispose after stop returns.
|
||||
if let Err(e) = self.unit.stop() {
|
||||
warn!(target: "chanora_audio", error = %e, "vpio stop on drop failed");
|
||||
} else {
|
||||
info!(target: "chanora_audio", "ios VPIO audio unit stopped");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,9 @@ pub mod transmit_selector;
|
||||
#[cfg(target_os = "linux")]
|
||||
mod sdl_output;
|
||||
|
||||
#[cfg(target_os = "ios")]
|
||||
mod ios_voice_unit;
|
||||
|
||||
pub use engine::{AudioEngine, AudioEngineConfig};
|
||||
pub use ptt::{
|
||||
AudioTransmitGate, MissedKeyUpWatchdog, PttBackendDescriptor, PttCapabilityLevel,
|
||||
|
||||
Reference in New Issue
Block a user