diff --git a/apps/chanora_flutter/pubspec.yaml b/apps/chanora_flutter/pubspec.yaml index 12cc436..b5b7baa 100644 --- a/apps/chanora_flutter/pubspec.yaml +++ b/apps/chanora_flutter/pubspec.yaml @@ -16,7 +16,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html # In Windows, build-name is used as the major, minor, and patch parts # of the product and file versions while build-number is used as the build suffix. -version: 1.0.0-rc.8+62 +version: 1.0.0-rc.8+63 environment: sdk: ^3.11.5 diff --git a/crates/chanora_audio/src/ios_voice_unit.rs b/crates/chanora_audio/src/ios_voice_unit.rs index d5bbf3d..bb0fbe7 100644 --- a/crates/chanora_audio/src/ios_voice_unit.rs +++ b/crates/chanora_audio/src/ios_voice_unit.rs @@ -75,28 +75,40 @@ //! * AVAudioSession category / mode configuration — Swift owns the //! session (it must be set up before Flutter loads). -use std::sync::atomic::{AtomicBool, AtomicU32}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; +use audiopus::coder::Encoder as OpusEncoder; +use audiopus::{ + Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels, + SampleRate as OpusSampleRate, +}; use coreaudio::audio_unit::audio_format::LinearPcmFlags; use coreaudio::audio_unit::render_callback::{self, data}; use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat}; use coreaudio::audio_unit::IOType; use tokio::sync::mpsc; -use tracing::{info, warn}; +use tracing::{debug, error, info, warn}; use tsclientlib::audio::AudioHandler; use crate::engine::SessionAudioId; use crate::AudioError; -use chanora_protocol::OutPacket; +use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket}; /// 20 ms at 48 kHz mono — one Opus frame's worth of samples. /// Aligning the AudioUnit IO buffer to this frame size keeps the /// jitter-buffer / encoder handshake tight (no fractional-frame /// reads inside fill_buffer or accumulator drift inside the /// capture pipeline). -#[allow(dead_code)] // Used in commits 3/4 when callbacks land. -const FRAME_SAMPLES_MONO: u32 = 960; +const FRAME_SAMPLES_MONO: usize = 960; + +/// Maximum size of an encoded Opus frame in bytes (per RFC 6716 +/// §3.2.1). Same constant the cpal-side `CaptureState` uses; we +/// duplicate it here instead of cross-importing from engine.rs +/// because engine.rs's copy is cfg-gated to non-iOS for cpal-only +/// reasons. Post-step-5 review may dedupe by promoting both to a +/// shared `crate::framing` module. +const MAX_OPUS_FRAME: usize = 1275; /// Sample rate every layer above us assumes. Matches the Opus /// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the @@ -116,6 +128,178 @@ const OUTPUT_BUS: Element = Element::Output; /// samples in. const INPUT_BUS: Element = Element::Input; +/// Capture pipeline state owned by the VPIO input callback. The +/// AudioUnit hands us 48 kHz signed-int16 mono PCM directly (no +/// downmix or resample needed — VPIO's hardware-side mix-down +/// from whatever the route's native format is happens before we +/// see the samples). All this struct does is gate on PTT, scale +/// by mic_gain, accumulate to a 20 ms / 960-sample frame, encode +/// to Opus, and try-send the resulting packet on the protocol +/// queue. +/// +/// Mirrors the cpal-side `CaptureState` in engine.rs but is +/// type-specialised to i16 (the cpal version is generic over +/// `T: ToF32` to handle arbitrary HAL formats). Same Opus VoIP +/// tuning (32 kbps, complexity 10, inband FEC, 5% packet-loss +/// budget) lifted verbatim from `try_open_capture` so iOS audio +/// quality matches every other platform. +/// +/// This struct is moved into the VPIO `set_input_callback` closure +/// and is therefore `'static + Send`. The OpusEncoder + Vec + arrays +/// are all owned; the two atomics + sender are `Arc<...>` clones +/// shared with `AudioEngine`. +struct IosCaptureState { + encoder: OpusEncoder, + /// 48 kHz mono PCM scratch accumulating to FRAME_SAMPLES_MONO + /// per encode. Capacity 2x to absorb cpal-style buffer-size + /// jitter without reallocating. + pcm_accum: Vec, + opus_out: [u8; MAX_OPUS_FRAME], + voice_out_tx: mpsc::Sender, + /// PTT transmission gate. Read once per outbound frame; this + /// struct never mutates the flag (SAD-075 / SDD-089). + transmit_active: Arc, + frames_sent: Arc, + mic_gain: f32, +} + +impl IosCaptureState { + /// Build a VoIP-tuned Opus encoder + the capture-state wrapper. + /// Encoder configuration is the same as cpal-side + /// `try_open_capture` (engine.rs) so audio quality is platform- + /// neutral. + fn new( + voice_out_tx: mpsc::Sender, + transmit_active: Arc, + frames_sent: Arc, + mic_gain: f32, + ) -> Result { + let mut encoder = OpusEncoder::new( + OpusSampleRate::Hz48000, + OpusChannels::Mono, + OpusApp::Voip, + ) + .map_err(|e| AudioError::Opus(format!("encoder new (ios): {e}")))?; + + // VoIP-tuned settings — bitrate 32 kbps, complexity 10, + // inband FEC on, packet-loss-perc 5. Soft-fail each setter + // with a warn log to match the cpal-side behaviour: an + // unusual libopus build that rejects one setter shouldn't + // tank the whole pipeline. Full rationale + RFC citations + // are in engine.rs::try_open_capture line ~640. + if let Err(e) = encoder.set_bitrate(OpusBitrate::BitsPerSecond(32_000)) { + warn!(target: "chanora_audio", error = %e, "opus(ios): set_bitrate(32000) failed"); + } + if let Err(e) = encoder.set_complexity(10) { + warn!(target: "chanora_audio", error = %e, "opus(ios): set_complexity(10) failed"); + } + if let Err(e) = encoder.set_inband_fec(true) { + warn!(target: "chanora_audio", error = %e, "opus(ios): set_inband_fec(true) failed"); + } + if let Err(e) = encoder.set_packet_loss_perc(5) { + warn!(target: "chanora_audio", error = %e, "opus(ios): set_packet_loss_perc(5) failed"); + } + info!( + target: "chanora_audio", + bitrate_bps = 32_000, + complexity = 10, + inband_fec = true, + packet_loss_perc = 5, + "ios VPIO opus encoder tuned for VoIP" + ); + + Ok(Self { + encoder, + pcm_accum: Vec::with_capacity(FRAME_SAMPLES_MONO * 2), + opus_out: [0u8; MAX_OPUS_FRAME], + voice_out_tx, + transmit_active, + frames_sent, + mic_gain, + }) + } + + /// Consume the i16 mono buffer delivered by VPIO, accumulate + /// to a 20 ms frame boundary, encode + send when PTT is held. + /// + /// VPIO's input element delivers samples already at the + /// stream format we pinned (48 kHz Int16 mono interleaved). + /// In practice "interleaved mono" is the same byte layout as + /// "planar mono" so we just take the buffer as-is. + fn ingest_i16(&mut self, samples: &[i16]) { + if !self.transmit_active.load(Ordering::Relaxed) { + // Drain accumulator while muted so we don't pop on the + // PTT release edge. Matches cpal-side behaviour. + self.pcm_accum.clear(); + return; + } + + // Mic-gain application. When gain==1.0 we skip the + // multiply + saturate loop entirely — that's the common + // case and the loop is the inner-most hot path of the + // realtime audio thread. + if (self.mic_gain - 1.0).abs() < f32::EPSILON { + self.pcm_accum.extend_from_slice(samples); + } else { + let gain = self.mic_gain; + self.pcm_accum.extend(samples.iter().map(|&s| { + // Saturating mul-then-cast keeps the signal in + // the i16 envelope. Clipping in this branch is + // expected — if the user pushed mic_gain past 1.0 + // and is shouting, the alternative is wrap-around + // distortion which sounds far worse. + let scaled = (s as f32) * gain; + scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16 + })); + } + + // Drain complete 20 ms frames out of the accumulator, encode + // each, send the resulting Opus packet on the protocol + // queue. The `while` covers the case where a single VPIO + // callback delivers more than one frame's worth (rare on + // iOS where the HW IO buffer duration aligns with the + // Opus frame, but always possible during route changes). + while self.pcm_accum.len() >= FRAME_SAMPLES_MONO { + // Use a stack-allocated frame buffer to avoid the + // per-callback allocation a `drain(..N).collect()` + // would incur. The encoder doesn't need ownership. + let mut frame = [0i16; FRAME_SAMPLES_MONO]; + frame.copy_from_slice(&self.pcm_accum[..FRAME_SAMPLES_MONO]); + self.pcm_accum.drain(..FRAME_SAMPLES_MONO); + + match self.encoder.encode(&frame, &mut self.opus_out[..]) { + Ok(len) => { + let packet = OutAudio::new(&AudioData::C2S { + id: 0, + codec: CodecType::OpusVoice, + data: &self.opus_out[..len], + }); + match self.voice_out_tx.try_send(packet) { + Ok(()) => { + self.frames_sent.fetch_add(1, Ordering::Relaxed); + } + Err(mpsc::error::TrySendError::Full(_)) => { + warn!( + target: "chanora_audio", + "ios VPIO: voice_out queue full; dropping frame" + ); + } + Err(mpsc::error::TrySendError::Closed(_)) => { + debug!( + target: "chanora_audio", + "ios VPIO: voice_out closed; capture pipeline stopping" + ); + } + } + } + Err(e) => { + error!(target: "chanora_audio", error = %e, "ios VPIO opus encode failed"); + } + } + } + } +} + /// Live iOS VPIO AudioUnit wrapper. Construct + start = audio /// flowing; drop = audio stopped. pub struct IosVoiceUnit { @@ -142,24 +326,25 @@ impl IosVoiceUnit { /// * `_output_gain` / `_output_muted` — same atomics the cpal /// and SDL output paths read on every callback so the master /// volume + local-mute UI works identically across backends. - /// * `_voice_out_tx` — channel the capture pipeline sends + /// * `voice_out_tx` — channel the capture pipeline sends /// encoded `OutPacket`s on. - /// * `_transmit_active` — PTT gate flag the capture pipeline + /// * `transmit_active` — PTT gate flag the capture pipeline /// consults before encoding. - /// * `_frames_sent` — counter the bridge stats surface reads. - /// * `_mic_gain` — pre-encode amplitude scale. + /// * `frames_sent` — counter the bridge stats surface reads. + /// * `mic_gain` — pre-encode amplitude scale. /// - /// In Commit 1 the parameters are accepted but the callbacks - /// emit silence / drop input. Commits 3 + 4 wire them up. + /// Capture wiring landed in commit 3; playback wiring lands + /// in commit 4 (the render callback still emits silence + /// until then). #[allow(clippy::too_many_arguments)] pub fn start( _handler: Arc>>, _output_gain: Arc, _output_muted: Arc, - _voice_out_tx: mpsc::Sender, - _transmit_active: Arc, - _frames_sent: Arc, - _mic_gain: f32, + voice_out_tx: mpsc::Sender, + transmit_active: Arc, + frames_sent: Arc, + mic_gain: f32, ) -> Result { // Construct the VoiceProcessingIO AudioUnit. cpal exposes // `Default::default()` which on iOS picks the inferior @@ -171,6 +356,37 @@ impl IosVoiceUnit { let mut unit = AudioUnit::new_uninitialized(IOType::VoiceProcessingIO) .map_err(|e| AudioError::Backend(format!("vpio audio unit new: {e}")))?; + // Enable input I/O on element 1. VPIO's input element is + // OFF by default — without this `kAudioOutputUnitProperty_EnableIO` + // toggle no audio flows in and the input callback never + // fires. The constant is 2003 per Apple's headers + // (`AudioUnitProperties.h`). The value is a u32 with + // 1 = enabled. Element::Input on `Scope::Input` is the + // mic side of the unit (element 1 of bus 1; despite the + // confusing nomenclature, `Scope::Input` here means + // "input to the unit" i.e. mic samples coming IN). + // + // Output element 0 is enabled by default for any + // `kAudioUnitType_Output` subtype (which VPIO is), so we + // don't need to toggle anything for playback. + // + // Apple's documented sequence for VPIO setup: + // 1. AudioComponentInstanceNew (= AudioUnit::new_uninitialized) + // 2. EnableIO on element 1 (= this set_property call) + // 3. Set stream format on both elements + // 4. Install callbacks + // 5. AudioUnitInitialize (= unit.initialize) + // 6. AudioOutputUnitStart (= unit.start) + const K_AUDIO_OUTPUT_UNIT_PROPERTY_ENABLE_IO: u32 = 2003; + let enable: u32 = 1; + unit.set_property( + K_AUDIO_OUTPUT_UNIT_PROPERTY_ENABLE_IO, + Scope::Input, + Element::Input, + Some(&enable), + ) + .map_err(|e| AudioError::Backend(format!("vpio enable input I/O: {e}")))?; + // Stream format. Apple's iOS canonical format for // AudioUnits is Linear PCM, 16-bit signed integer samples // (see "Canonical formats" in the Audio Unit Hosting Guide @@ -205,19 +421,26 @@ impl IosVoiceUnit { )) })?; - // Enable I/O on the input bus (off by default for VPIO). - // The output bus is enabled by default. We can't use cpal's - // set_input_callback abstraction here because the underlying - // property toggle (kAudioOutputUnitProperty_EnableIO with - // value 1 on input scope, element 1) is what coreaudio-rs's - // `set_input_callback` already does internally as the first - // step of installing the callback. Set the callback now - // (Commit 1 = drop input) and the enable bit comes with it. - unit.set_input_callback(|args: render_callback::Args>| { - // Commit 1: drop captured samples. The buffer is filled - // by the framework; we read nothing. Commit 3 wires - // this into CaptureState::ingest_i16. - let _ = args; + // Build the capture pipeline and move it into the input + // callback. The Opus encoder + accumulator + opus_out + // scratch are owned by the closure — no Mutex needed + // because the input callback is the sole writer/reader on + // the audio thread. + let mut capture_state = IosCaptureState::new( + voice_out_tx, + transmit_active, + frames_sent, + mic_gain, + )?; + + unit.set_input_callback(move |args: render_callback::Args>| { + // VPIO with our pinned stream format delivers + // interleaved Int16 mono. `args.data.buffer` is a + // `&mut [i16]` of length num_frames * channels = N * 1. + // coreaudio-rs handles the AudioBufferList plumbing + + // the AudioUnitRender call internally before invoking + // this closure. + capture_state.ingest_i16(args.data.buffer); Ok(()) }) .map_err(|e| AudioError::Backend(format!("vpio set input callback: {e}")))?;