diff --git a/crates/chanora_audio/src/ios_voice_unit.rs b/crates/chanora_audio/src/ios_voice_unit.rs index f581c31..e9e0554 100644 --- a/crates/chanora_audio/src/ios_voice_unit.rs +++ b/crates/chanora_audio/src/ios_voice_unit.rs @@ -74,11 +74,10 @@ use coreaudio::audio_unit::audio_format::LinearPcmFlags; use coreaudio::audio_unit::render_callback::{self, data}; use coreaudio::audio_unit::IOType; use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat}; +use crossbeam::queue::ArrayQueue; use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; -#[cfg(target_os = "macos")] -use crate::audio_event_queue::{AudioCommand, AudioEventQueue}; use crate::mobile_voice_backend::VoiceAudioParams; use crate::AudioError; use chanora_protocol::OutPacket; @@ -653,6 +652,46 @@ impl IosVoiceUnit { ) .map_err(|e| AudioError::Backend(format!("vpio enable input I/O: {e}")))?; + // VPIO defaults to maximum "duck others" — when our voice plays, + // every other app's audio (Music, Safari, Discord, ...) is heavily + // attenuated. That is correct for a phone call but wrong for a + // chat client running alongside music or game audio. There is no + // public "off" switch; the lowest publicly-exposed level is Min + // and disabling Advanced Ducking turns off the voice-activity- + // driven dynamic ducking. This matches what Hume, Moonshine, and + // similar open-source VoIP/voice apps configure. + // Apple ref: kAUVoiceIOProperty_OtherAudioDuckingConfiguration + // Property ID 2108, Global scope, Output element (= 0). + // Struct layout matches AUVoiceIOOtherAudioDuckingConfiguration + // from : + // Boolean (u8) mEnableAdvancedDucking + AUVoiceIOOtherAudio- + // DuckingLevel (u32) mDuckingLevel, #[repr(C)] yields 8 bytes + // with the 3-byte natural alignment pad before the u32. + #[repr(C)] + struct AuVoiceIoOtherAudioDuckingConfiguration { + m_enable_advanced_ducking: u8, + m_ducking_level: u32, + } + const K_AU_VOICE_IO_PROPERTY_OTHER_AUDIO_DUCKING_CONFIGURATION: u32 = 2108; + const K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_LEVEL_MIN: u32 = 10; + let ducking_config = AuVoiceIoOtherAudioDuckingConfiguration { + m_enable_advanced_ducking: 0, + m_ducking_level: K_AU_VOICE_IO_OTHER_AUDIO_DUCKING_LEVEL_MIN, + }; + // Apple introduced this property in iOS 17 / macOS 14. On older + // OS versions VPIO returns kAudioUnitErr_InvalidProperty (-10879) + // — log it but never fail VPIO startup over a ducking knob. + if let Err(e) = unit.set_property( + K_AU_VOICE_IO_PROPERTY_OTHER_AUDIO_DUCKING_CONFIGURATION, + Scope::Global, + Element::Output, + Some(&ducking_config), + ) { + tracing::debug!( + "vpio set OtherAudioDuckingConfiguration failed (older OS?): {e}" + ); + } + // Note: we keep VPIO's voice processing chain ENABLED // (AEC + AGC + NS on the mic path) because it gives us // clean capture for free. The historical playback @@ -779,41 +818,165 @@ impl IosVoiceUnit { // callback's actual output channels. Same as Linux/SDL, // just stereo-f32 -> interleaved-i16 converted at the // boundary. - // Preallocate to Apple's VPIO MaximumFramesPerSlice default - // (4096 frames) * 2 (stereo) = 8192 f32. The render callback - // is realtime; growing this Vec inside the callback would - // allocate on the audio thread and risk an underrun. We - // never resize below `needed` after this point — the `len() - // < needed` branch in the callback is a defensive no-op for - // the impossible case where the audio unit later raises - // MaximumFramesPerSlice beyond our preallocation. - let mut scratch_stereo: Vec = vec![0.0; 4096 * 2]; + // macOS cadence-fix (commit-f): + // + // The render callback is the wrong place to call AudioHandler::fill_buffer. + // macOS VPIO invokes us with `num_frames=512` (= 10.67 ms @ 48 kHz) which + // is NOT a multiple of the 20 ms Opus frame size that AudioHandler expects + // inside fill_buffer. The mismatch (a) leaves fill_buffer unable to + // satisfy the request on most calls and (b) advances MAX_PACKET_LOSSES, + // which removes the talker and starts PLC silence/decay. + // + // Converged fix across Mumble (Speex jitter buffer + decoded PCM FIFO), + // WebRTC NetEQ (adaptive 80ms prebuffer + sync buffer), Songbird (Fill→ + // Drain playout buffer), cpal/rodio (SPSC ring + zero-fill on underrun), + // tsclientlib's own SDL example, and the upstream tsclientlib::audio + // contract: separate ingress quantum (20 ms decoded PCM = 1920 stereo + // f32) from egress quantum (whatever VPIO asks for), connected by a + // lock-free ring of decoded PCM. + // + // Layout: + // * macOS: spawn a tokio producer task paced at 20 ms; each tick + // try-locks AudioHandler, calls fill_buffer(1920), pushes 1920 + // f32 into the ring. The render callback only pops — no lock, + // no Opus decode, no allocator on the audio thread. 60 ms + // prebuffer (3 × 20 ms) is held before the callback starts + // draining, matching Mumble's playout margin and WebRTC's + // kStartDelayMs order of magnitude. Crossbeam ArrayQueue + // is used because we already depend on crossbeam-queue. + // * iOS: keep the existing direct fill_buffer path — VPIO on iOS + // requests 480-frame slices that ARE 20 ms aligned so the + // cadence mismatch does not arise there. + #[cfg(target_os = "macos")] + { + // 100 ms capacity = 9600 stereo f32. Sized so that the 60 ms + // prebuffer plus a few jitter spikes fit without forcing the + // producer to drop frames. crossbeam ArrayQueue is fixed-cap + // and lock-free SPSC-ish (MPMC but wait-free per end); for + // single producer + single consumer it's effectively SPSC. + const RING_CAPACITY: usize = 12000; + const PULL_SAMPLES: usize = 1920; // one 20 ms Opus frame, stereo + const PREBUFFER_SAMPLES: usize = 9600; // 100 ms @ 48 kHz stereo + + let pcm_ring: Arc> = Arc::new(ArrayQueue::new(RING_CAPACITY)); + let pcm_ring_producer = pcm_ring.clone(); + let pcm_ring_consumer = pcm_ring.clone(); + let handler_for_producer = params.handler.clone(); + let output_gain_for_render = params.output_gain.clone(); + let output_muted_for_render = params.output_muted.clone(); + + tokio::spawn(async move { + let mut pull_scratch: Vec = vec![0.0; PULL_SAMPLES]; + let mut interval = + tokio::time::interval(std::time::Duration::from_millis(20)); + interval.set_missed_tick_behavior( + tokio::time::MissedTickBehavior::Delay, + ); + loop { + interval.tick().await; + match handler_for_producer.try_lock() { + Ok(mut h) => { + pull_scratch.fill(0.0); + let _ = h.fill_buffer(&mut pull_scratch[..]); + for &s in &pull_scratch { + pcm_ring_producer.force_push(s); + } + } + Err(std::sync::TryLockError::WouldBlock) => {} + Err(std::sync::TryLockError::Poisoned(e)) => { + warn!(target: "chanora_audio", + "producer: AudioHandler mutex poisoned: {e}"); + break; + } + } + } + }); + + unit.set_render_callback(move |args: render_callback::Args>| { + let render_callback::Args { + data, + num_frames, + .. + } = args; + let out: &mut [i16] = data.buffer; + let out_channels = data.channels; + let needed = num_frames * out_channels; + + if pcm_ring_consumer.len() < PREBUFFER_SAMPLES { + for sample in &mut out[..needed] { *sample = 0; } + return Ok(()); + } + + // Pop L,R as a pair per frame. The pairing is + // load-bearing — `written += 2` half-buffer bug. + let mut written_frames: usize = 0; + while written_frames < num_frames { + let l = match pcm_ring_consumer.pop() { + Some(v) => v, + None => break, + }; + let r = pcm_ring_consumer.pop().unwrap_or(0.0); + let mut frame = [l, r]; + crate::voice_render::limit_peak_inplace(&mut frame, 0.99); + let (l_lim, r_lim) = (frame[0], frame[1]); + let base = written_frames * out_channels; + if out_channels == 1 { + let mono = (l_lim + r_lim) * 0.5; + out[base] = (mono.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; + } else { + out[base] = + (l_lim.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; + out[base + 1] = + (r_lim.clamp(-1.0, 1.0) * i16::MAX as f32) as i16; + } + written_frames += 1; + } + + if written_frames < num_frames { + let remaining = num_frames - written_frames; + for f in 0..remaining { + let base = (written_frames + f) * out_channels; + for c in 0..out_channels { out[base + c] = 0; } + } + } + + let gain = f32::from_bits( + output_gain_for_render.load(Ordering::Relaxed), + ); + let muted = + output_muted_for_render.load(Ordering::Relaxed); + if muted { + for sample in &mut out[..needed] { *sample = 0; } + } else if gain != 1.0 { + for sample in &mut out[..needed] { + *sample = (((*sample as f32) * gain) + .clamp(i16::MIN as f32, i16::MAX as f32)) + as i16; + } + } + + Ok(()) + }) + .map_err(|e| AudioError::Backend(format!( + "audio unit set render callback: {e}" + )))?; + } + + // iOS path: direct fill_buffer in callback. iOS VPIO + // requests 480-frame slices that align with tsclientlib's + // 20 ms Opus frame, so the cadence mismatch that macOS + // hits does not arise here. macOS has its own cfg-gated + // producer-task path above. #[cfg(target_os = "ios")] + { + let mut scratch_stereo: Vec = vec![0.0; 4096 * 2]; let handler_for_render = params.handler.clone(); - #[cfg(target_os = "macos")] - let mut handler_for_render = params.handler; - #[cfg(target_os = "macos")] - let event_consumer = AudioEventQueue::consumer(¶ms.event_producer.queue()); let output_gain_for_render = params.output_gain.clone(); let output_muted_for_render = params.output_muted.clone(); let audio_processing_stats_for_render = params.audio_processing_stats.clone(); - let wav_recorder_for_render = wav_recorder.clone(); // Level meter decimation: the render callback fires ~93 - // times/sec, but the bridge consumer (`output_level_stream`) - // reads at ~30 Hz. Computing `sqrt()` + `log10()` every - // callback wastes real-time budget and causes buffer underruns - // on macOS CoreAudio (same regression as the capture side, see - // CaptureState::level_decimation_counter in engine.rs). + // times/sec, but the bridge consumer reads at ~30 Hz. let mut render_level_decimation: u32 = 0; - // Diagnostic counters (sampled every 100 callbacks ~= 2 s). - let mut cb_count: u64 = 0; - let mut last_num_frames: usize = 0; - let mut num_frames_changes: u32 = 0; - let mut callbacks_with_audio: u64 = 0; - let mut callbacks_with_silence: u64 = 0; - let mut render_ref_accum = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; - let mut render_ref_len: usize = 0; - let mut render_recorder_active = false; unit.set_render_callback(move |args: render_callback::Args>| { let render_callback::Args { data, @@ -832,31 +995,9 @@ impl IosVoiceUnit { // earlier callbacks (when scratch was bigger) would // leak through otherwise. scratch_stereo[..needed].fill(0.0); - #[cfg(target_os = "macos")] - { - for cmd in event_consumer.drain_controls() { - match cmd { - AudioCommand::SetVolume(id, vol) => { - if let Some(q) = handler_for_render.get_mut_queues().get_mut(&id) { - q.volume = vol; - } - } - AudioCommand::RemoveClient(id) => { - handler_for_render.get_mut_queues().remove(&id); - } - } - } - for pkt in event_consumer.drain_packets(50) { - if let Err(e) = handler_for_render.handle_packet(pkt.client_id, pkt.data) { - debug!(target: "chanora_audio", error = %e, "decode failed"); - } - } - let _removed = handler_for_render.fill_buffer(&mut scratch_stereo[..needed]); - } - #[cfg(target_os = "ios")] match handler_for_render.try_lock() { Ok(mut h) => { - let _removed = h.fill_buffer(&mut scratch_stereo[..needed]); + let _ = h.fill_buffer(&mut scratch_stereo[..needed]); } Err(std::sync::TryLockError::WouldBlock) => { audio_processing_stats_for_render.increment_callback_xrun(); @@ -867,7 +1008,9 @@ impl IosVoiceUnit { warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {e}"); } } - + // Peak limiter — multi-client mixes can sum past 0 dBFS; + // without this the downmix helper would hard-clip to i16::MAX. + crate::voice_render::limit_peak_inplace(&mut scratch_stereo[..needed], 0.99); let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed)); let muted = output_muted_for_render.load(Ordering::Relaxed); let mix_stats = crate::voice_render::downmix_stereo_f32_to_interleaved_i16( @@ -944,6 +1087,7 @@ impl IosVoiceUnit { Ok(()) }) .map_err(|e| AudioError::Backend(format!("audio unit set render callback: {e}")))?; + } // end #[cfg(target_os = "ios")] block // Finalise the unit — allocates internal buffers per the // stream formats we set above. After initialize() most diff --git a/crates/chanora_audio/src/voice_render.rs b/crates/chanora_audio/src/voice_render.rs index 6fe02fa..72039b1 100644 --- a/crates/chanora_audio/src/voice_render.rs +++ b/crates/chanora_audio/src/voice_render.rs @@ -110,6 +110,32 @@ pub(crate) fn downmix_stereo_f32_to_mono_f32(stereo: &[f32], out: &mut [f32]) { } } +/// In-place per-frame peak limiter. Scales the entire buffer so the +/// absolute peak equals `threshold`; returns the applied gain (1.0 = +/// no reduction). Used on the render path between `AudioHandler` and +/// the i16 downmix to prevent hard clipping when a multi-client mix +/// exceeds 0 dBFS. Per-frame scaling is sub-millisecond at 48 kHz, so +/// the pumping risk is negligible for speech; a look-ahead design was +/// rejected because it would add latency on top of the existing +/// jitter buffer. +pub(crate) fn limit_peak_inplace(samples: &mut [f32], threshold: f32) -> f32 { + if threshold <= 0.0 || !threshold.is_finite() { + return 1.0; + } + let peak = samples + .iter() + .map(|s| s.abs()) + .fold(0.0_f32, f32::max); + if peak <= threshold { + return 1.0; + } + let gain = threshold / peak; + for s in samples.iter_mut() { + *s *= gain; + } + gain +} + #[cfg(test)] mod tests { use super::*; @@ -171,4 +197,45 @@ mod tests { assert_eq!(out, [0.0, 0.0]); } + + #[test] + fn limit_peak_is_noop_below_threshold() { + let mut samples = [0.1_f32, -0.2, 0.3, -0.4]; + let gain = limit_peak_inplace(&mut samples, 0.95); + assert_eq!(gain, 1.0); + assert_eq!(samples, [0.1, -0.2, 0.3, -0.4]); + } + + #[test] + fn limit_peak_scales_above_threshold() { + let mut samples = [0.5_f32, 1.0, 2.0, -1.5]; + let gain = limit_peak_inplace(&mut samples, 0.95); + assert!((gain - 0.475).abs() < 1e-6, "gain = {gain}"); + assert!((samples[0] - 0.2375).abs() < 1e-6); + assert!((samples[1] - 0.475).abs() < 1e-6); + assert!((samples[2] - 0.95).abs() < 1e-6); + assert!((samples[3] - (-0.7125)).abs() < 1e-6); + } + + #[test] + fn limit_peak_handles_zero_and_invalid_thresholds() { + let mut samples = [0.5_f32, 1.0]; + assert_eq!(limit_peak_inplace(&mut samples, 0.0), 1.0); + assert_eq!(samples, [0.5, 1.0]); + assert_eq!(limit_peak_inplace(&mut samples, -1.0), 1.0); + assert_eq!(samples, [0.5, 1.0]); + assert_eq!(limit_peak_inplace(&mut samples, f32::NAN), 1.0); + assert_eq!(samples, [0.5, 1.0]); + } + + #[test] + fn limit_peak_then_downmix_produces_no_clipping() { + // Regression: multi-client mix previously hard-clamped to i16::MAX. + let mut scratch = [1.0_f32, 1.0, -0.5, -0.5, 0.8, 0.8]; + limit_peak_inplace(&mut scratch, 0.95); + let mut out = [0_i16; 3]; + let stats = downmix_stereo_f32_to_mono_i16(&scratch, &mut out, 1.0, false); + assert_eq!(stats.clipped_samples, 0); + assert!(stats.peak_i16 < i16::MAX); + } }