feat(macos): lock-free audio event queue and channel-aware render downmix

macOS realtime audio was suffering buffer underruns on CoreAudio's VPIO
output callback. Root cause was twofold: AudioHandler was decoded under
a Mutex held across the realtime callback, and the render path
hard-coded mono i16 output regardless of the channel count the
callback actually exposed (CoreAudio occasionally hands the callback
stereo or quad output buffers, in which case writing only every Nth
sample produced silence + clicks).

This change brings macOS in line with the lock-free Android audio
architecture introduced for output stutter elimination:

* chanora_audio: AudioPacket / AudioCommand / AudioEventQueue
  (previously gated to `target_os = "android"`) are now compiled on
  macOS too. The decode loop in AudioEngine pushes inbound packets
  into the queue; the VPIO render callback owns AudioHandler outright
  and drains the queue, so the realtime thread never blocks on a
  cross-thread mutex. set_client_volume also routes through the
  command queue on macOS instead of locking the handler.

* voice_render.rs: new downmix_stereo_f32_to_interleaved_i16 helper
  downmixes stereo f32 from AudioHandler to mono i16 and replicates
  that mono sample across every output channel the callback exposes.
  The existing downmix_stereo_f32_to_mono_i16 helper is retained for
  iOS, where VPIO is reliably configured for single-channel output
  via the AudioUnit stream format we pin at unit-create time.
  Compile-gated to ios + test so the macos build doesn't warn on
  dead code.

* ios_voice_unit.rs: render callback reads data.channels from the
  args struct and forwards it to the new interleaved helper, so the
  macOS path tolerates whatever channel count CoreAudio assigns. A
  level decimation counter avoids running sqrt+log10 on every
  callback (~93 Hz) when the Flutter consumer only reads at 30 Hz;
  same regression class as the capture-side fix already in engine.rs.

* mobile_voice_backend.rs: VoiceAudioParams now carries
  event_producer on macOS, and the AudioHandler is no longer wrapped
  in Arc<Mutex<…>> on macOS because ownership moves into the render
  callback. iOS keeps Arc<Mutex<…>> because its callback design
  shares the handler with the decode task.

* lib.rs: audio_event_queue module is now compiled on macOS in
  addition to android.

apps/chanora_flutter/lib/main.dart wraps the home tree in a Stack and
overlays AudioDebugStatsPanel on macOS so the live engine counters
(callback rate, drift, queue depth) used to diagnose the underrun are
visible while iterating on this code. iOS and other platforms are
unaffected.

apps/chanora_flutter/macos/Frameworks/chanora_bridge.framework binary
is rebuilt with these changes so flutter run on macOS picks up the new
realtime path without requiring developers to rebuild the Rust crate
locally. cargo check -p chanora_audio passes on macOS host.
This commit is contained in:
Edison Jwa
2026-06-07 23:27:46 +09:00
parent 1bc2fccd0a
commit 23bfddb6b4
7 changed files with 250 additions and 43 deletions
+10 -3
View File
@@ -34,6 +34,7 @@ import 'src/rust/api.dart' as rust;
import 'src/rust/frb_generated.dart';
import 'src/rust/lib.dart' as rust_err;
import 'widgets/audio_processing_config_state.dart';
import 'widgets/audio_debug_stats_panel.dart';
import 'widgets/chat_panel.dart';
import 'widgets/chat_views.dart';
import 'widgets/client_info_sheet.dart';
@@ -48,6 +49,7 @@ import 'widgets/voice_settings.dart';
import 'package:share_plus/share_plus.dart';
bool get _isMacOS => !kIsWeb && Platform.isMacOS;
bool get _showAudioDebugOverlay => _isMacOS;
const Color _appSurfaceColor = Color(0xFFFFFBFE);
@@ -151,9 +153,14 @@ class _ChanoraAppState extends State<ChanoraApp> {
themeMode: _themeMode,
localizationsDelegates: AppL10n.localizationsDelegates,
supportedLocales: AppL10n.supportedLocales,
home: _BetaHome(
themeMode: _themeMode,
onThemeModeChanged: _setThemeMode,
home: Stack(
children: [
_BetaHome(
themeMode: _themeMode,
onThemeModeChanged: _setThemeMode,
),
if (_showAudioDebugOverlay) const AudioDebugStatsPanel(),
],
),
),
);
@@ -10,6 +10,7 @@ const PACKET_QUEUE_CAPACITY: usize = 100;
const CONTROL_QUEUE_CAPACITY: usize = 32;
/// A raw inbound voice packet waiting to be inserted into AudioHandler.
#[derive(Debug)]
pub struct AudioPacket {
/// Client whose TeamSpeak audio packet this belongs to.
pub client_id: SessionAudioId,
@@ -18,12 +19,14 @@ pub struct AudioPacket {
}
/// Control commands from the main thread to the audio callback.
#[derive(Debug)]
pub enum AudioCommand {
/// Set a client's output volume.
SetVolume(SessionAudioId, f32),
/// Remove a client's decode queue.
// TODO: Wire to client disconnect path; handled in callback but no
// producer currently pushes this command.
#[allow(dead_code)]
RemoveClient(SessionAudioId),
}
@@ -118,3 +121,64 @@ impl AudioEventConsumer {
std::iter::from_fn(move || self.queue.control_queue.pop())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn empty_packet(id: u64) -> AudioPacket {
let audio = chanora_protocol::AudioData::S2C {
codec: chanora_protocol::CodecType::OpusVoice,
id: 0x1234,
from: 0x5678,
data: &[1, 2, 3],
};
let out = chanora_protocol::OutAudio::new(&audio);
AudioPacket {
client_id: SessionAudioId(id),
data: InAudioBuf::try_new(chanora_protocol::Direction::S2C, out.data().to_vec())
.unwrap(),
}
}
#[test]
fn packet_overflow_increments_drop_counter() {
let queue = AudioEventQueue::new();
let producer = AudioEventQueue::producer(&queue);
for i in 0..PACKET_QUEUE_CAPACITY {
let packet = empty_packet(i as u64);
assert!(producer.push_packet(packet).is_ok());
}
let overflow = empty_packet(999);
assert!(producer.push_packet(overflow).is_err());
assert_eq!(queue.packets_dropped.load(Ordering::Relaxed), 1);
}
#[test]
fn consumer_drains_packets_and_controls() {
let queue = AudioEventQueue::new();
let producer = AudioEventQueue::producer(&queue);
let consumer = AudioEventQueue::consumer(&queue);
producer
.push_control(AudioCommand::SetVolume(SessionAudioId(7), 0.5))
.unwrap();
producer.push_packet(empty_packet(42)).unwrap();
let packets: Vec<_> = consumer.drain_packets(8).collect();
assert_eq!(packets.len(), 1);
assert_eq!(packets[0].client_id, SessionAudioId(42));
let controls: Vec<_> = consumer.drain_controls().collect();
assert_eq!(controls.len(), 1);
match controls[0] {
AudioCommand::SetVolume(id, vol) => {
assert_eq!(id, SessionAudioId(7));
assert_eq!(vol, 0.5);
}
AudioCommand::RemoveClient(_) => panic!("unexpected remove-client command"),
}
}
}
+32 -6
View File
@@ -52,7 +52,7 @@ use tsclientlib::audio::AudioHandler;
use chanora_protocol::{InboundVoice, OutPacket};
#[cfg(target_os = "android")]
#[cfg(any(target_os = "android", target_os = "macos"))]
use crate::audio_event_queue::{AudioCommand, AudioEventQueue, AudioPacket};
use crate::AudioError;
@@ -307,9 +307,9 @@ pub struct AudioEngine {
output_muted: Arc<AtomicBool>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
#[cfg(not(target_os = "android"))]
#[cfg(not(any(target_os = "android", target_os = "macos")))]
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
#[cfg(target_os = "android")]
#[cfg(any(target_os = "android", target_os = "macos"))]
audio_event_producer: crate::audio_event_queue::AudioEventProducer,
#[cfg(target_os = "android")]
voice_out_tx: mpsc::Sender<OutPacket>,
@@ -1246,15 +1246,25 @@ impl AudioEngine {
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
#[cfg(target_os = "ios")]
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
Arc::new(Mutex::new(AudioHandler::new()));
#[cfg(target_os = "macos")]
let event_queue = AudioEventQueue::new();
#[cfg(target_os = "macos")]
let event_producer = AudioEventQueue::producer(&event_queue);
let voice_out_tx_for_backend = voice_out_tx.clone();
// Construct the live iOS voice backend. Platform VPIO stays
// the default shipping path; Sonora/RemoteIO remains opt-in.
let ios_voice_backend =
open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams {
#[cfg(target_os = "ios")]
handler: audio_handler.clone(),
#[cfg(target_os = "macos")]
handler: AudioHandler::new(),
#[cfg(target_os = "macos")]
event_producer: event_producer.clone(),
output_gain: output_gain.clone(),
output_muted: output_muted.clone(),
voice_out_tx: voice_out_tx_for_backend,
@@ -1278,8 +1288,11 @@ impl AudioEngine {
// VPIO render callback (commit 4) finds decoded frames
// waiting.
let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
let handler_for_task = audio_handler.clone();
let frames_received_for_task = frames_received.clone();
#[cfg(target_os = "ios")]
let handler_for_task = audio_handler.clone();
#[cfg(target_os = "macos")]
let event_producer_for_task = event_producer.clone();
tokio::spawn(async move {
loop {
tokio::select! {
@@ -1291,12 +1304,22 @@ impl AudioEngine {
match item {
Some(v) => {
let id = SessionAudioId(v.from_client);
#[cfg(target_os = "ios")]
{
let mut h = handler_for_task.lock().unwrap();
if let Err(e) = h.handle_packet(id, v.packet) {
debug!(target: "chanora_audio", error = %e, "decode failed");
} else {
frames_received_for_task.fetch_add(1, Ordering::Relaxed);
}
}
#[cfg(target_os = "macos")]
{
let packet = AudioPacket { client_id: id, data: v.packet };
if event_producer_for_task.push_packet(packet).is_ok() {
frames_received_for_task.fetch_add(1, Ordering::Relaxed);
}
}
}
None => break,
}
@@ -1313,7 +1336,10 @@ impl AudioEngine {
output_muted,
audio_processing_config,
audio_processing_stats,
#[cfg(target_os = "ios")]
audio_handler,
#[cfg(target_os = "macos")]
audio_event_producer: event_producer,
_ios_voice_backend: Mutex::new(Some(ios_voice_backend)),
shutdown_tx: Some(shutdown_tx),
capture_active,
@@ -1667,7 +1693,7 @@ impl AudioEngine {
/// `0.0..4.0`.
pub fn set_client_volume(&self, client_id: u64, volume: f32) {
let clamped = volume.clamp(0.0, 4.0);
#[cfg(target_os = "android")]
#[cfg(any(target_os = "android", target_os = "macos"))]
{
let mut cmd = AudioCommand::SetVolume(SessionAudioId(client_id), clamped);
for _ in 0..64 {
@@ -1686,7 +1712,7 @@ impl AudioEngine {
"set_client_volume: control queue full after 64 retries — volume not applied"
);
}
#[cfg(not(target_os = "android"))]
#[cfg(not(any(target_os = "android", target_os = "macos")))]
{
match self.audio_handler.lock() {
Ok(mut h) => {
+66 -27
View File
@@ -77,6 +77,8 @@ use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamForma
use tokio::sync::mpsc;
use tracing::{debug, error, info, warn};
#[cfg(target_os = "macos")]
use crate::audio_event_queue::{AudioCommand, AudioEventQueue};
use crate::mobile_voice_backend::VoiceAudioParams;
use crate::AudioError;
use chanora_protocol::OutPacket;
@@ -737,13 +739,14 @@ impl IosVoiceUnit {
// buffer + mix. Same primitive cpal + SDL output
// paths use; this is the platform-neutral playback
// contract from `tsclientlib::audio::AudioHandler`.
// 2. Downmix to i16 mono with master gain. VPIO expects
// mono int16 (the stream format we pinned above);
// the handler produces stereo f32. We average L+R
// to a single mono channel rather than dropping R —
// the cpal-side mono-output path made the same
// mistake briefly (commit 6a4dbad / fix) and lost
// half the spatial mix.
// 2. Downmix to mono i16 with master gain, then copy that
// mono sample across every output channel the callback
// exposes. We still request mono Int16 from VPIO, but
// the callback must respect the actual channel count it
// receives. The handler itself produces stereo f32, so
// we average L+R rather than dropping R — the cpal-side
// mono-output path made the same mistake briefly
// (commit 6a4dbad / fix) and lost half the spatial mix.
// 3. Local-mute zeroes the output but STILL drains
// AudioHandler in step 1 so its jitter buffer
// doesn't grow unbounded while muted. This is the
@@ -772,15 +775,28 @@ impl IosVoiceUnit {
//
// Revert to direct call: render callback locks
// AudioHandler, asks for `num_frames` stereo frames, and
// immediately downmixes to i16 mono into the output
// buffer. Same as Linux/SDL, just stereo-f32 -> mono-i16
// converted at the boundary.
// immediately downmixes to mono i16 replicated across the
// callback's actual output channels. Same as Linux/SDL,
// just stereo-f32 -> interleaved-i16 converted at the
// boundary.
let mut scratch_stereo: Vec<f32> = Vec::with_capacity(2048);
#[cfg(target_os = "ios")]
let handler_for_render = params.handler.clone();
#[cfg(target_os = "macos")]
let mut handler_for_render = params.handler;
#[cfg(target_os = "macos")]
let event_consumer = AudioEventQueue::consumer(&params.event_producer.queue());
let output_gain_for_render = params.output_gain.clone();
let output_muted_for_render = params.output_muted.clone();
let audio_processing_stats_for_render = params.audio_processing_stats.clone();
let wav_recorder_for_render = wav_recorder.clone();
// Level meter decimation: the render callback fires ~93
// times/sec, but the bridge consumer (`output_level_stream`)
// reads at ~30 Hz. Computing `sqrt()` + `log10()` every
// callback wastes real-time budget and causes buffer underruns
// on macOS CoreAudio (same regression as the capture side, see
// CaptureState::level_decimation_counter in engine.rs).
let mut render_level_decimation: u32 = 0;
// Diagnostic counters (sampled every 100 callbacks ~= 2 s).
let mut cb_count: u64 = 0;
let mut last_num_frames: usize = 0;
@@ -791,8 +807,13 @@ impl IosVoiceUnit {
let mut render_ref_len: usize = 0;
let mut render_recorder_active = false;
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
let out: &mut [i16] = args.data.buffer;
let num_frames = out.len();
let render_callback::Args {
data,
num_frames,
..
} = args;
let out: &mut [i16] = data.buffer;
let out_channels = data.channels;
// AudioHandler produces 48 kHz stereo f32 (= num_frames * 2 floats).
let needed = num_frames * 2;
if scratch_stereo.len() < needed {
@@ -803,19 +824,33 @@ impl IosVoiceUnit {
// earlier callbacks (when scratch was bigger) would
// leak through otherwise.
scratch_stereo[..needed].fill(0.0);
// Non-blocking fill on the realtime callback thread.
// If the inbound forwarder currently owns this mutex,
// emit this period as silence instead of blocking and
// risking an AudioUnit underrun pop/click.
match handler_for_render.try_lock() {
#[cfg(target_os = "macos")]
{
for cmd in event_consumer.drain_controls() {
match cmd {
AudioCommand::SetVolume(id, vol) => {
if let Some(q) = handler_for_render.get_mut_queues().get_mut(&id) {
q.volume = vol;
}
}
AudioCommand::RemoveClient(id) => {
handler_for_render.get_mut_queues().remove(&id);
}
}
}
for pkt in event_consumer.drain_packets(50) {
if let Err(e) = handler_for_render.handle_packet(pkt.client_id, pkt.data) {
debug!(target: "chanora_audio", error = %e, "decode failed");
}
}
let _removed = handler_for_render.fill_buffer(&mut scratch_stereo[..needed]);
}
#[cfg(target_os = "ios")]
match handler_for_render.lock() {
Ok(mut h) => {
let _removed = h.fill_buffer(&mut scratch_stereo[..needed]);
}
Err(std::sync::TryLockError::WouldBlock) => {
audio_processing_stats_for_render.increment_callback_xrun();
// scratch_stereo is already zeroed above.
}
Err(std::sync::TryLockError::Poisoned(e)) => {
Err(e) => {
// Never panic on the realtime IO thread.
warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {e}");
}
@@ -823,19 +858,23 @@ impl IosVoiceUnit {
let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed));
let muted = output_muted_for_render.load(Ordering::Relaxed);
let mix_stats = crate::voice_render::downmix_stereo_f32_to_mono_i16(
let mix_stats = crate::voice_render::downmix_stereo_f32_to_interleaved_i16(
&scratch_stereo[..needed],
out,
out_channels,
gain,
muted,
);
if mix_stats.clipped_samples > 0 {
audio_processing_stats_for_render.add_clipped_samples(mix_stats.clipped_samples);
}
audio_processing_stats_for_render.update_render(
crate::frame::dbfs(&scratch_stereo[..needed]),
num_frames as u32,
);
render_level_decimation = render_level_decimation.wrapping_add(1);
if render_level_decimation % 3 == 0 {
audio_processing_stats_for_render.update_render(
crate::frame::dbfs(&scratch_stereo[..needed]),
num_frames as u32,
);
}
if let Ok(guard) = wav_recorder_for_render.try_lock() {
if let Some(rec) = guard.as_ref() {
+1 -1
View File
@@ -28,7 +28,7 @@
#![warn(missing_docs)]
#[cfg(target_os = "android")]
#[cfg(any(target_os = "android", target_os = "macos"))]
mod audio_event_queue;
pub mod audio_processing;
pub mod debug_wav;
@@ -67,7 +67,7 @@ pub type BackendEventTx = mpsc::UnboundedSender<BackendEvent>;
pub type AudioSessionId = i32;
/// Engine-owned state shared with mobile voice audio callbacks.
#[cfg_attr(not(target_os = "android"), derive(Clone))]
#[cfg_attr(target_os = "ios", derive(Clone))]
pub(crate) struct VoiceAudioParams {
/// Opus-encoded voice packets sent on this channel toward the
/// protocol layer.
@@ -78,15 +78,15 @@ pub(crate) struct VoiceAudioParams {
pub frames_sent: Arc<AtomicU32>,
/// Pre-encode amplitude scale (1.0 = unity).
pub mic_gain: f32,
/// AudioHandler owned by the Android output callback.
#[cfg(target_os = "android")]
/// AudioHandler owned by the Android/macOS output callback.
#[cfg(any(target_os = "android", target_os = "macos"))]
pub handler: AudioHandler<SessionAudioId>,
/// Producer used by Android engine tasks to feed the output callback.
#[cfg(target_os = "android")]
/// Producer used by Android/macOS engine tasks to feed the output callback.
#[cfg(any(target_os = "android", target_os = "macos"))]
pub event_producer: crate::audio_event_queue::AudioEventProducer,
/// AudioHandler that inbound decode+mix feeds into; the output
/// callback pulls mixed stereo f32 from it.
#[cfg(not(target_os = "android"))]
#[cfg(not(any(target_os = "android", target_os = "macos")))]
pub handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
/// Master output gain (f32 bits stored in AtomicU32 for lock-free
/// cross-thread read from the realtime audio callback).
+71
View File
@@ -12,6 +12,7 @@ pub(crate) struct RenderDownmixStats {
/// The helper is allocation-free and safe for realtime render callbacks.
/// If the stereo source is shorter than expected, the remainder of `out`
/// is filled with silence.
#[cfg(any(target_os = "ios", test))]
pub(crate) fn downmix_stereo_f32_to_mono_i16(
stereo: &[f32],
out: &mut [i16],
@@ -47,6 +48,53 @@ pub(crate) fn downmix_stereo_f32_to_mono_i16(
}
}
pub(crate) fn downmix_stereo_f32_to_interleaved_i16(
stereo: &[f32],
out: &mut [i16],
out_channels: usize,
gain: f32,
muted: bool,
) -> RenderDownmixStats {
if out_channels == 0 {
out.fill(0);
return RenderDownmixStats::default();
}
if muted {
out.fill(0);
return RenderDownmixStats::default();
}
let available_frames = stereo.len() / 2;
let requested_frames = out.len() / out_channels;
if available_frames < requested_frames {
out.fill(0);
}
let mut peak = 0_u16;
let mut clipped_samples = 0_u64;
for (dst_frame, lr) in out
.chunks_exact_mut(out_channels)
.zip(stereo.chunks_exact(2))
{
let mono = (lr[0] + lr[1]) * 0.5 * gain;
let clamped = mono.clamp(-1.0, 1.0);
if (mono - clamped).abs() > f32::EPSILON {
clipped_samples = clipped_samples.saturating_add(1);
}
let sample = (clamped * i16::MAX as f32) as i16;
for dst in dst_frame.iter_mut() {
*dst = sample;
}
peak = peak.max(sample.unsigned_abs());
}
RenderDownmixStats {
peak_i16: peak.min(i16::MAX as u16) as i16,
clipped_samples,
}
}
/// Downmix interleaved stereo f32 samples into mono f32 samples.
///
/// Used for software-AEC render references and debug WAV taps.
@@ -91,6 +139,29 @@ mod tests {
assert_eq!(stats, RenderDownmixStats::default());
}
#[test]
fn downmix_interleaved_i16_copies_mono_to_each_channel() {
let stereo = [1.0_f32, -1.0, 0.25, 0.25];
let mut out = [0_i16; 4];
let stats = downmix_stereo_f32_to_interleaved_i16(&stereo, &mut out, 2, 1.0, false);
assert_eq!(out, [0, 0, 8191, 8191]);
assert_eq!(stats.peak_i16, 8191);
assert_eq!(stats.clipped_samples, 0);
}
#[test]
fn downmix_interleaved_i16_mutes_all_channels() {
let stereo = [1.0_f32, 1.0, -1.0, -1.0];
let mut out = [123_i16; 6];
let stats = downmix_stereo_f32_to_interleaved_i16(&stereo, &mut out, 3, 1.0, true);
assert_eq!(out, [0, 0, 0, 0, 0, 0]);
assert_eq!(stats, RenderDownmixStats::default());
}
#[test]
fn downmix_f32_fills_missing_tail_with_silence() {
let stereo = [1.0_f32, -1.0];