diff --git a/apps/chanora_flutter/lib/main.dart b/apps/chanora_flutter/lib/main.dart index bfcd012..c7ae27f 100644 --- a/apps/chanora_flutter/lib/main.dart +++ b/apps/chanora_flutter/lib/main.dart @@ -34,6 +34,7 @@ import 'src/rust/api.dart' as rust; import 'src/rust/frb_generated.dart'; import 'src/rust/lib.dart' as rust_err; import 'widgets/audio_processing_config_state.dart'; +import 'widgets/audio_debug_stats_panel.dart'; import 'widgets/chat_panel.dart'; import 'widgets/chat_views.dart'; import 'widgets/client_info_sheet.dart'; @@ -48,6 +49,7 @@ import 'widgets/voice_settings.dart'; import 'package:share_plus/share_plus.dart'; bool get _isMacOS => !kIsWeb && Platform.isMacOS; +bool get _showAudioDebugOverlay => _isMacOS; const Color _appSurfaceColor = Color(0xFFFFFBFE); @@ -151,9 +153,14 @@ class _ChanoraAppState extends State { themeMode: _themeMode, localizationsDelegates: AppL10n.localizationsDelegates, supportedLocales: AppL10n.supportedLocales, - home: _BetaHome( - themeMode: _themeMode, - onThemeModeChanged: _setThemeMode, + home: Stack( + children: [ + _BetaHome( + themeMode: _themeMode, + onThemeModeChanged: _setThemeMode, + ), + if (_showAudioDebugOverlay) const AudioDebugStatsPanel(), + ], ), ), ); diff --git a/crates/chanora_audio/src/audio_event_queue.rs b/crates/chanora_audio/src/audio_event_queue.rs index 0a4a185..fb32419 100644 --- a/crates/chanora_audio/src/audio_event_queue.rs +++ b/crates/chanora_audio/src/audio_event_queue.rs @@ -10,6 +10,7 @@ const PACKET_QUEUE_CAPACITY: usize = 100; const CONTROL_QUEUE_CAPACITY: usize = 32; /// A raw inbound voice packet waiting to be inserted into AudioHandler. +#[derive(Debug)] pub struct AudioPacket { /// Client whose TeamSpeak audio packet this belongs to. pub client_id: SessionAudioId, @@ -18,12 +19,14 @@ pub struct AudioPacket { } /// Control commands from the main thread to the audio callback. +#[derive(Debug)] pub enum AudioCommand { /// Set a client's output volume. SetVolume(SessionAudioId, f32), /// Remove a client's decode queue. // TODO: Wire to client disconnect path; handled in callback but no // producer currently pushes this command. + #[allow(dead_code)] RemoveClient(SessionAudioId), } @@ -118,3 +121,64 @@ impl AudioEventConsumer { std::iter::from_fn(move || self.queue.control_queue.pop()) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_packet(id: u64) -> AudioPacket { + let audio = chanora_protocol::AudioData::S2C { + codec: chanora_protocol::CodecType::OpusVoice, + id: 0x1234, + from: 0x5678, + data: &[1, 2, 3], + }; + let out = chanora_protocol::OutAudio::new(&audio); + AudioPacket { + client_id: SessionAudioId(id), + data: InAudioBuf::try_new(chanora_protocol::Direction::S2C, out.data().to_vec()) + .unwrap(), + } + } + + #[test] + fn packet_overflow_increments_drop_counter() { + let queue = AudioEventQueue::new(); + let producer = AudioEventQueue::producer(&queue); + + for i in 0..PACKET_QUEUE_CAPACITY { + let packet = empty_packet(i as u64); + assert!(producer.push_packet(packet).is_ok()); + } + + let overflow = empty_packet(999); + assert!(producer.push_packet(overflow).is_err()); + assert_eq!(queue.packets_dropped.load(Ordering::Relaxed), 1); + } + + #[test] + fn consumer_drains_packets_and_controls() { + let queue = AudioEventQueue::new(); + let producer = AudioEventQueue::producer(&queue); + let consumer = AudioEventQueue::consumer(&queue); + + producer + .push_control(AudioCommand::SetVolume(SessionAudioId(7), 0.5)) + .unwrap(); + producer.push_packet(empty_packet(42)).unwrap(); + + let packets: Vec<_> = consumer.drain_packets(8).collect(); + assert_eq!(packets.len(), 1); + assert_eq!(packets[0].client_id, SessionAudioId(42)); + + let controls: Vec<_> = consumer.drain_controls().collect(); + assert_eq!(controls.len(), 1); + match controls[0] { + AudioCommand::SetVolume(id, vol) => { + assert_eq!(id, SessionAudioId(7)); + assert_eq!(vol, 0.5); + } + AudioCommand::RemoveClient(_) => panic!("unexpected remove-client command"), + } + } +} diff --git a/crates/chanora_audio/src/engine.rs b/crates/chanora_audio/src/engine.rs index 87f6007..dc4e5a4 100644 --- a/crates/chanora_audio/src/engine.rs +++ b/crates/chanora_audio/src/engine.rs @@ -52,7 +52,7 @@ use tsclientlib::audio::AudioHandler; use chanora_protocol::{InboundVoice, OutPacket}; -#[cfg(target_os = "android")] +#[cfg(any(target_os = "android", target_os = "macos"))] use crate::audio_event_queue::{AudioCommand, AudioEventQueue, AudioPacket}; use crate::AudioError; @@ -307,9 +307,9 @@ pub struct AudioEngine { output_muted: Arc, audio_processing_config: Arc>, audio_processing_stats: Arc, - #[cfg(not(target_os = "android"))] + #[cfg(not(any(target_os = "android", target_os = "macos")))] audio_handler: Arc>>, - #[cfg(target_os = "android")] + #[cfg(any(target_os = "android", target_os = "macos"))] audio_event_producer: crate::audio_event_queue::AudioEventProducer, #[cfg(target_os = "android")] voice_out_tx: mpsc::Sender, @@ -1246,15 +1246,25 @@ impl AudioEngine { let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default())); let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); + #[cfg(target_os = "ios")] let audio_handler: Arc>> = Arc::new(Mutex::new(AudioHandler::new())); + #[cfg(target_os = "macos")] + let event_queue = AudioEventQueue::new(); + #[cfg(target_os = "macos")] + let event_producer = AudioEventQueue::producer(&event_queue); let voice_out_tx_for_backend = voice_out_tx.clone(); // Construct the live iOS voice backend. Platform VPIO stays // the default shipping path; Sonora/RemoteIO remains opt-in. let ios_voice_backend = open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams { + #[cfg(target_os = "ios")] handler: audio_handler.clone(), + #[cfg(target_os = "macos")] + handler: AudioHandler::new(), + #[cfg(target_os = "macos")] + event_producer: event_producer.clone(), output_gain: output_gain.clone(), output_muted: output_muted.clone(), voice_out_tx: voice_out_tx_for_backend, @@ -1278,8 +1288,11 @@ impl AudioEngine { // VPIO render callback (commit 4) finds decoded frames // waiting. let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); - let handler_for_task = audio_handler.clone(); let frames_received_for_task = frames_received.clone(); + #[cfg(target_os = "ios")] + let handler_for_task = audio_handler.clone(); + #[cfg(target_os = "macos")] + let event_producer_for_task = event_producer.clone(); tokio::spawn(async move { loop { tokio::select! { @@ -1291,12 +1304,22 @@ impl AudioEngine { match item { Some(v) => { let id = SessionAudioId(v.from_client); + #[cfg(target_os = "ios")] + { let mut h = handler_for_task.lock().unwrap(); if let Err(e) = h.handle_packet(id, v.packet) { debug!(target: "chanora_audio", error = %e, "decode failed"); } else { frames_received_for_task.fetch_add(1, Ordering::Relaxed); } + } + #[cfg(target_os = "macos")] + { + let packet = AudioPacket { client_id: id, data: v.packet }; + if event_producer_for_task.push_packet(packet).is_ok() { + frames_received_for_task.fetch_add(1, Ordering::Relaxed); + } + } } None => break, } @@ -1313,7 +1336,10 @@ impl AudioEngine { output_muted, audio_processing_config, audio_processing_stats, + #[cfg(target_os = "ios")] audio_handler, + #[cfg(target_os = "macos")] + audio_event_producer: event_producer, _ios_voice_backend: Mutex::new(Some(ios_voice_backend)), shutdown_tx: Some(shutdown_tx), capture_active, @@ -1667,7 +1693,7 @@ impl AudioEngine { /// `0.0..4.0`. pub fn set_client_volume(&self, client_id: u64, volume: f32) { let clamped = volume.clamp(0.0, 4.0); - #[cfg(target_os = "android")] + #[cfg(any(target_os = "android", target_os = "macos"))] { let mut cmd = AudioCommand::SetVolume(SessionAudioId(client_id), clamped); for _ in 0..64 { @@ -1686,7 +1712,7 @@ impl AudioEngine { "set_client_volume: control queue full after 64 retries — volume not applied" ); } - #[cfg(not(target_os = "android"))] + #[cfg(not(any(target_os = "android", target_os = "macos")))] { match self.audio_handler.lock() { Ok(mut h) => { diff --git a/crates/chanora_audio/src/ios_voice_unit.rs b/crates/chanora_audio/src/ios_voice_unit.rs index 39a9e37..0c67494 100644 --- a/crates/chanora_audio/src/ios_voice_unit.rs +++ b/crates/chanora_audio/src/ios_voice_unit.rs @@ -77,6 +77,8 @@ use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamForma use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; +#[cfg(target_os = "macos")] +use crate::audio_event_queue::{AudioCommand, AudioEventQueue}; use crate::mobile_voice_backend::VoiceAudioParams; use crate::AudioError; use chanora_protocol::OutPacket; @@ -737,13 +739,14 @@ impl IosVoiceUnit { // buffer + mix. Same primitive cpal + SDL output // paths use; this is the platform-neutral playback // contract from `tsclientlib::audio::AudioHandler`. - // 2. Downmix to i16 mono with master gain. VPIO expects - // mono int16 (the stream format we pinned above); - // the handler produces stereo f32. We average L+R - // to a single mono channel rather than dropping R — - // the cpal-side mono-output path made the same - // mistake briefly (commit 6a4dbad / fix) and lost - // half the spatial mix. + // 2. Downmix to mono i16 with master gain, then copy that + // mono sample across every output channel the callback + // exposes. We still request mono Int16 from VPIO, but + // the callback must respect the actual channel count it + // receives. The handler itself produces stereo f32, so + // we average L+R rather than dropping R — the cpal-side + // mono-output path made the same mistake briefly + // (commit 6a4dbad / fix) and lost half the spatial mix. // 3. Local-mute zeroes the output but STILL drains // AudioHandler in step 1 so its jitter buffer // doesn't grow unbounded while muted. This is the @@ -772,15 +775,28 @@ impl IosVoiceUnit { // // Revert to direct call: render callback locks // AudioHandler, asks for `num_frames` stereo frames, and - // immediately downmixes to i16 mono into the output - // buffer. Same as Linux/SDL, just stereo-f32 -> mono-i16 - // converted at the boundary. + // immediately downmixes to mono i16 replicated across the + // callback's actual output channels. Same as Linux/SDL, + // just stereo-f32 -> interleaved-i16 converted at the + // boundary. let mut scratch_stereo: Vec = Vec::with_capacity(2048); + #[cfg(target_os = "ios")] let handler_for_render = params.handler.clone(); + #[cfg(target_os = "macos")] + let mut handler_for_render = params.handler; + #[cfg(target_os = "macos")] + let event_consumer = AudioEventQueue::consumer(¶ms.event_producer.queue()); let output_gain_for_render = params.output_gain.clone(); let output_muted_for_render = params.output_muted.clone(); let audio_processing_stats_for_render = params.audio_processing_stats.clone(); let wav_recorder_for_render = wav_recorder.clone(); + // Level meter decimation: the render callback fires ~93 + // times/sec, but the bridge consumer (`output_level_stream`) + // reads at ~30 Hz. Computing `sqrt()` + `log10()` every + // callback wastes real-time budget and causes buffer underruns + // on macOS CoreAudio (same regression as the capture side, see + // CaptureState::level_decimation_counter in engine.rs). + let mut render_level_decimation: u32 = 0; // Diagnostic counters (sampled every 100 callbacks ~= 2 s). let mut cb_count: u64 = 0; let mut last_num_frames: usize = 0; @@ -791,8 +807,13 @@ impl IosVoiceUnit { let mut render_ref_len: usize = 0; let mut render_recorder_active = false; unit.set_render_callback(move |args: render_callback::Args>| { - let out: &mut [i16] = args.data.buffer; - let num_frames = out.len(); + let render_callback::Args { + data, + num_frames, + .. + } = args; + let out: &mut [i16] = data.buffer; + let out_channels = data.channels; // AudioHandler produces 48 kHz stereo f32 (= num_frames * 2 floats). let needed = num_frames * 2; if scratch_stereo.len() < needed { @@ -803,19 +824,33 @@ impl IosVoiceUnit { // earlier callbacks (when scratch was bigger) would // leak through otherwise. scratch_stereo[..needed].fill(0.0); - // Non-blocking fill on the realtime callback thread. - // If the inbound forwarder currently owns this mutex, - // emit this period as silence instead of blocking and - // risking an AudioUnit underrun pop/click. - match handler_for_render.try_lock() { + #[cfg(target_os = "macos")] + { + for cmd in event_consumer.drain_controls() { + match cmd { + AudioCommand::SetVolume(id, vol) => { + if let Some(q) = handler_for_render.get_mut_queues().get_mut(&id) { + q.volume = vol; + } + } + AudioCommand::RemoveClient(id) => { + handler_for_render.get_mut_queues().remove(&id); + } + } + } + for pkt in event_consumer.drain_packets(50) { + if let Err(e) = handler_for_render.handle_packet(pkt.client_id, pkt.data) { + debug!(target: "chanora_audio", error = %e, "decode failed"); + } + } + let _removed = handler_for_render.fill_buffer(&mut scratch_stereo[..needed]); + } + #[cfg(target_os = "ios")] + match handler_for_render.lock() { Ok(mut h) => { let _removed = h.fill_buffer(&mut scratch_stereo[..needed]); } - Err(std::sync::TryLockError::WouldBlock) => { - audio_processing_stats_for_render.increment_callback_xrun(); - // scratch_stereo is already zeroed above. - } - Err(std::sync::TryLockError::Poisoned(e)) => { + Err(e) => { // Never panic on the realtime IO thread. warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {e}"); } @@ -823,19 +858,23 @@ impl IosVoiceUnit { let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed)); let muted = output_muted_for_render.load(Ordering::Relaxed); - let mix_stats = crate::voice_render::downmix_stereo_f32_to_mono_i16( + let mix_stats = crate::voice_render::downmix_stereo_f32_to_interleaved_i16( &scratch_stereo[..needed], out, + out_channels, gain, muted, ); if mix_stats.clipped_samples > 0 { audio_processing_stats_for_render.add_clipped_samples(mix_stats.clipped_samples); } - audio_processing_stats_for_render.update_render( - crate::frame::dbfs(&scratch_stereo[..needed]), - num_frames as u32, - ); + render_level_decimation = render_level_decimation.wrapping_add(1); + if render_level_decimation % 3 == 0 { + audio_processing_stats_for_render.update_render( + crate::frame::dbfs(&scratch_stereo[..needed]), + num_frames as u32, + ); + } if let Ok(guard) = wav_recorder_for_render.try_lock() { if let Some(rec) = guard.as_ref() { diff --git a/crates/chanora_audio/src/lib.rs b/crates/chanora_audio/src/lib.rs index cf58813..87d24fd 100644 --- a/crates/chanora_audio/src/lib.rs +++ b/crates/chanora_audio/src/lib.rs @@ -28,7 +28,7 @@ #![warn(missing_docs)] -#[cfg(target_os = "android")] +#[cfg(any(target_os = "android", target_os = "macos"))] mod audio_event_queue; pub mod audio_processing; pub mod debug_wav; diff --git a/crates/chanora_audio/src/mobile_voice_backend.rs b/crates/chanora_audio/src/mobile_voice_backend.rs index 15754e8..582088f 100644 --- a/crates/chanora_audio/src/mobile_voice_backend.rs +++ b/crates/chanora_audio/src/mobile_voice_backend.rs @@ -67,7 +67,7 @@ pub type BackendEventTx = mpsc::UnboundedSender; pub type AudioSessionId = i32; /// Engine-owned state shared with mobile voice audio callbacks. -#[cfg_attr(not(target_os = "android"), derive(Clone))] +#[cfg_attr(target_os = "ios", derive(Clone))] pub(crate) struct VoiceAudioParams { /// Opus-encoded voice packets sent on this channel toward the /// protocol layer. @@ -78,15 +78,15 @@ pub(crate) struct VoiceAudioParams { pub frames_sent: Arc, /// Pre-encode amplitude scale (1.0 = unity). pub mic_gain: f32, - /// AudioHandler owned by the Android output callback. - #[cfg(target_os = "android")] + /// AudioHandler owned by the Android/macOS output callback. + #[cfg(any(target_os = "android", target_os = "macos"))] pub handler: AudioHandler, - /// Producer used by Android engine tasks to feed the output callback. - #[cfg(target_os = "android")] + /// Producer used by Android/macOS engine tasks to feed the output callback. + #[cfg(any(target_os = "android", target_os = "macos"))] pub event_producer: crate::audio_event_queue::AudioEventProducer, /// AudioHandler that inbound decode+mix feeds into; the output /// callback pulls mixed stereo f32 from it. - #[cfg(not(target_os = "android"))] + #[cfg(not(any(target_os = "android", target_os = "macos")))] pub handler: Arc>>, /// Master output gain (f32 bits stored in AtomicU32 for lock-free /// cross-thread read from the realtime audio callback). diff --git a/crates/chanora_audio/src/voice_render.rs b/crates/chanora_audio/src/voice_render.rs index 80c11d6..6fe02fa 100644 --- a/crates/chanora_audio/src/voice_render.rs +++ b/crates/chanora_audio/src/voice_render.rs @@ -12,6 +12,7 @@ pub(crate) struct RenderDownmixStats { /// The helper is allocation-free and safe for realtime render callbacks. /// If the stereo source is shorter than expected, the remainder of `out` /// is filled with silence. +#[cfg(any(target_os = "ios", test))] pub(crate) fn downmix_stereo_f32_to_mono_i16( stereo: &[f32], out: &mut [i16], @@ -47,6 +48,53 @@ pub(crate) fn downmix_stereo_f32_to_mono_i16( } } +pub(crate) fn downmix_stereo_f32_to_interleaved_i16( + stereo: &[f32], + out: &mut [i16], + out_channels: usize, + gain: f32, + muted: bool, +) -> RenderDownmixStats { + if out_channels == 0 { + out.fill(0); + return RenderDownmixStats::default(); + } + + if muted { + out.fill(0); + return RenderDownmixStats::default(); + } + + let available_frames = stereo.len() / 2; + let requested_frames = out.len() / out_channels; + if available_frames < requested_frames { + out.fill(0); + } + + let mut peak = 0_u16; + let mut clipped_samples = 0_u64; + for (dst_frame, lr) in out + .chunks_exact_mut(out_channels) + .zip(stereo.chunks_exact(2)) + { + let mono = (lr[0] + lr[1]) * 0.5 * gain; + let clamped = mono.clamp(-1.0, 1.0); + if (mono - clamped).abs() > f32::EPSILON { + clipped_samples = clipped_samples.saturating_add(1); + } + let sample = (clamped * i16::MAX as f32) as i16; + for dst in dst_frame.iter_mut() { + *dst = sample; + } + peak = peak.max(sample.unsigned_abs()); + } + + RenderDownmixStats { + peak_i16: peak.min(i16::MAX as u16) as i16, + clipped_samples, + } +} + /// Downmix interleaved stereo f32 samples into mono f32 samples. /// /// Used for software-AEC render references and debug WAV taps. @@ -91,6 +139,29 @@ mod tests { assert_eq!(stats, RenderDownmixStats::default()); } + #[test] + fn downmix_interleaved_i16_copies_mono_to_each_channel() { + let stereo = [1.0_f32, -1.0, 0.25, 0.25]; + let mut out = [0_i16; 4]; + + let stats = downmix_stereo_f32_to_interleaved_i16(&stereo, &mut out, 2, 1.0, false); + + assert_eq!(out, [0, 0, 8191, 8191]); + assert_eq!(stats.peak_i16, 8191); + assert_eq!(stats.clipped_samples, 0); + } + + #[test] + fn downmix_interleaved_i16_mutes_all_channels() { + let stereo = [1.0_f32, 1.0, -1.0, -1.0]; + let mut out = [123_i16; 6]; + + let stats = downmix_stereo_f32_to_interleaved_i16(&stereo, &mut out, 3, 1.0, true); + + assert_eq!(out, [0, 0, 0, 0, 0, 0]); + assert_eq!(stats, RenderDownmixStats::default()); + } + #[test] fn downmix_f32_fills_missing_tail_with_silence() { let stereo = [1.0_f32, -1.0];