From 1efeaac19d598ce183e52261ddbd499e506ff7c1 Mon Sep 17 00:00:00 2001 From: Edison Jwa Date: Thu, 11 Jun 2026 12:05:54 +0900 Subject: [PATCH] refactor(audio): split engine.rs into engine/ module directory (TODO-022) Split monolithic engine.rs (3,244 lines) into focused modules: mod.rs (1,106), capture.rs (515), render.rs (176), lifecycle.rs (1,149). Capture and render are desktop-only (cfg-gated). No behavioral changes. All public API paths preserved. --- crates/chanora_audio/src/engine.rs | 3241 ------------------ crates/chanora_audio/src/engine/capture.rs | 515 +++ crates/chanora_audio/src/engine/lifecycle.rs | 1149 +++++++ crates/chanora_audio/src/engine/mod.rs | 1106 ++++++ crates/chanora_audio/src/engine/render.rs | 176 + 5 files changed, 2946 insertions(+), 3241 deletions(-) delete mode 100644 crates/chanora_audio/src/engine.rs create mode 100644 crates/chanora_audio/src/engine/capture.rs create mode 100644 crates/chanora_audio/src/engine/lifecycle.rs create mode 100644 crates/chanora_audio/src/engine/mod.rs create mode 100644 crates/chanora_audio/src/engine/render.rs diff --git a/crates/chanora_audio/src/engine.rs b/crates/chanora_audio/src/engine.rs deleted file mode 100644 index fcea714..0000000 --- a/crates/chanora_audio/src/engine.rs +++ /dev/null @@ -1,3241 +0,0 @@ -//! Audio engine — owns platform-native capture/playback, the Opus -//! encoder, and the tsclientlib `AudioHandler` for decode+mix. -//! -//! The engine is started after a protocol connection is established -//! and stopped before disconnect. It does not retry on device -//! change. - -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -use std::sync::{Arc, Mutex}; - -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -use cpal::{SampleFormat, SizedSample}; -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -use std::collections::hash_map::DefaultHasher; -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -use std::hash::{Hash, Hasher}; -use tokio::sync::mpsc; -use tracing::{debug, info}; -// `error!` and `warn!` are used only inside the cpal capture / -// playback paths (`build_input_stream`, `build_output_stream`, -// `try_open_capture` log lines). Cfg-gate the imports too so iOS -// builds don't carry an unused-imports warning. -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -use tracing::{error, warn}; - -#[cfg(any(target_os = "ios", target_os = "android"))] -use tracing::warn; - -use tsclientlib::audio::AudioHandler; - -use chanora_protocol::{InboundVoice, OutPacket}; - -#[cfg(target_os = "android")] -use crate::audio_event_queue::{AudioCommand, AudioEventQueue, AudioPacket}; -use crate::AudioError; - -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -use audiopus::coder::Encoder as OpusEncoder; -/// Stable Chanora-side identifier for AudioHandler bookkeeping. -/// We only ever have one connection at a time (DEC-006), so this is -/// trivially unique. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub struct SessionAudioId(pub u64); - -/// Audio framing: 48 kHz mono, 20 ms = 960 samples per frame. -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -const SAMPLE_RATE: u32 = 48_000; -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -const FRAME_SAMPLES: usize = 48_000 / 50; // 960 - -/// List of available audio devices from the platform. -#[derive(Debug, Clone)] -pub struct AudioDeviceList { - /// Available input (capture) devices. - pub input_devices: Vec, - /// Available output (playback) devices. - pub output_devices: Vec, -} - -/// Info about a single audio device. -#[derive(Debug, Clone)] -pub struct AudioDeviceInfo { - /// Stable platform-reported device identifier. - pub id: String, - /// Human-readable device name from the OS. - pub name: String, - /// Additional device details useful for disambiguation. - pub details: String, - /// True if the OS reports this as the default device. - pub is_default: bool, -} - -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -fn desktop_device_id(device: &cpal::Device) -> Option { - device.id().ok().map(|id| id.to_string()) -} - -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -fn short_device_id(id: &str) -> String { - let mut hasher = DefaultHasher::new(); - id.hash(&mut hasher); - format!("{:016x}", hasher.finish()) -} - -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -fn describe_device(device: &cpal::Device) -> Option { - let id = desktop_device_id(device)?; - let description = device.description().ok(); - let name = description - .as_ref() - .map(|d| d.name().trim().to_owned()) - .filter(|name| !name.is_empty()) - .unwrap_or_else(|| format!("Device {}", &short_device_id(&id)[..8])); - - let mut details = Vec::new(); - if let Some(description) = description.as_ref() { - if let Some(manufacturer) = description.manufacturer() { - details.push(manufacturer.to_owned()); - } - if let Some(driver) = description.driver() { - details.push(driver.to_owned()); - } - let device_type = description.device_type(); - if device_type != cpal::device_description::DeviceType::Unknown { - details.push(format!("{device_type}")); - } - let interface_type = description.interface_type(); - if interface_type != cpal::device_description::InterfaceType::Unknown { - details.push(format!("{interface_type}")); - } - } - details.push(format!("id={}", short_device_id(&id))); - - Some(AudioDeviceInfo { - id, - name, - details: details.join(" · "), - is_default: false, - }) -} - -/// Enumerate available audio input and output devices. -/// On mobile platforms (iOS, Android) returns an empty list because -/// device selection is managed by the OS audio session. -#[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") -))] -pub fn list_audio_devices() -> AudioDeviceList { - use cpal::traits::HostTrait; - let mut list = AudioDeviceList { - input_devices: Vec::new(), - output_devices: Vec::new(), - }; - let host = cpal::default_host(); - let default_in = host - .default_input_device() - .and_then(|device| desktop_device_id(&device)); - let default_out = host - .default_output_device() - .and_then(|device| desktop_device_id(&device)); - if let Ok(devices) = host.input_devices() { - for d in devices { - if let Some(mut device) = describe_device(&d) { - device.is_default = default_in - .as_ref() - .is_some_and(|default_id| default_id == &device.id); - list.input_devices.push(device); - } - } - } - if let Ok(devices) = host.output_devices() { - for d in devices { - if let Some(mut device) = describe_device(&d) { - device.is_default = default_out - .as_ref() - .is_some_and(|default_id| default_id == &device.id); - list.output_devices.push(device); - } - } - } - list -} - -#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))] -/// Enumerate available audio input and output devices. -pub fn list_audio_devices() -> AudioDeviceList { - AudioDeviceList { - input_devices: Vec::new(), - output_devices: Vec::new(), - } -} - -/// Engine configuration. -#[derive(Clone)] -pub struct AudioEngineConfig { - /// Input gain applied before encoding (1.0 = pass-through). - pub mic_gain: f32, - /// Initial PTT state. When false the encoder is bypassed and no - /// outbound packets are produced. - pub ptt_initial: bool, - /// Audio-effect toggles. The struct is honoured by *naming* but - /// the filters themselves are still no-op in Beta — see the - /// crate-level docs and DEC-007/008/009/010. - pub effects: crate::AudioEffects, - /// A.5 mobile: prefer the OS-provided "voice communication" - /// audio source on mobile platforms (Android - /// `MediaRecorder.AudioSource.VOICE_COMMUNICATION`, iOS - /// `AVAudioSession.Mode.voiceChat`). On Linux desktop this is - /// ignored — the DSP chain stays a no-op and we use the - /// default ALSA/PipeWire source. - /// - /// Android status: this is enforced by the Oboe-only backend, - /// which opens input with `VoiceCommunication` and output with - /// voice-communication usage / speech content. Setting `false` - /// is rejected on Android because the P0 path intentionally has - /// no generic mobile-audio fallback. - pub mobile_voice_preset: bool, - /// Optional input device id override. When `None`, the system - /// default input device is used. Set to a device id from - /// [`list_audio_devices`] to pin a specific microphone. - pub input_device_id: Option, - /// Optional output device id override. - pub output_device_id: Option, - /// Optional selector used by P1 VoiceActivity to publish VAD state. - #[doc(hidden)] - pub voice_activity_selector: Option>, -} - -impl std::fmt::Debug for AudioEngineConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AudioEngineConfig") - .field("mic_gain", &self.mic_gain) - .field("ptt_initial", &self.ptt_initial) - .field("effects", &self.effects) - .field("mobile_voice_preset", &self.mobile_voice_preset) - .field("input_device_id", &self.input_device_id) - .field("output_device_id", &self.output_device_id) - .field( - "voice_activity_selector", - &self.voice_activity_selector.as_ref().map(|_| "present"), - ) - .finish() - } -} - -impl Default for AudioEngineConfig { - fn default() -> Self { - Self { - mic_gain: 1.0, - ptt_initial: false, - effects: crate::AudioEffects::default(), - mobile_voice_preset: true, - input_device_id: None, - output_device_id: None, - voice_activity_selector: None, - } - } -} - -/// Running audio engine. Drop = stop. -pub struct AudioEngine { - /// `transmit_active` is the authoritative gate for outbound - /// voice — the Opus encoder feed consults this flag once per - /// 20 ms frame. PTT subsystems (focused widget, future - /// Windows / macOS / Linux global backends) drive this flag - /// through [`Self::set_transmit_active`]; nothing else is - /// permitted to flip it (SAD-075 / SDD-089). - transmit_gate: crate::ptt::AudioTransmitGate, - frames_sent: Arc, - frames_received: Arc, - /// Master output gain as f32 bits in an AtomicU32. Default 1.0. - /// Adjusted via [`Self::set_output_gain`] from the bridge. - output_gain: Arc, - /// Master output mute. When true the output callback fills the - /// device buffer with silence regardless of incoming voice - /// frames. Used for self-output-mute on the local device, - /// independent of the server-side mute the protocol layer - /// broadcasts. - output_muted: Arc, - audio_processing_config: Arc>, - audio_processing_stats: Arc, - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - silero_vad_worker: Arc>>, - #[cfg(not(target_os = "android"))] - audio_handler: Arc>>, - #[cfg(target_os = "android")] - audio_event_producer: crate::audio_event_queue::AudioEventProducer, - #[cfg(target_os = "android")] - voice_out_tx: mpsc::Sender, - #[cfg(target_os = "android")] - voice_activity_selector: Option>, - #[cfg(target_os = "android")] - mic_gain: f32, - // Streams must be dropped to stop audio. Both are `!Send` because - // cpal's Stream isn't Send on some backends; we keep them in an - // Option wrapped by Mutex so stop() can move them out. On Linux - // the output side is `crate::sdl_output::SdlOutput` instead of a - // cpal Stream (see the SDD note inside `sdl_output.rs`); the - // same unsafe Send/Sync impl below covers both. On iOS both - // sides collapse into a single `IosVoiceUnit` (one - // VoiceProcessingIO AudioUnit hosts mic + speaker) — cpal is - // unused on iOS for the reasons documented in - // `ios_voice_unit.rs`. - #[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") - ))] - _input_stream: Mutex>, - #[cfg(target_os = "linux")] - _output_stream: Mutex>, - #[cfg(all( - not(target_os = "linux"), - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") - ))] - _output_stream: Mutex>, - #[cfg(any(target_os = "ios", target_os = "macos"))] - _ios_voice_backend: Mutex>, - /// SDD-111..SDD-115: Android Oboe voice backend. Owns the input - /// and output streams, SDD-113 hardware-effect handles, and the - /// foreground-service lifecycle; tearing it down on engine drop - /// unbinds effects and stops the service in SDD-115 reverse - /// order. - #[cfg(target_os = "android")] - _android_voice_unit: Arc>>, - /// Persist the Android stream request so route-change reopen uses the - /// same effect and latency policy as the original session start. - #[cfg(target_os = "android")] - android_voice_stream_config: Arc>, - /// SDD-108 §1/§2: refcount-composable audio-mode controller. - /// Snapshots `AudioManager.getMode()` on the 0 → 1 transition and - /// restores it on the 1 → 0 transition. Held in a `Mutex` so the - /// snapshot/restore critical section is serialized across - /// composed callers (SDD-108 §2). Engine-scoped because the - /// mode lifecycle is bound to the voice-session lifecycle - /// (SDD-108 §3). - #[cfg(target_os = "android")] - audio_mode_stack: Arc>, - // Hand the inbound-voice forwarder task a shutdown signal. - shutdown_tx: Option>, - /// True if the capture stream actually opened. If false (typical - /// in headless environments with null sources, or where the user - /// denied microphone permission), PTT becomes a no-op and - /// `frames_sent` stays at 0. - capture_active: bool, -} - -// cpal::Stream is not Send. We keep the engine pinned to the thread -// it was constructed on — `chanora_core` spawns it inside a -// `tokio::task::spawn_blocking` so the streams stay on that worker. -// This `unsafe impl Send` is necessary because the outer Arc -// is stored in core's session and must move into a task. The streams -// themselves are only mutated through the Mutex and are dropped on -// the same thread that owns them. -// -// SAFETY: cpal's Stream is not Send because the underlying audio API -// callback thread may not be transferable. We never invoke methods on -// the streams from any thread but the owning one; we only ever *drop* -// them, which cpal documents as safe from any thread for ALSA and -// PipeWire (Linux backend used here). For Windows/macOS the contract -// may differ; production Beta+ work must revisit per-platform. -unsafe impl Send for AudioEngine {} -unsafe impl Sync for AudioEngine {} - -#[cfg(any(target_os = "ios", target_os = "macos"))] -enum IosVoiceBackend { - Vpio(crate::ios_voice_unit::IosVoiceUnit), -} - -#[cfg(any(target_os = "ios", target_os = "macos"))] -impl IosVoiceBackend { - fn restart(&mut self) -> Result<(), AudioError> { - #[cfg(target_os = "ios")] - { - match self { - Self::Vpio(unit) => unit.restart(), - } - } - #[cfg(target_os = "macos")] - { - let _ = self; - Ok(()) - } - } - - fn pause(&mut self) -> Result<(), AudioError> { - #[cfg(target_os = "ios")] - { - match self { - Self::Vpio(unit) => unit.pause(), - } - } - #[cfg(target_os = "macos")] - { - let _ = self; - Ok(()) - } - } - - fn resume(&mut self) -> Result<(), AudioError> { - #[cfg(target_os = "ios")] - { - match self { - Self::Vpio(unit) => unit.resume(), - } - } - #[cfg(target_os = "macos")] - { - let _ = self; - Ok(()) - } - } -} - -#[cfg(any(target_os = "ios", target_os = "macos"))] -fn open_ios_voice_backend( - params: crate::mobile_voice_backend::VoiceAudioParams, -) -> Result { - let unit = crate::ios_voice_unit::IosVoiceUnit::start(params)?; - Ok(IosVoiceBackend::Vpio(unit)) -} - -impl AudioEngine { - #[cfg(target_os = "android")] - fn spawn_android_backend_event_task( - android_voice_unit: Arc>>, - android_voice_stream_config: Arc< - Mutex, - >, - voice_out_tx: mpsc::Sender, - transmit_gate: crate::ptt::AudioTransmitGate, - frames_sent: Arc, - event_producer: crate::audio_event_queue::AudioEventProducer, - output_gain: Arc, - output_muted: Arc, - voice_activity_selector: Option>, - audio_processing_config: Arc>, - audio_processing_stats: Arc, - mic_gain: f32, - audio_mode_stack: Arc>, - mut event_rx: crate::mobile_voice_backend::BackendEventRx, - ) { - use crate::mobile_voice_backend::BackendEvent; - - tokio::spawn(async move { - while let Some(event) = event_rx.recv().await { - match event { - BackendEvent::Disconnected => { - warn!(target: "chanora_audio", "android: backend disconnected; reopening voice unit"); - crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); - crate::android_voice_unit::chanora_android_abandon_audio_focus(); - crate::android_voice_unit::clear_global_event_sender(); - - if let Some(mut unit) = android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() { - use crate::mobile_voice_backend::MobileVoiceAudioBackend; - if let Err(e) = unit.close() { - warn!( - target: "chanora_audio", - error = %e, - "android: failed to close disconnected voice unit before reopen" - ); - } - } - - if crate::android_voice_unit::chanora_android_start_voice_service() { - info!( - target: "chanora_audio", - "android: voice foreground service restart dispatched after backend disconnect" - ); - } - - match android_get_audio_mode() { - Ok(current_mode) => { - if current_mode != ANDROID_MODE_IN_COMMUNICATION { - match android_set_audio_mode(ANDROID_MODE_IN_COMMUNICATION) { - Ok(()) => info!( - target: "chanora_audio", - prior_mode = current_mode, - "android: AudioManager mode re-engaged after backend disconnect" - ), - Err(e) => warn!( - target: "chanora_audio", - error = %e, - prior_mode = current_mode, - "android: failed to re-engage MODE_IN_COMMUNICATION after backend disconnect" - ), - } - } - } - Err(e) => warn!( - target: "chanora_audio", - error = %e, - "android: AudioManager.getMode failed during backend reopen" - ), - } - - let cfg_av = android_voice_stream_config.lock().unwrap_or_else(|e| e.into_inner()).clone(); - let params = crate::mobile_voice_backend::VoiceAudioParams { - voice_out_tx: voice_out_tx.clone(), - transmit_active: transmit_gate.flag_arc(), - frames_sent: frames_sent.clone(), - mic_gain, - handler: AudioHandler::new(), - event_producer: event_producer.clone(), - output_gain: output_gain.clone(), - output_muted: output_muted.clone(), - voice_activity_selector: voice_activity_selector.clone(), - audio_processing_config: audio_processing_config.clone(), - audio_processing_stats: audio_processing_stats.clone(), - }; - - match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) { - Ok(mut reopened) => { - use crate::mobile_voice_backend::MobileVoiceAudioBackend; - match reopened.start() { - Ok(()) => { - if let Some(next_rx) = reopened.take_event_rx() { - Self::spawn_android_backend_event_task( - android_voice_unit.clone(), - android_voice_stream_config.clone(), - voice_out_tx.clone(), - transmit_gate.clone(), - frames_sent.clone(), - event_producer.clone(), - output_gain.clone(), - output_muted.clone(), - voice_activity_selector.clone(), - audio_processing_config.clone(), - audio_processing_stats.clone(), - mic_gain, - audio_mode_stack.clone(), - next_rx, - ); - } - crate::android_voice_unit::register_global_event_sender( - reopened.event_sender(), - ); - if crate::android_voice_unit::chanora_android_request_audio_focus() { - info!( - target: "chanora_audio", - "android: audio focus re-requested after backend disconnect" - ); - } - if crate::android_voice_unit::chanora_android_start_bluetooth_sco() { - info!( - target: "chanora_audio", - "android: bluetooth route re-engaged after backend disconnect" - ); - } - *android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()) = Some(reopened); - } - Err(e) => warn!( - target: "chanora_audio", - error = %e, - "android: reopened voice unit failed to start after backend disconnect" - ), - } - } - Err(e) => warn!( - target: "chanora_audio", - error = %e, - "android: failed to reopen voice unit after backend disconnect" - ), - } - } - BackendEvent::FocusLost => { - warn!( - target: "chanora_audio", - "android: audio focus lost permanently (SDD-109); engine should leave session" - ); - } - BackendEvent::FocusTransient => { - info!( - target: "chanora_audio", - "android: transient audio focus loss (SDD-109); pausing capture" - ); - } - BackendEvent::FocusTransientCanDuck => { - info!( - target: "chanora_audio", - "android: transient audio focus loss with ducking (SDD-109); continuing" - ); - } - BackendEvent::FocusGain => { - info!( - target: "chanora_audio", - "android: audio focus regained (SDD-109); resuming capture" - ); - } - BackendEvent::BluetoothScoStateChanged(s) => { - info!( - target: "chanora_audio", - sco_state = s, - "android: Bluetooth SCO state changed (SDD-110)" - ); - } - } - } - }); - } - - /// Start the engine: open capture + playback streams, spawn the - /// inbound-voice forwarder, return a handle. - pub fn start( - cfg: AudioEngineConfig, - voice_out_tx: mpsc::Sender, - voice_in_rx: mpsc::Receiver, - ) -> Result { - let gate = crate::ptt::AudioTransmitGate::new(cfg.ptt_initial); - Self::start_with_gate(cfg, voice_out_tx, voice_in_rx, gate) - } - - /// Start the engine using an externally-owned - /// [`AudioTransmitGate`]. The gate is shared with whatever - /// upstream (typically [`crate::TransmitModeSelector`]) is the - /// authoritative writer of `transmit_active`. See SAD-083. - pub fn start_with_gate( - cfg: AudioEngineConfig, - voice_out_tx: mpsc::Sender, - voice_in_rx: mpsc::Receiver, - transmit_gate: crate::ptt::AudioTransmitGate, - ) -> Result { - Self::start_with_gate_platform(cfg, voice_out_tx, voice_in_rx, transmit_gate) - } - - // Apple platforms route to a separate backend (VoiceProcessingIO - // via coreaudio-rs) because cpal does not expose the native - // voice-processing AudioUnit controls Chanora needs for VoIP. - #[cfg(any(target_os = "ios", target_os = "macos"))] - fn start_with_gate_platform( - cfg: AudioEngineConfig, - voice_out_tx: mpsc::Sender, - voice_in_rx: mpsc::Receiver, - transmit_gate: crate::ptt::AudioTransmitGate, - ) -> Result { - Self::start_with_gate_ios(cfg, voice_out_tx, voice_in_rx, transmit_gate) - } - - #[cfg(target_os = "android")] - fn start_with_gate_platform( - cfg: AudioEngineConfig, - voice_out_tx: mpsc::Sender, - voice_in_rx: mpsc::Receiver, - transmit_gate: crate::ptt::AudioTransmitGate, - ) -> Result { - Self::start_with_gate_android(cfg, voice_out_tx, voice_in_rx, transmit_gate) - } - - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - fn start_with_gate_platform( - cfg: AudioEngineConfig, - voice_out_tx: mpsc::Sender, - voice_in_rx: mpsc::Receiver, - transmit_gate: crate::ptt::AudioTransmitGate, - ) -> Result { - Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate) - } - - /// Non-Apple/non-Android implementation: cpal capture + (cpal | SDL2) output. - /// Kept as a separate function so the Apple and Android paths can - /// short-circuit at the top of `start_with_gate` without dragging a 200-line - /// cfg-gated block. Body is the pre-iOS-port code, unchanged - /// except for the new function name + signature. - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - fn start_with_gate_cpal( - cfg: AudioEngineConfig, - voice_out_tx: mpsc::Sender, - mut voice_in_rx: mpsc::Receiver, - transmit_gate: crate::ptt::AudioTransmitGate, - ) -> Result { - let host = cpal::default_host(); - info!( - target: "chanora_audio", - host_id = ?host.id(), - "starting audio engine: cpal host selected" - ); - - /// Helper: find a device by stable id, falling back to default. - fn find_device( - host: &cpal::Host, - default_fn: DefaultFn, - all_fn: AllFn, - prefer: Option<&str>, - ) -> Option - where - DefaultFn: Fn(&cpal::Host) -> Option, - AllFn: Fn(&cpal::Host) -> Result, - Devices: IntoIterator, - { - if let Some(id) = prefer { - if let Ok(devices) = all_fn(host) { - for d in devices { - if desktop_device_id(&d).as_deref() == Some(id) { - return Some(d); - } - } - } - } - default_fn(host) - } - - let in_dev = find_device( - &host, - cpal::Host::default_input_device, - cpal::Host::input_devices, - cfg.input_device_id.as_deref(), - ) - .ok_or(AudioError::NoInputDevice)?; - - let out_dev = find_device( - &host, - cpal::Host::default_output_device, - cpal::Host::output_devices, - cfg.output_device_id.as_deref(), - ) - .ok_or(AudioError::NoOutputDevice)?; - - info!( - target: "chanora_audio", - in_device = %in_dev.description().map(|d| d.name().to_owned()).unwrap_or_default(), - out_device = %out_dev.description().map(|d| d.name().to_owned()).unwrap_or_default(), - "starting audio engine" - ); - - // Log the cpal-reported default configs *before* trying to - // open streams, so a downstream stream-build failure can - // be cross-referenced against what the platform reported - // as its default format. Some locale / driver combinations - // on Windows have been observed to expose configs that - // accept device enumeration but reject `default_*_config` - // afterwards (reported on the ko-KR Windows 11 host as - // "Start audio button not work"). Promote what would - // otherwise be silent or laconic errors into structured - // log records the user can paste back. - match in_dev.default_input_config() { - Ok(c) => info!( - target: "chanora_audio", - channels = c.channels(), - sample_rate = c.sample_rate(), - sample_format = ?c.sample_format(), - "default_input_config reported" - ), - Err(e) => warn!( - target: "chanora_audio", - error = %e, - "default_input_config FAILED — capture will be disabled" - ), - } - match out_dev.default_output_config() { - Ok(c) => info!( - target: "chanora_audio", - channels = c.channels(), - sample_rate = c.sample_rate(), - sample_format = ?c.sample_format(), - "default_output_config reported" - ), - Err(e) => warn!( - target: "chanora_audio", - error = %e, - "default_output_config FAILED — output stream will fail to build" - ), - } - - let transmit_flag_for_capture = transmit_gate.flag_arc(); - let frames_sent = Arc::new(AtomicU32::new(0)); - let frames_received = Arc::new(AtomicU32::new(0)); - let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits())); - let output_muted = Arc::new(AtomicBool::new(false)); - let (audio_processing_config, audio_processing_stats, silero_vad_worker) = - new_desktop_audio_processing_state(); - - // ---------- Capture ---------- - // Capture is best-effort. If the platform default input - // device refuses any supported config (typical for - // headless null sources or for users who deny the mic - // permission) we log and continue — playback alone is - // still useful. PTT becomes a no-op in that case. - let capture_result = try_open_capture( - &in_dev, - voice_out_tx, - transmit_flag_for_capture, - frames_sent.clone(), - cfg.mic_gain, - cfg.voice_activity_selector.clone(), - audio_processing_config.clone(), - silero_vad_worker.clone(), - audio_processing_stats.clone(), - ); - let (input_stream, capture_active) = match capture_result { - Ok(s) => (Some(s), true), - Err(e) => { - warn!( - target: "chanora_audio", - error = %e, - "capture stream unavailable; continuing with playback only" - ); - (None, false) - } - }; - if let Some(s) = &input_stream { - s.play() - .map_err(|e| AudioError::Backend(format!("input play: {e}")))?; - } - - // ---------- Playback ---------- - let audio_handler: Arc>> = - Arc::new(Mutex::new(AudioHandler::new())); - - // Linux uses SDL2 for output (Qint / upstream tsclientlib - // pattern). cpal's Linux backend opens raw ALSA which routes - // through `dmix`+`plug` and produces audible crackling / - // popping on the 48 kHz → device-rate step. SDL2 on the same - // box routes through PipeWire's PA bridge (or PulseAudio) - // whose resampler is high-quality. We keep cpal on Windows - // and macOS — both have native backends (WASAPI / CoreAudio) - // without this problem. See `crates/chanora_audio/src/sdl_output.rs` - // for the full rationale. - #[cfg(target_os = "linux")] - let output_stream = crate::sdl_output::SdlOutput::start( - audio_handler.clone(), - output_gain.clone(), - output_muted.clone(), - )?; - - #[cfg(not(target_os = "linux"))] - let output_stream = { - let out_cfg = out_dev - .default_output_config() - .map_err(|e| AudioError::StreamConfig(format!("output default: {e}")))?; - let out_format = out_cfg.sample_format(); - let dev_sample_rate = out_cfg.sample_rate(); - let dev_channels = out_cfg.channels() as usize; - // Buffer-size rationale: - // * Windows (WASAPI via cpal): the default period is - // small enough to expose audio-thread scheduler - // jitter on shared-mode endpoints. Pinning at 2048 - // frames (~46 ms @ 44.1 kHz) gives the Opus decode - // callback enough headroom while still being well - // under voice-chat latency tolerance. - // * Apple platforms do not reach this cpal path; they - // use direct VoiceProcessingIO AudioUnits. - #[cfg(target_os = "windows")] - let buffer_size = cpal::BufferSize::Fixed(2048); - #[cfg(not(target_os = "windows"))] - let buffer_size = cpal::BufferSize::Default; - let out_stream_cfg = cpal::StreamConfig { - channels: out_cfg.channels(), - sample_rate: out_cfg.sample_rate(), - buffer_size, - }; - info!( - target: "chanora_audio", - dev_sample_rate, - dev_channels, - buffer_size = ?buffer_size, - "output stream using device native config (no 48k force)" - ); - - let stream = match out_format { - SampleFormat::F32 => build_output_stream::( - &out_dev, - &out_stream_cfg, - audio_handler.clone(), - output_gain.clone(), - output_muted.clone(), - dev_sample_rate, - dev_channels, - )?, - SampleFormat::I16 => build_output_stream::( - &out_dev, - &out_stream_cfg, - audio_handler.clone(), - output_gain.clone(), - output_muted.clone(), - dev_sample_rate, - dev_channels, - )?, - SampleFormat::U16 => build_output_stream::( - &out_dev, - &out_stream_cfg, - audio_handler.clone(), - output_gain.clone(), - output_muted.clone(), - dev_sample_rate, - dev_channels, - )?, - other => { - return Err(AudioError::StreamConfig(format!( - "unsupported output format: {other:?}" - ))) - } - }; - stream - .play() - .map_err(|e| AudioError::Backend(format!("output play: {e}")))?; - stream - }; - - // ---------- Inbound forwarder ---------- - let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); - let handler_for_task = audio_handler.clone(); - let frames_received_for_task = frames_received.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = &mut shutdown_rx => { - debug!(target: "chanora_audio", "inbound forwarder shutting down"); - break; - } - item = voice_in_rx.recv() => { - match item { - Some(v) => { - let id = SessionAudioId(v.from_client); - let mut h = handler_for_task.lock().unwrap_or_else(|e| e.into_inner()); - if let Err(e) = h.handle_packet(id, v.packet) { - debug!(target: "chanora_audio", error = %e, "decode failed"); - } else { - frames_received_for_task.fetch_add(1, Ordering::Relaxed); - } - } - None => break, - } - } - } - } - }); - - Ok(Self { - transmit_gate, - frames_sent, - frames_received, - output_gain, - output_muted, - audio_processing_config, - audio_processing_stats, - silero_vad_worker, - audio_handler, - _input_stream: Mutex::new(input_stream), - _output_stream: Mutex::new(Some(output_stream)), - shutdown_tx: Some(shutdown_tx), - capture_active, - }) - } - - #[cfg(target_os = "android")] - fn start_with_gate_android( - cfg: AudioEngineConfig, - voice_out_tx: mpsc::Sender, - mut voice_in_rx: mpsc::Receiver, - transmit_gate: crate::ptt::AudioTransmitGate, - ) -> Result { - use crate::mobile_voice_backend::MobileVoiceAudioBackend; - - info!(target: "chanora_audio", "starting audio engine: Android Oboe backend"); - - let transmit_flag_for_capture = transmit_gate.flag_arc(); - let frames_sent = Arc::new(AtomicU32::new(0)); - let frames_received = Arc::new(AtomicU32::new(0)); - let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits())); - let output_muted = Arc::new(AtomicBool::new(false)); - let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default())); - let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); - - let event_queue = AudioEventQueue::new(); - let event_producer = AudioEventQueue::producer(&event_queue); - - if !cfg.mobile_voice_preset { - return Err(AudioError::Backend( - "android: mobile_voice_preset=false is unsupported for Oboe-only backend" - .to_string(), - )); - } - - let mut audio_mode_stack = crate::mode_stack::ModeStack::new(); - - if crate::android_voice_unit::chanora_android_start_voice_service() { - info!( - target: "chanora_audio", - "android: voice foreground service start dispatched (SDD-115)" - ); - } else { - warn!( - target: "chanora_audio", - "android: foreground service start failed; capture may be denied in background (SDD-115)" - ); - } - - match android_get_audio_mode() { - Ok(prior_now) => { - let outcome = audio_mode_stack.acquire(prior_now); - if let crate::mode_stack::ModeAcquire::FirstAcquire { prior } = outcome { - match android_set_audio_mode(ANDROID_MODE_IN_COMMUNICATION) { - Ok(()) => info!( - target: "chanora_audio", - prior_mode = prior, - "android: AudioManager mode set to MODE_IN_COMMUNICATION (SDD-108)" - ), - Err(e) => { - warn!( - target: "chanora_audio", - error = %e, - prior_mode = prior, - "android: setMode failed; rolling back ModeStack acquire (SDD-108 §5)" - ); - let _ = audio_mode_stack.release(); - } - } - } - } - Err(e) => warn!( - target: "chanora_audio", - error = %e, - "android: AudioManager.getMode failed; skipping mode engage (SDD-108)" - ), - } - - let cfg_av = crate::mobile_voice_backend::AndroidVoiceStreamConfig { - effects: cfg.effects, - ..Default::default() - }; - let params = crate::mobile_voice_backend::VoiceAudioParams { - voice_out_tx: voice_out_tx.clone(), - transmit_active: transmit_flag_for_capture, - frames_sent: frames_sent.clone(), - mic_gain: cfg.mic_gain, - handler: AudioHandler::new(), - event_producer: event_producer.clone(), - output_gain: output_gain.clone(), - output_muted: output_muted.clone(), - voice_activity_selector: cfg.voice_activity_selector.clone(), - audio_processing_config: audio_processing_config.clone(), - audio_processing_stats: audio_processing_stats.clone(), - }; - let mut android_voice_unit = - match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) { - Ok(unit) => unit, - Err(e) => { - Self::rollback_android_startup_resources(&mut audio_mode_stack); - return Err(AudioError::Backend(format!( - "android: failed to open Oboe voice unit: {e}" - ))); - } - }; - - if let Err(e) = android_voice_unit.start() { - if let Err(close_err) = android_voice_unit.close() { - warn!( - target: "chanora_audio", - error = %close_err, - "android: AndroidVoiceUnit::close failed during startup rollback" - ); - } - Self::rollback_android_startup_resources(&mut audio_mode_stack); - return Err(AudioError::Backend(format!( - "android: failed to start Oboe voice unit: {e}" - ))); - } - - let android_voice_unit = Arc::new(Mutex::new(Some(android_voice_unit))); - let android_voice_stream_config = Arc::new(Mutex::new(cfg_av.clone())); - let audio_mode_stack = Arc::new(Mutex::new(audio_mode_stack)); - - if let Some(event_rx) = android_voice_unit - .lock() - .unwrap() - .as_mut() - .and_then(|unit| unit.take_event_rx()) - { - Self::spawn_android_backend_event_task( - android_voice_unit.clone(), - android_voice_stream_config.clone(), - voice_out_tx.clone(), - transmit_gate.clone(), - frames_sent.clone(), - event_producer.clone(), - output_gain.clone(), - output_muted.clone(), - cfg.voice_activity_selector.clone(), - audio_processing_config.clone(), - audio_processing_stats.clone(), - cfg.mic_gain, - audio_mode_stack.clone(), - event_rx, - ); - } - - crate::android_voice_unit::register_global_event_sender( - android_voice_unit - .lock() - .unwrap() - .as_ref() - .expect("android voice unit installed") - .event_sender(), - ); - - if crate::android_voice_unit::chanora_android_request_audio_focus() { - info!( - target: "chanora_audio", - "android: audio focus requested (SDD-109)" - ); - } else { - warn!( - target: "chanora_audio", - "android: audio focus request failed; engine operates without focus (SDD-109)" - ); - } - - if crate::android_voice_unit::chanora_android_start_bluetooth_sco() { - info!( - target: "chanora_audio", - "android: Bluetooth SCO started (SDD-110)" - ); - } else { - warn!( - target: "chanora_audio", - "android: Bluetooth SCO start failed; BT HFP may not route correctly (SDD-110)" - ); - } - - let capture_active = true; - - let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); - let event_producer_for_task = event_producer.clone(); - let frames_received_for_task = frames_received.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = &mut shutdown_rx => { - debug!(target: "chanora_audio", "inbound forwarder shutting down"); - break; - } - item = voice_in_rx.recv() => { - match item { - Some(v) => { - let id = SessionAudioId(v.from_client); - let packet = AudioPacket { client_id: id, data: v.packet }; - if event_producer_for_task.push_packet(packet).is_ok() { - frames_received_for_task.fetch_add(1, Ordering::Relaxed); - } - } - None => break, - } - } - } - } - }); - - Ok(Self { - transmit_gate, - frames_sent, - frames_received, - output_gain, - output_muted, - audio_processing_config, - audio_processing_stats, - audio_event_producer: event_producer, - voice_out_tx, - voice_activity_selector: cfg.voice_activity_selector.clone(), - mic_gain: cfg.mic_gain, - _android_voice_unit: android_voice_unit, - android_voice_stream_config, - audio_mode_stack, - shutdown_tx: Some(shutdown_tx), - capture_active, - }) - } - - /// Apple implementation: a single VoiceProcessingIO AudioUnit - /// drives mic capture + speaker playback (see - /// `ios_voice_unit.rs` for why cpal is unsuitable on iOS). - /// This mirrors `start_with_gate_cpal` in scaffolding — - /// atomics, audio handler, inbound forwarder task — but - /// replaces the two cpal stream constructions with a single - /// `IosVoiceUnit::start` call. - #[cfg(any(target_os = "ios", target_os = "macos"))] - fn start_with_gate_ios( - cfg: AudioEngineConfig, - voice_out_tx: mpsc::Sender, - mut voice_in_rx: mpsc::Receiver, - transmit_gate: crate::ptt::AudioTransmitGate, - ) -> Result { - info!( - target: "chanora_audio", - "starting audio engine: Apple VoiceProcessingIO backend" - ); - - // Apple audio-session configuration is performed Swift-side - // in `apps/chanora_flutter/ios/Runner/AppDelegate.swift` - // BEFORE Flutter starts its audio pipeline. The category + - // mode pair set there (`.playAndRecord` + `.default`) is - // what VPIO binds against. Logging here just records that - // the engine-start path acknowledges the request; the - // actual session mutation lives in Swift because it must - // happen before Dart loads. - if cfg.mobile_voice_preset { - info!( - target: "chanora_audio", - "ios: voice-chat session mode requested — AVAudioSession configured in AppDelegate" - ); - } - - let transmit_flag_for_capture = transmit_gate.flag_arc(); - let frames_sent = Arc::new(AtomicU32::new(0)); - let frames_received = Arc::new(AtomicU32::new(0)); - let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits())); - let output_muted = Arc::new(AtomicBool::new(false)); - let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default())); - let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); - - #[cfg(any(target_os = "ios", target_os = "macos"))] - let audio_handler: Arc>> = - Arc::new(Mutex::new(AudioHandler::new())); - let voice_out_tx_for_backend = voice_out_tx.clone(); - - // Construct the live iOS voice backend. Platform VPIO stays - // the default shipping path; Sonora/RemoteIO remains opt-in. - let ios_voice_backend = - open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams { - handler: audio_handler.clone(), - output_gain: output_gain.clone(), - output_muted: output_muted.clone(), - voice_out_tx: voice_out_tx_for_backend, - transmit_active: transmit_flag_for_capture, - frames_sent: frames_sent.clone(), - mic_gain: cfg.mic_gain, - voice_activity_selector: cfg.voice_activity_selector.clone(), - audio_processing_config: audio_processing_config.clone(), - audio_processing_stats: audio_processing_stats.clone(), - })?; - - // Capture is always considered active on iOS — VPIO's - // input element is wired up by the AudioUnit itself, no - // separate "did the capture stream open" question to - // answer. If the user denied mic permission VPIO will - // simply hand us silence buffers. - let capture_active = true; - - // Inbound forwarder: same shape as the non-iOS path. Pumps - // Opus packets from `voice_in_rx` into AudioHandler so the - // VPIO render callback (commit 4) finds decoded frames - // waiting. - let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); - let frames_received_for_task = frames_received.clone(); - let handler_for_task = audio_handler.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = &mut shutdown_rx => { - debug!(target: "chanora_audio", "inbound forwarder shutting down"); - break; - } - item = voice_in_rx.recv() => { - match item { - Some(v) => { - let id = SessionAudioId(v.from_client); - let mut h = handler_for_task.lock().unwrap_or_else(|e| e.into_inner()); - let res = h.handle_packet(id, v.packet); - drop(h); - match res { - Ok(_) => { - frames_received_for_task.fetch_add(1, Ordering::Relaxed); - } - Err(e) => { - debug!(target: "chanora_audio", error = %e, "decode failed"); - } - } - } - None => break, - } - } - } - } - }); - - Ok(Self { - transmit_gate, - frames_sent, - frames_received, - output_gain, - output_muted, - audio_processing_config, - audio_processing_stats, - audio_handler, - _ios_voice_backend: Mutex::new(Some(ios_voice_backend)), - shutdown_tx: Some(shutdown_tx), - capture_active, - }) - } - - #[cfg(target_os = "android")] - fn rollback_android_startup_resources(audio_mode_stack: &mut crate::mode_stack::ModeStack) { - crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); - crate::android_voice_unit::chanora_android_abandon_audio_focus(); - match release_android_audio_mode_for_startup_rollback(audio_mode_stack) { - crate::mode_stack::ModeRelease::LastRelease { prior } => { - match android_set_audio_mode(prior) { - Ok(()) => info!( - target: "chanora_audio", - restored_mode = prior, - "android: AudioManager mode restored during startup rollback" - ), - Err(e) => warn!( - target: "chanora_audio", - error = %e, - restored_mode = prior, - "android: failed to restore AudioManager mode during startup rollback" - ), - } - } - crate::mode_stack::ModeRelease::StillHeld => { - info!( - target: "chanora_audio", - "android: audio mode still held during startup rollback" - ); - } - crate::mode_stack::ModeRelease::AlreadyReleased => {} - } - - if crate::android_voice_unit::chanora_android_stop_voice_service() { - info!( - target: "chanora_audio", - "android: voice foreground service stopped during startup rollback" - ); - } - } - - /// Stop the engine. Idempotent. - pub fn stop(&mut self) { - if let Some(tx) = self.shutdown_tx.take() { - let _ = tx.send(()); - } - // Drop the streams, which stops their callback threads. - // Each platform has a slightly different backend; the - // common contract is that dropping the wrapper stops - // audio. iOS collapses input + output into one - // `IosVoiceUnit` (see `ios_voice_unit.rs`); every other - // platform has separate cpal input + cpal/SDL output. - #[cfg(all( - not(target_os = "ios"), - not(target_os = "macos"), - not(target_os = "android") - ))] - { - let _ = self._input_stream.lock().unwrap_or_else(|e| e.into_inner()).take(); - let _ = self._output_stream.lock().unwrap_or_else(|e| e.into_inner()).take(); - } - #[cfg(any(target_os = "ios", target_os = "macos"))] - { - let _ = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()).take(); - } - // SDD-115 reverse-order teardown on Android: - // 1) stop Bluetooth SCO + abandon audio focus; - // 2) close the voice unit (releases SDD-113 hardware - // effects then stops + closes the Oboe streams); - // 3) restore the prior audio mode (SDD-108 §1 on 1 → 0 - // transition); - // 4) stop the foreground service. - #[cfg(target_os = "android")] - { - crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); - crate::android_voice_unit::chanora_android_abandon_audio_focus(); - crate::android_voice_unit::clear_global_event_sender(); - if let Some(mut unit) = self._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() { - use crate::mobile_voice_backend::MobileVoiceAudioBackend; - if let Err(e) = unit.close() { - warn!( - target: "chanora_audio", - error = %e, - "android: AndroidVoiceUnit::close failed (SDD-115)" - ); - } - } - // SDD-108 §1/§2: release the audio-mode stack. Only the - // 1 → 0 transition writes the platform; mid-stack - // releases stay engaged. Underflow (release without a - // matching acquire) is clamped without panic per - // SDD-108 §1. - // SDD-108 §5: tolerate a poisoned mutex on the teardown - // path — if a panicking thread held the lock, we still - // need to drive the release to completion (otherwise the - // platform stays in MODE_IN_COMMUNICATION). - let release = self - .audio_mode_stack - .lock() - .unwrap_or_else(|e| e.into_inner()) - .release(); - match release { - crate::mode_stack::ModeRelease::LastRelease { prior } => { - match android_set_audio_mode(prior) { - Ok(()) => info!( - target: "chanora_audio", - restored_mode = prior, - "android: AudioManager mode restored (SDD-108)" - ), - Err(e) => warn!( - target: "chanora_audio", - error = %e, - restored_mode = prior, - "android: failed to restore prior AudioManager mode (SDD-108)" - ), - } - } - crate::mode_stack::ModeRelease::StillHeld => { - info!( - target: "chanora_audio", - "android: audio mode still held by composed session (SDD-108)" - ); - } - crate::mode_stack::ModeRelease::AlreadyReleased => { - // No engage ever happened (e.g. mobile_voice_preset - // was false, or getMode failed). Silent no-op. - } - } - if crate::android_voice_unit::chanora_android_stop_voice_service() { - info!( - target: "chanora_audio", - "android: voice foreground service stop dispatched (SDD-115)" - ); - } - } - info!(target: "chanora_audio", "audio engine stopped"); - } - - /// iOS-only: restart the underlying VoiceProcessingIO unit after - /// route changes. - pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> { - #[cfg(any(target_os = "ios", target_os = "macos"))] - { - let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()); - let backend = guard - .as_mut() - .ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?; - backend.restart() - } - #[cfg(not(any(target_os = "ios", target_os = "macos")))] - { - Ok(()) - } - } - - /// Android-only: reopen the underlying Oboe voice backend after a - /// route/device change while preserving the engine-owned state. - pub fn android_restart_voice_unit(&self) -> Result<(), AudioError> { - #[cfg(target_os = "android")] - { - use crate::mobile_voice_backend::MobileVoiceAudioBackend; - - crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); - crate::android_voice_unit::chanora_android_abandon_audio_focus(); - crate::android_voice_unit::clear_global_event_sender(); - - if let Some(mut unit) = self._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() { - unit.close().map_err(|e| { - AudioError::Backend(format!( - "android: failed to close Oboe voice unit during route restart: {e}" - )) - })?; - } - - if crate::android_voice_unit::chanora_android_start_voice_service() { - info!( - target: "chanora_audio", - "android: voice foreground service restart dispatched after route change" - ); - } - - if crate::android_voice_unit::chanora_android_request_audio_focus() { - info!( - target: "chanora_audio", - "android: audio focus re-requested after route change" - ); - } - - let cfg_av = self.android_voice_stream_config.lock().unwrap_or_else(|e| e.into_inner()).clone(); - let params = crate::mobile_voice_backend::VoiceAudioParams { - voice_out_tx: self.voice_out_tx.clone(), - transmit_active: self.transmit_gate.flag_arc(), - frames_sent: self.frames_sent.clone(), - mic_gain: self.mic_gain, - handler: AudioHandler::new(), - event_producer: self.audio_event_producer.clone(), - output_gain: self.output_gain.clone(), - output_muted: self.output_muted.clone(), - voice_activity_selector: self.voice_activity_selector.clone(), - audio_processing_config: self.audio_processing_config.clone(), - audio_processing_stats: self.audio_processing_stats.clone(), - }; - let mut reopened = crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) - .map_err(|e| { - AudioError::Backend(format!("android: failed to reopen Oboe voice unit: {e}")) - })?; - reopened.start().map_err(|e| { - AudioError::Backend(format!("android: failed to restart Oboe voice unit: {e}")) - })?; - if let Some(event_rx) = reopened.take_event_rx() { - Self::spawn_android_backend_event_task( - self._android_voice_unit.clone(), - self.android_voice_stream_config.clone(), - self.voice_out_tx.clone(), - self.transmit_gate.clone(), - self.frames_sent.clone(), - self.audio_event_producer.clone(), - self.output_gain.clone(), - self.output_muted.clone(), - self.voice_activity_selector.clone(), - self.audio_processing_config.clone(), - self.audio_processing_stats.clone(), - self.mic_gain, - self.audio_mode_stack.clone(), - event_rx, - ); - } - crate::android_voice_unit::register_global_event_sender(reopened.event_sender()); - - if crate::android_voice_unit::chanora_android_start_bluetooth_sco() { - info!( - target: "chanora_audio", - "android: bluetooth route re-engaged after route change" - ); - } - - let mut guard = self._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()); - *guard = Some(reopened); - Ok(()) - } - #[cfg(not(target_os = "android"))] - { - Ok(()) - } - } - - /// iOS-only: pause the underlying VoiceProcessingIO unit. - pub fn ios_pause_voice_unit(&self) -> Result<(), AudioError> { - #[cfg(any(target_os = "ios", target_os = "macos"))] - { - let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()); - let unit = guard - .as_mut() - .ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?; - unit.pause() - } - #[cfg(not(any(target_os = "ios", target_os = "macos")))] - { - Ok(()) - } - } - - /// iOS-only: resume the underlying VoiceProcessingIO unit. - pub fn ios_resume_voice_unit(&self) -> Result<(), AudioError> { - #[cfg(any(target_os = "ios", target_os = "macos"))] - { - let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()); - let unit = guard - .as_mut() - .ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?; - unit.resume() - } - #[cfg(not(any(target_os = "ios", target_os = "macos")))] - { - Ok(()) - } - } - - /// Set the **transmission gate** (SRS-201). When true the - /// encoder feed is allowed to emit Opus frames; when false the - /// captured audio is discarded before encoding. This is the - /// only writer permitted on `transmit_active` (SAD-075 / - /// SDD-089). Push-to-Talk subsystems — focused PTT today, - /// per-platform global backends in a follow-up — call this - /// method exclusively. No-op when capture is inactive. - pub fn set_transmit_active(&self, active: bool) { - self.transmit_gate.set(active); - } - - /// Current transmit gate state. - pub fn transmit_active(&self) -> bool { - self.transmit_gate.load() - } - - /// Shared handle to the underlying transmit gate (SAD-075 / - /// SDD-089). Returned for diagnostics and integration tests - /// only; never mutate the underlying atomic directly — use - /// [`Self::set_transmit_active`] instead. - pub fn transmit_gate(&self) -> &crate::ptt::AudioTransmitGate { - &self.transmit_gate - } - - /// True if the capture stream opened. When false, the engine - /// runs in playback-only mode and the transmit gate is a - /// no-op (no frames will ever be encoded). - pub fn capture_active(&self) -> bool { - self.capture_active - } - - /// Number of Opus frames sent since the engine started. - pub fn frames_sent(&self) -> u32 { - self.frames_sent.load(Ordering::Relaxed) - } - - /// Number of inbound voice packets received and decoded. - pub fn frames_received(&self) -> u32 { - self.frames_received.load(Ordering::Relaxed) - } - - /// Current microphone input level in dBFS (-120.0 = silence, 0.0 = clipping). - pub fn input_level(&self) -> f32 { - self.audio_processing_stats.input_dbfs() - } - - /// Current audio-processing config snapshot. - pub fn audio_processing_config_snapshot(&self) -> crate::AudioProcessingConfig { - self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner()).clone() - } - - /// Apply a voice-processing config after validating iOS invariants. - pub fn set_audio_processing_config( - &self, - mut config: crate::AudioProcessingConfig, - ) -> Result<(), AudioError> { - #[cfg(target_os = "ios")] - config.validate_for_ios()?; - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - { - config.processing_backend = crate::AudioBackend::Noop; - } - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - self.apply_desktop_vad_backend(&config); - let mut guard = self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner()); - *guard = config; - Ok(()) - } - - /// Re-apply the current audio processing config. - /// - /// Used by the core layer to trigger VAD worker reload after a - /// model-path change (the epoch increments but the worker is only - /// reconstructed when `set_audio_processing_config` is called). - pub fn reload_audio_processing_config(&self) -> Result<(), AudioError> { - let config = self.audio_processing_config_snapshot(); - self.set_audio_processing_config(config) - } - - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - fn apply_desktop_vad_backend(&self, config: &crate::AudioProcessingConfig) { - apply_desktop_vad_backend_to_worker( - config, - &self.silero_vad_worker, - self.audio_processing_stats.as_ref(), - ); - } - - /// Current voice-processing stats snapshot. - pub fn audio_processing_stats(&self) -> crate::AudioProcessingStats { - let config = self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner()).clone(); - self.audio_processing_stats.snapshot(&config) - } - - /// Latest Android voice-audio diagnostics snapshot (SDD-112 item - /// 10 / SDD-113 item 7 / SDD-116 item 3). On non-Android targets - /// this always returns `None`. On Android it returns `Some(...)` - /// once `AndroidVoiceUnit::open()` has published a snapshot; the - /// slot is cleared on `close()` / `Drop`. Per SDD-090 the - /// snapshot contains only device-side technical scalars — no PII. - pub fn android_diagnostics( - &self, - ) -> Option { - crate::mobile_voice_backend::current_android_audio_diagnostics() - } - - /// Set master output mute. When true the output stream emits - /// silence regardless of incoming voice frames. - pub fn set_output_muted(&self, muted: bool) { - self.output_muted.store(muted, Ordering::Relaxed); - } - - /// True if the master output is currently muted locally. - pub fn output_muted(&self) -> bool { - self.output_muted.load(Ordering::Relaxed) - } - - /// Set master output gain. 1.0 is unity; 0.0 is silent. Values - /// above 1.0 amplify (and may clip downstream). Clamped to a - /// sensible range internally. - pub fn set_output_gain(&self, gain: f32) { - let clamped = gain.clamp(0.0, 4.0); - self.output_gain.store(clamped.to_bits(), Ordering::Relaxed); - } - - /// Current master output gain. - pub fn output_gain(&self) -> f32 { - f32::from_bits(self.output_gain.load(Ordering::Relaxed)) - } - - /// Set per-client output volume (SRS-075). `1.0` is unity, `0.0` - /// mutes. Values above `1.0` amplify and may clip. Clamped to - /// `0.0..4.0`. - pub fn set_client_volume(&self, client_id: u64, volume: f32) { - let clamped = volume.clamp(0.0, 4.0); - #[cfg(target_os = "android")] - { - let mut cmd = AudioCommand::SetVolume(SessionAudioId(client_id), clamped); - for _ in 0..64 { - match self.audio_event_producer.push_control(cmd) { - Ok(()) => return, - Err(returned) => { - cmd = returned; - std::thread::yield_now(); - } - } - } - tracing::warn!( - target: "chanora_audio", - client_id, - volume = clamped, - "set_client_volume: control queue full after 64 retries — volume not applied" - ); - } - #[cfg(not(target_os = "android"))] - { - match self.audio_handler.lock() { - Ok(mut h) => { - if let Some(q) = h.get_mut_queues().get_mut(&SessionAudioId(client_id)) { - q.volume = clamped; - } - } - Err(e) => { - tracing::warn!( - target: "chanora_audio", - client_id, - volume = clamped, - error = %e, - "set_client_volume: audio_handler lock poisoned — volume not applied" - ); - } - } - } - } -} - -impl Drop for AudioEngine { - fn drop(&mut self) { - self.stop(); - } -} - -#[cfg(any(test, target_os = "android"))] -fn release_android_audio_mode_for_startup_rollback( - audio_mode_stack: &mut crate::mode_stack::ModeStack, -) -> crate::mode_stack::ModeRelease { - audio_mode_stack.release() -} - -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -fn new_desktop_audio_processing_state() -> ( - Arc>, - Arc, - Arc>>, -) { - let mut config = crate::AudioProcessingConfig::default(); - // Desktop cpal capture does not use a platform voice-processing API. - // The global default (PlatformVoiceProcessing) is correct for iOS/macOS - // VPIO but would mislabel the desktop path in bridge diagnostics and - // set `platform_voice_processing_enabled = true` when no such - // processing exists. Override to Noop; the bridge/UI stats layer - // will then report the accurate backend. - config.processing_backend = crate::AudioBackend::Noop; - let audio_processing_config = Arc::new(Mutex::new(config.clone())); - let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); - let silero_vad_worker = Arc::new(Mutex::new(None)); - apply_desktop_vad_backend_to_worker( - &config, - &silero_vad_worker, - audio_processing_stats.as_ref(), - ); - ( - audio_processing_config, - audio_processing_stats, - silero_vad_worker, - ) -} - -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -fn apply_desktop_vad_backend_to_worker( - config: &crate::AudioProcessingConfig, - silero_vad_worker: &Arc>>, - audio_processing_stats: &crate::SharedAudioProcessingStats, -) { - if config.vad_backend != crate::VadBackend::SileroOnnx { - let mut worker_guard = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner()); - if worker_guard.is_some() { - info!( - target: "chanora_audio", - backend = config.vad_backend.as_str(), - "desktop: Silero ONNX VAD worker cleared because another VAD backend is selected" - ); - } - *worker_guard = None; - audio_processing_stats.set_vad_fallback_active(false); - return; - } - - // Load model and spawn worker BEFORE taking the lock so the - // realtime capture callback is not blocked on try_lock() during - // model I/O + thread spawn. The old worker (if any) is dropped - // after the new one is installed under the short lock hold. - let model_path = crate::vad::silero_model_bundle_path(); - info!( - target: "chanora_audio", - path = %model_path, - "desktop: Silero ONNX VAD selected; loading model worker" - ); - let new_worker = crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path); - { - let mut worker_guard = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner()); - *worker_guard = new_worker; - } - // Check result after releasing the lock. Re-acquire is cheap and - // ensures we log the correct state without holding the mutex. - let worker_installed = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner()).is_some(); - if worker_installed { - info!( - target: "chanora_audio", - path = %model_path, - "desktop: Silero ONNX VAD worker loaded" - ); - audio_processing_stats.set_vad_fallback_active(false); - } else { - warn!( - target: "chanora_audio", - path = %model_path, - "desktop: Silero ONNX VAD worker unavailable; WebRTC fallback will be used" - ); - audio_processing_stats.set_vad_fallback_active(true); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - #[test] - fn desktop_capture_voice_activity_opens_selector_from_speech() { - let gate = crate::ptt::AudioTransmitGate::new(false); - let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone())); - selector.set_mode(crate::TransmitMode::VoiceActivity); - selector.set_in_channel(true); - - let encoder = crate::opus_voice::new_voip_encoder("desktop VAD test").unwrap(); - let (voice_out_tx, _voice_out_rx) = mpsc::channel::(16); - let frames_sent = Arc::new(AtomicU32::new(0)); - let voice_out_tx = crate::opus_voice::start_out_packet_worker( - voice_out_tx, - frames_sent, - "desktop-vad-test", - ) - .unwrap(); - let stats = Arc::new(crate::SharedAudioProcessingStats::default()); - let mut capture = CaptureState::new( - encoder, - SAMPLE_RATE, - 1, - 1.0, - voice_out_tx, - gate.flag_arc(), - Some(selector.clone()), - Arc::new(Mutex::new(crate::AudioProcessingConfig { - vad_backend: crate::VadBackend::WebrtcVad, - ..crate::AudioProcessingConfig::default() - })), - Arc::new(Mutex::new(None)), - stats.clone(), - ); - - let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; - for (idx, sample) in voiced.iter_mut().enumerate() { - let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32; - *sample = phase.sin() * 0.4; - } - for _ in 0..6 { - capture.ingest(&voiced); - } - - assert!( - selector.voice_activity_open(), - "desktop capture must feed VAD and open VoiceActivity selector before transmit is already active" - ); - assert!( - gate.load(), - "VoiceActivity selector should publish transmit gate" - ); - let snapshot = stats.snapshot(&crate::AudioProcessingConfig::default()); - assert!(snapshot.vad_active, "stats should expose active VAD"); - } - - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - #[test] - fn desktop_startup_audio_processing_state_applies_default_silero_fallback() { - let _guard = crate::vad::SILERO_MODEL_PATH_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - crate::vad::clear_silero_model_path_for_test(); - - let (config, stats, worker) = new_desktop_audio_processing_state(); - - assert_eq!( - config.lock().unwrap_or_else(|e| e.into_inner()).vad_backend, - crate::VadBackend::SileroOnnx, - "desktop startup config should keep the default Silero backend selected" - ); - assert!( - worker.lock().unwrap_or_else(|e| e.into_inner()).is_none(), - "missing startup model should not create an ONNX worker" - ); - let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner())); - assert!( - snapshot.vad_fallback_active, - "desktop startup should mark WebRTC fallback active when the default Silero worker cannot load" - ); - crate::vad::clear_silero_model_path_for_test(); - } - - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - #[test] - fn desktop_set_audio_processing_config_normalizes_default_backend_to_noop() { - let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default())); - let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); - let engine = AudioEngine { - transmit_gate: crate::ptt::AudioTransmitGate::new(false), - frames_sent: Arc::new(AtomicU32::new(0)), - frames_received: Arc::new(AtomicU32::new(0)), - output_gain: Arc::new(AtomicU32::new(1.0_f32.to_bits())), - output_muted: Arc::new(AtomicBool::new(false)), - audio_processing_config: audio_processing_config.clone(), - audio_processing_stats, - silero_vad_worker: Arc::new(Mutex::new(None)), - audio_handler: Arc::new(Mutex::new(AudioHandler::new())), - _input_stream: Mutex::new(None), - _output_stream: Mutex::new(None), - shutdown_tx: None, - capture_active: false, - }; - - engine - .set_audio_processing_config(crate::AudioProcessingConfig::default()) - .unwrap(); - - assert_eq!( - engine.audio_processing_config_snapshot().processing_backend, - crate::AudioBackend::Noop, - "desktop setter should report the actual no-op processing backend for default configs" - ); - } - - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - #[test] - fn desktop_capture_silero_selection_reports_fallback_when_worker_unavailable() { - let _guard = crate::vad::SILERO_MODEL_PATH_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - crate::vad::clear_silero_model_path_for_test(); - - let gate = crate::ptt::AudioTransmitGate::new(false); - let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone())); - selector.set_mode(crate::TransmitMode::VoiceActivity); - selector.set_in_channel(true); - - let encoder = crate::opus_voice::new_voip_encoder("desktop Silero fallback test").unwrap(); - let (voice_out_tx, _voice_out_rx) = mpsc::channel::(16); - let frames_sent = Arc::new(AtomicU32::new(0)); - let voice_out_tx = crate::opus_voice::start_out_packet_worker( - voice_out_tx, - frames_sent, - "desktop-silero-fallback-test", - ) - .unwrap(); - let stats = Arc::new(crate::SharedAudioProcessingStats::default()); - let config = Arc::new(Mutex::new(crate::AudioProcessingConfig { - vad_backend: crate::VadBackend::SileroOnnx, - ..crate::AudioProcessingConfig::default() - })); - let mut capture = CaptureState::new( - encoder, - SAMPLE_RATE, - 1, - 1.0, - voice_out_tx, - gate.flag_arc(), - Some(selector), - config.clone(), - Arc::new(Mutex::new(None)), - stats.clone(), - ); - - let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; - for (idx, sample) in voiced.iter_mut().enumerate() { - let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32; - *sample = phase.sin() * 0.4; - } - capture.ingest(&voiced); - - let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner())); - assert_eq!(snapshot.vad_backend, crate::VadBackend::SileroOnnx); - assert!( - snapshot.vad_fallback_active, - "desktop Silero selection should make WebRTC fallback visible when ONNX worker cannot load" - ); - crate::vad::clear_silero_model_path_for_test(); - } - - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - #[test] - fn desktop_capture_vad_consumes_only_new_pcm_while_transmitting() { - let gate = crate::ptt::AudioTransmitGate::new(true); - let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone())); - selector.set_mode(crate::TransmitMode::VoiceActivity); - selector.set_in_channel(true); - - let encoder = crate::opus_voice::new_voip_encoder("desktop VAD duplicate test").unwrap(); - let (voice_out_tx, _voice_out_rx) = mpsc::channel::(16); - let frames_sent = Arc::new(AtomicU32::new(0)); - let voice_out_tx = crate::opus_voice::start_out_packet_worker( - voice_out_tx, - frames_sent, - "desktop-vad-duplicate-test", - ) - .unwrap(); - let stats = Arc::new(crate::SharedAudioProcessingStats::default()); - let mut capture = CaptureState::new( - encoder, - SAMPLE_RATE, - 1, - 1.0, - voice_out_tx, - gate.flag_arc(), - Some(selector), - Arc::new(Mutex::new(crate::AudioProcessingConfig { - vad_backend: crate::VadBackend::Disabled, - ..crate::AudioProcessingConfig::default() - })), - Arc::new(Mutex::new(None)), - stats, - ); - - capture.pcm_accum.extend(std::iter::repeat_n( - 0.1_f32, - crate::frame::FRAME_10MS_SAMPLES + 240, - )); - capture.pending_10ms[..240].fill(0.1); - capture.pending_10ms_len = 240; - capture.capture_frame_seq = 1; - capture.pcm_accum.extend(std::iter::repeat_n(0.2_f32, 240)); - - capture.process_pending_vad_frames(crate::frame::FRAME_10MS_SAMPLES + 240); - - assert_eq!( - capture.capture_frame_seq, 2, - "VAD should consume only the 240 samples appended by the current ingest and complete one pending 10 ms frame" - ); - } - - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] - #[test] - fn desktop_capture_silero_stale_enqueued_worker_uses_fallback() { - let gate = crate::ptt::AudioTransmitGate::new(false); - let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone())); - selector.set_mode(crate::TransmitMode::VoiceActivity); - selector.set_in_channel(true); - - let encoder = - crate::opus_voice::new_voip_encoder("desktop stale Silero fallback test").unwrap(); - let (voice_out_tx, _voice_out_rx) = mpsc::channel::(16); - let frames_sent = Arc::new(AtomicU32::new(0)); - let voice_out_tx = crate::opus_voice::start_out_packet_worker( - voice_out_tx, - frames_sent, - "desktop-stale-silero-fallback-test", - ) - .unwrap(); - let stats = Arc::new(crate::SharedAudioProcessingStats::default()); - let config = Arc::new(Mutex::new(crate::AudioProcessingConfig { - vad_backend: crate::VadBackend::SileroOnnx, - ..crate::AudioProcessingConfig::default() - })); - let worker = Arc::new(Mutex::new(Some( - crate::vad::silero_onnx::SileroOnnxVadWorker::stale_test_worker(), - ))); - let mut capture = CaptureState::new( - encoder, - SAMPLE_RATE, - 1, - 1.0, - voice_out_tx, - gate.flag_arc(), - Some(selector), - config.clone(), - worker, - stats.clone(), - ); - - let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; - for (idx, sample) in voiced.iter_mut().enumerate() { - let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32; - *sample = phase.sin() * 0.4; - } - - capture.process_10ms_capture_frame(&voiced); - - let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner())); - assert!( - snapshot.vad_fallback_active, - "stale Silero worker output should report active WebRTC fallback" - ); - assert!( - snapshot.vad_probability > 0.5, - "stale Silero worker output should use WebRTC fallback probability instead of forced silence" - ); - } - - #[test] - fn android_startup_rollback_releases_acquired_mode_snapshot() { - let mut stack = crate::mode_stack::ModeStack::new(); - let _ = stack.acquire(7); - - let release = release_android_audio_mode_for_startup_rollback(&mut stack); - - assert_eq!( - release, - crate::mode_stack::ModeRelease::LastRelease { prior: 7 } - ); - assert_eq!(stack.refcount(), 0); - assert_eq!(stack.snapshot(), None); - } -} - -// ---------- Capture pipeline ---------- - -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -fn try_open_capture( - in_dev: &cpal::Device, - voice_out_tx: mpsc::Sender, - transmit_active: Arc, - frames_sent: Arc, - mic_gain: f32, - voice_activity_selector: Option>, - audio_processing_config: Arc>, - silero_vad_worker: Arc>>, - audio_processing_stats: Arc, -) -> Result { - let in_cfg = in_dev - .default_input_config() - .map_err(|e| AudioError::StreamConfig(format!("input default: {e}")))?; - let in_sample_rate = in_cfg.sample_rate(); - let in_channels = in_cfg.channels() as usize; - let in_format = in_cfg.sample_format(); - // Buffer-size rationale (same shape as the output path): - // * Windows: pin to 2048 frames to avoid the small-period - // jitter of WASAPI shared mode. - // * macOS / iOS: CoreAudio picks a HAL-friendly default; - // iOS RemoteIO rejects arbitrary buffer-size requests. - // * Linux: same SDL2-vs-cpal split as the output path; we - // still use cpal for capture but leave Default since - // PipeWire's ALSA shim works well there. - let mut in_stream_cfg: cpal::StreamConfig = in_cfg.into(); - #[cfg(target_os = "windows")] - { - in_stream_cfg.buffer_size = cpal::BufferSize::Fixed(2048); - } - #[cfg(not(target_os = "windows"))] - { - in_stream_cfg.buffer_size = cpal::BufferSize::Default; - } - - let opus_enc = crate::opus_voice::new_voip_encoder("cpal capture")?; - - let capture_state = Arc::new(Mutex::new(CaptureState::new( - opus_enc, - in_sample_rate, - in_channels, - mic_gain, - crate::opus_voice::start_out_packet_worker( - voice_out_tx, - frames_sent.clone(), - "cpal-capture", - )?, - transmit_active, - voice_activity_selector, - audio_processing_config, - silero_vad_worker, - audio_processing_stats, - ))); - - let stream = match in_format { - SampleFormat::F32 => build_input_stream::(in_dev, &in_stream_cfg, capture_state)?, - SampleFormat::I16 => build_input_stream::(in_dev, &in_stream_cfg, capture_state)?, - SampleFormat::U16 => build_input_stream::(in_dev, &in_stream_cfg, capture_state)?, - other => { - return Err(AudioError::StreamConfig(format!( - "unsupported input format: {other:?}" - ))) - } - }; - Ok(stream) -} - -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -struct CaptureState { - encoder: OpusEncoder, - in_sample_rate: u32, - in_channels: usize, - mic_gain: f32, - /// 48 kHz mono buffer accumulated to FRAME_SAMPLES before each encode. - pcm_accum: Vec, - /// Resampling state for non-48k sources (simple linear resampler). - resample_pos: f64, - /// Last input sample carried over from the previous cpal callback - /// so the resampler can interpolate across the buffer boundary - /// without dropping continuity. Without this, every cpal period - /// boundary produces a discontinuity → audible buzz / popping - /// roughly at the period rate (~100 Hz for a 10 ms period on - /// Linux ALSA defaults). - resample_last: f32, - opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME], - voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender, - /// The PTT transmission gate. Read once per outbound frame; the - /// CaptureState never mutates this flag. - transmit_active: Arc, - voice_activity_selector: Option>, - vad_detector: crate::vad::WebRtcFallbackVad, - silero_vad_worker: Arc>>, - silero_model_epoch: u64, - current_vad_backend: crate::VadBackend, - fallback_warned_backend: Option, - capture_frame_seq: u64, - vad_state: crate::voice_activity::VoiceActivityStateMachine, - audio_processing_config: Arc>, - pending_10ms: [f32; crate::frame::FRAME_10MS_SAMPLES], - pending_10ms_len: usize, - /// Pre-allocated mono downmix buffer. Resized in-place each - /// callback; `clear()` retains capacity. SDD-094 realtime-thread - /// invariant: this avoids the heap allocation that the prior fix - /// at engine.rs:1389-1397 (output-side `scratch` buffer) - /// addressed; the capture side has the same pattern and the same - /// user-perceptible cost on glibc malloc when the callback runs - /// on a realtime SCHED_FIFO thread. - mono_scratch: Vec, - /// Pre-allocated Opus-frame buffer. Sized to FRAME_SAMPLES (960) - /// on construction; reused across frames. `clear()` retains - /// capacity so the drain-into-frame path skips the allocator - /// after warmup. Same precedent as `mono_scratch` above. - frame_scratch: Vec, - audio_processing_stats: Arc, - /// Time-based decimation for the level meter. The cpal callback - /// cadence depends on platform (256 frames at 48 kHz ≈ 187 Hz, - /// 512 frames ≈ 94 Hz, 1024 frames ≈ 47 Hz) and can change at - /// runtime on device or sample-rate switch. The bridge consumer - /// (`input_level_stream`) only reads at ~30 Hz, so computing - /// `sqrt()` + `log10()` on every callback wastes real-time budget - /// and caused buffer underruns on macOS CoreAudio with small - /// buffer sizes. We only emit a new dBFS sample after at least - /// [LEVEL_METER_INTERVAL] has elapsed since the previous emit, - /// which is platform-cadence-independent. - last_level_emit: std::time::Instant, -} - -/// Minimum interval between input-level dBFS samples sent to the -/// bridge. Matches the consumer rate (`input_level_stream` at ~30 Hz). -/// Time-based gating is robust to cpal buffer-size and sample-rate -/// changes that a fixed callback-count would not be. -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -const LEVEL_METER_INTERVAL: std::time::Duration = std::time::Duration::from_millis(33); - -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -impl CaptureState { - fn new( - encoder: OpusEncoder, - in_sample_rate: u32, - in_channels: usize, - mic_gain: f32, - voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender, - transmit_active: Arc, - voice_activity_selector: Option>, - audio_processing_config: Arc>, - silero_vad_worker: Arc>>, - audio_processing_stats: Arc, - ) -> Self { - Self { - encoder, - in_sample_rate, - in_channels, - mic_gain, - pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2), - resample_pos: 0.0, - resample_last: 0.0, - opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME], - voice_out_tx, - transmit_active, - voice_activity_selector, - vad_detector: crate::vad::WebRtcFallbackVad::default(), - silero_vad_worker, - silero_model_epoch: crate::vad::silero_model_epoch(), - current_vad_backend: crate::VadBackend::Disabled, - fallback_warned_backend: None, - capture_frame_seq: 0, - vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), - audio_processing_config, - pending_10ms: [0.0; crate::frame::FRAME_10MS_SAMPLES], - pending_10ms_len: 0, - mono_scratch: Vec::with_capacity(4096), - frame_scratch: Vec::with_capacity(FRAME_SAMPLES), - audio_processing_stats, - // Start in the past so the first ingest emits immediately. - last_level_emit: std::time::Instant::now() - .checked_sub(LEVEL_METER_INTERVAL) - .unwrap_or_else(std::time::Instant::now), - } - } - - /// Consume an arbitrary-rate, multichannel cpal buffer; produce - /// 48 kHz mono frames; encode and send when `transmit_active` - /// is true (PTT engaged). - // TRACKED(realtime-audio): This method runs on the cpal audio callback - // thread with a ~10 ms deadline. Known pre-existing violations of the - // realtime safety constraint (see TODO-006 in master-todo-list): - // 1. Blocking Mutex::lock().unwrap() on self (via cpal callback) - // 2. Potential allocation in mono_scratch.reserve() and - // pcm_accum.extend_from_slice() when buffer capacity is exceeded - // 3. Encode-path warn!/error! logging via send_voip_frame closures - fn ingest(&mut self, buf: &[T]) { - // 1. Down-mix to mono (pre-gain). Always performed so the level - // meter reflects real mic input even when PTT is released. - let in_channels = self.in_channels; - let mic_gain = self.mic_gain; - self.mono_scratch.clear(); - let frame_count = buf.len() / in_channels.max(1); - self.mono_scratch.reserve(frame_count); - for frame in buf.chunks(in_channels) { - let sum: f32 = frame.iter().map(|s| s.to_f32_sample()).sum(); - self.mono_scratch.push(sum / frame.len() as f32); - } - - // Level meter: pay sqrt() + log10() only when at least - // LEVEL_METER_INTERVAL has elapsed, regardless of the - // platform's cpal callback cadence. - let now = std::time::Instant::now(); - if now.duration_since(self.last_level_emit) >= LEVEL_METER_INTERVAL { - self.last_level_emit = now; - self.audio_processing_stats - .set_input_dbfs(crate::frame::dbfs(&self.mono_scratch)); - } - - if mic_gain != 1.0 { - for s in &mut self.mono_scratch { - *s *= mic_gain; - } - } - - // 2. Resample to 48 kHz if needed. We re-borrow - // `mono_scratch` as a shared slice per branch to satisfy - // the borrow checker against `&mut self` on the - // resample path. - let vad_start_offset = self.pcm_accum.len(); - if self.in_sample_rate == SAMPLE_RATE { - // Disjoint-borrow: copy the slice into pcm_accum without - // aliasing &mut self. - let (src, dst) = (&self.mono_scratch, &mut self.pcm_accum); - dst.extend_from_slice(src); - } else { - // resample_into_accum reads `mono` and writes - // `self.pcm_accum` / `self.resample_*`. These fields are - // disjoint from `self.mono_scratch`, but the function - // signature takes `&mut self`, so we take ownership of - // the scratch buffer briefly via `std::mem::take`, run - // the resampler, then move the buffer back so its - // capacity is preserved across callbacks. - let mono = std::mem::take(&mut self.mono_scratch); - self.resample_into_accum(&mono); - self.mono_scratch = mono; - } - - self.process_pending_vad_frames(vad_start_offset); - - if !self.transmit_active.load(Ordering::Relaxed) { - self.pcm_accum.clear(); - return; - } - - // 3. Encode any complete frames. Clamp each sample to - // [-1.0, 1.0] before handing to libopus's float encoder — - // out-of-range samples are hard-clipped inside libopus, - // which produces audible distortion on transient peaks. - // Soft-clamping at the engine boundary preserves headroom - // and matches what every other VoIP client does. - while self.pcm_accum.len() >= FRAME_SAMPLES { - // Reuse `self.frame_scratch` to avoid a per-frame Vec - // allocation on the realtime audio thread; same - // rationale as the engine.rs:1389-1397 precedent. - let frame = &mut self.frame_scratch; - frame.clear(); - frame.extend(self.pcm_accum.drain(..FRAME_SAMPLES)); - for s in frame.iter_mut() { - if *s > 1.0 { - *s = 1.0; - } else if *s < -1.0 { - *s = -1.0; - } - } - match self - .encoder - .encode_float(&frame[..], &mut self.opus_out[..]) - { - Ok(len) => { - crate::opus_voice::send_voip_frame( - &self.voice_out_tx, - &self.opus_out, - len, - || { - warn!( - target: "chanora_audio", - "voice_out queue full; dropping frame" - ); - }, - || { - warn!(target: "chanora_audio", "voice_out closed; stopping send"); - }, - ); - } - Err(e) => { - error!(target: "chanora_audio", error = %e, "opus encode failed"); - } - } - } - } - - fn process_pending_vad_frames(&mut self, start_offset: usize) { - let mut offset = start_offset.min(self.pcm_accum.len()); - while offset < self.pcm_accum.len() { - let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len; - let take = remaining.min(self.pcm_accum.len() - offset); - self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take] - .copy_from_slice(&self.pcm_accum[offset..offset + take]); - self.pending_10ms_len += take; - offset += take; - - if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES { - let frame = self.pending_10ms; - self.process_10ms_capture_frame(&frame); - self.pending_10ms_len = 0; - } - } - } - - fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) { - // Realtime callback: do not log here. Publish state via atomics - // and let a non-realtime consumer translate transitions into - // info/warn events. The transmit/diagnostic stats stream - // already exposes vad_fallback_active for this purpose. - self.fallback_warned_backend = Some(failed_backend); - } - - fn sync_vad_backend(&mut self, voice_activity_mode: bool, vad_backend: crate::VadBackend) { - if !voice_activity_mode { - self.current_vad_backend = crate::VadBackend::Disabled; - self.fallback_warned_backend = None; - self.audio_processing_stats.set_vad_fallback_active(false); - return; - } - - let silero_epoch = crate::vad::silero_model_epoch(); - let silero_changed = - vad_backend == crate::VadBackend::SileroOnnx && silero_epoch != self.silero_model_epoch; - if vad_backend == self.current_vad_backend && !silero_changed { - return; - } - - self.current_vad_backend = vad_backend; - self.silero_model_epoch = silero_epoch; - self.fallback_warned_backend = None; - self.vad_state.reset(); - - match vad_backend { - crate::VadBackend::SileroOnnx => { - let worker_available = self - .silero_vad_worker - .try_lock() - .map(|worker| worker.is_some()) - .unwrap_or(false); - if worker_available { - self.audio_processing_stats.set_vad_fallback_active(false); - } else { - self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); - self.audio_processing_stats.set_vad_fallback_active(true); - } - } - crate::VadBackend::WebrtcVad => { - self.audio_processing_stats.set_vad_fallback_active(false); - } - crate::VadBackend::EnergyDebug => { - self.audio_processing_stats.set_vad_fallback_active(true); - } - crate::VadBackend::Disabled => { - self.audio_processing_stats.set_vad_fallback_active(false); - } - } - } - - fn process_10ms_capture_frame(&mut self, frame: &[f32; crate::frame::FRAME_10MS_SAMPLES]) { - let input_dbfs = crate::frame::dbfs(frame); - let (vad_backend, vad_hangover) = self - .audio_processing_config - .try_lock() - .map(|cfg| (cfg.vad_backend, cfg.vad_hangover_ms)) - .unwrap_or(( - crate::VadBackend::WebrtcVad, - crate::voice_activity::VAD_HANGOVER_MS, - )); - let voice_activity_mode = self - .voice_activity_selector - .as_ref() - .map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity) - .unwrap_or(false); - - if voice_activity_mode { - self.sync_vad_backend(true, vad_backend); - self.vad_state.configure( - crate::voice_activity::VAD_OPEN_AFTER_MS, - vad_hangover, - crate::voice_activity::VAD_MIN_TX_MS, - ); - } else { - self.sync_vad_backend(false, vad_backend); - } - - let (vad_probability, gate_open, used_fallback_vad) = if voice_activity_mode { - self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1); - let capture_seq = self.capture_frame_seq; - let mut used_fallback_vad = false; - let vad = match vad_backend { - crate::VadBackend::Disabled => crate::vad::VadOutput { - probability: 1.0, - speech: true, - }, - crate::VadBackend::SileroOnnx => { - // Single `try_lock` that both probes availability and - // sends the frame. The guard is scoped to the block - // so it drops before the fallback path (which needs - // `&mut self` for `mark_vad_fallback_active`). - // Returns Some(VadOutput) on a successful, non-stale - // send; None means "fall back to WebRTC VAD". - let worker_output = { - let guard = self.silero_vad_worker.try_lock().ok(); - guard.and_then(|guard| { - let worker = guard.as_ref()?; - if worker.try_send(capture_seq, frame) && !worker.is_stale(capture_seq) - { - let p = worker.latest_probability(); - Some(crate::vad::VadOutput { - probability: p, - speech: p >= 0.5, - }) - } else { - None - } - }) - }; - if let Some(output) = worker_output { - output - } else { - used_fallback_vad = true; - self.mark_vad_fallback_active(vad_backend); - crate::vad::VoiceActivityDetector::process_10ms( - &mut self.vad_detector, - frame, - ) - } - } - crate::VadBackend::WebrtcVad | crate::VadBackend::EnergyDebug => { - used_fallback_vad = vad_backend == crate::VadBackend::EnergyDebug; - crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, frame) - } - }; - ( - vad.probability, - self.vad_state.update(vad.speech), - used_fallback_vad, - ) - } else { - (0.0, false, false) - }; - self.audio_processing_stats - .set_vad_fallback_active(used_fallback_vad); - - let vad_active = voice_activity_mode && gate_open; - if let Some(selector) = &self.voice_activity_selector { - selector.set_voice_activity_open(vad_active); - } - self.audio_processing_stats.update_capture( - input_dbfs, - input_dbfs, - vad_probability, - vad_active, - self.transmit_active.load(Ordering::Relaxed), - ); - self.audio_processing_stats - .record_capture_frame(frame.iter().all(|sample| sample.abs() <= 0.000_001)); - } - - /// Simple linear resampler for `in_sample_rate → 48000`. - /// - /// The resampler maintains continuity across cpal buffer - /// boundaries by treating `self.resample_last` as a virtual - /// sample at fractional index `0.0`, followed by the incoming - /// `mono` slice at indices `1.0..=mono.len()`. Without the - /// virtual anchor sample, the first interpolation point of - /// every new buffer collapses to `mono[0]` for both `a` and - /// `b`, producing a sample-and-hold step at every cpal period - /// boundary. On Linux ALSA defaults that's a ~100 Hz buzz / - /// popping. Production-quality work would use a windowed sinc - /// kernel; this carries the previous sample only and keeps the - /// CPU cost trivial. - fn resample_into_accum(&mut self, mono: &[f32]) { - if mono.is_empty() { - return; - } - let ratio = self.in_sample_rate as f64 / SAMPLE_RATE as f64; - let mut pos = self.resample_pos; - // The virtual buffer has length mono.len() + 1: index 0 is - // the carried-over last sample, indices 1..=mono.len() are - // the new buffer. We emit output samples while `pos` is - // strictly less than mono.len() so we always have a valid - // right-hand neighbour. The leftover sub-sample offset is - // carried over via `resample_pos` (rebased to the next - // buffer's virtual index 0 below). - while pos < mono.len() as f64 { - let i = pos.floor() as isize; - let frac = pos - i as f64; - let a = if i <= 0 { - self.resample_last - } else { - mono[(i - 1) as usize] - }; - let b = if i < mono.len() as isize { - mono[i as usize] - } else { - // Should not happen given the while-condition, but - // guard for the boundary where ratio < 1.0 and `pos` - // can step past mono.len() in the last iteration. - a - }; - self.pcm_accum - .push((a as f64 + frac * (b - a) as f64) as f32); - pos += ratio; - } - // Carry the leftover sub-sample offset, rebased so the next - // buffer's virtual index 0 is the new `resample_last`. - self.resample_pos = pos - mono.len() as f64; - // Anchor for the next buffer's interpolation. - self.resample_last = *mono.last().unwrap(); - } -} - -/// Per-sample format conversion to f32 in the range [-1.0, 1.0]. -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -trait ToF32 { - fn to_f32_sample(self) -> f32; -} -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -impl ToF32 for f32 { - fn to_f32_sample(self) -> f32 { - self - } -} -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -impl ToF32 for i16 { - fn to_f32_sample(self) -> f32 { - f32::from(self) / f32::from(i16::MAX) - } -} -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -impl ToF32 for u16 { - fn to_f32_sample(self) -> f32 { - (f32::from(self) - f32::from(i16::MAX) - 1.0) / f32::from(i16::MAX) - } -} - -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -fn build_input_stream( - device: &cpal::Device, - config: &cpal::StreamConfig, - state: Arc>, -) -> Result -where - T: SizedSample + ToF32 + Send + 'static, -{ - let stream = device - .build_input_stream( - *config, - move |data: &[T], _: &cpal::InputCallbackInfo| { - let mut s = state.lock().unwrap_or_else(|e| e.into_inner()); - s.ingest(data); - }, - move |e| { - error!(target: "chanora_audio", error = %e, "input stream error"); - }, - None, - ) - .map_err(|e| AudioError::Backend(format!("build_input_stream: {e}")))?; - Ok(stream) -} - -// ---------- Playback pipeline ---------- - -#[cfg(not(target_os = "linux"))] -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -fn build_output_stream( - device: &cpal::Device, - config: &cpal::StreamConfig, - handler: Arc>>, - output_gain: Arc, - output_muted: Arc, - dev_sample_rate: u32, - dev_channels: usize, -) -> Result -where - T: SizedSample + FromF32 + Send + 'static, -{ - // Per-channel resampler state shared across cpal callbacks for - // continuity at buffer boundaries. AudioHandler produces 48 kHz - // stereo f32; we map the first two device channels to L/R and - // fill any extra channels with silence. `pos` carries the - // fractional source-sample offset; `last_l` / `last_r` are the - // anchor samples from the previous callback (avoid the sample- - // and-hold step that would otherwise pop at every period - // boundary — same trick as the capture-side resampler). - let resample_ratio = SAMPLE_RATE as f64 / dev_sample_rate as f64; - // `same_rate` is the common case where the device is already at - // 48 kHz — bypass the resampler entirely. - let same_rate = dev_sample_rate == SAMPLE_RATE; - let resample_state: Arc> = - Arc::new(Mutex::new(PlaybackResampleState { - pos: 0.0, - last_l: 0.0, - last_r: 0.0, - })); - // Reusable scratch buffer for 48 kHz stereo samples coming out - // of AudioHandler. Allocating a fresh `Vec` per cpal callback on - // glibc malloc was costing measurable time on the realtime audio - // thread and contributing to the popping users heard. We resize - // the buffer to the per-callback need and only grow the - // backing allocation when it must — typical buffer-size jitter - // stays under the high-water mark and skips the allocator - // entirely after the first few callbacks. - let mut scratch: Vec = Vec::with_capacity(8192); - // Diagnostic: log a warning if the cpal callback wall-clock - // exceeds the period budget so we can correlate user-perceived - // popping with measurable underruns. The threshold is half a - // period at 48 kHz / 2 ch / 1024-frame typical period — about - // 10 ms. We rate-limit the warning to once per second. - let mut last_slow_warn = std::time::Instant::now() - .checked_sub(std::time::Duration::from_secs(2)) - .unwrap_or_else(std::time::Instant::now); - let stream = device - .build_output_stream( - *config, - move |out: &mut [T], _: &cpal::OutputCallbackInfo| { - let cb_start = std::time::Instant::now(); - let muted = output_muted.load(Ordering::Relaxed); - let dev_frames = out.len() / dev_channels.max(1); - let src_frames = if same_rate { - dev_frames - } else { - // Ask for a few extra source frames so we never - // starve on the resample fractional boundary. - ((dev_frames as f64 * resample_ratio).ceil() as usize) + 2 - }; - let needed = src_frames * 2; - if scratch.len() < needed { - scratch.resize(needed, 0.0); - } - // Zero the live slice; AudioHandler::fill_buffer - // writes silence into untouched samples, but - // resizing up from a smaller call leaves residual - // values from earlier callbacks. Use `fill` which - // optimises to memset on f32. - scratch[..needed].fill(0.0); - // Lock-and-decode. We deliberately hold the lock - // only for the duration of fill_buffer; the inbound - // forwarder uses handle_packet which is queue-fast. - { - let mut h = handler.lock().unwrap_or_else(|e| e.into_inner()); - h.fill_buffer(&mut scratch[..needed]); - } - - if muted { - for dst in out.iter_mut() { - *dst = T::from_f32_sample(0.0); - } - } else { - let gain = f32::from_bits(output_gain.load(Ordering::Relaxed)); - if same_rate && dev_channels == 2 { - // Fast path: device is already 48 kHz stereo. - for (dst, s) in out.iter_mut().zip(scratch[..needed].iter().copied()) { - *dst = T::from_f32_sample(s * gain); - } - } else { - // Resample 48 kHz stereo → device-rate × - // device-channels with continuity across - // callback boundaries. - let mut state = resample_state.lock().unwrap_or_else(|e| e.into_inner()); - let mut pos = state.pos; - let mut last_l = state.last_l; - let mut last_r = state.last_r; - for frame_idx in 0..dev_frames { - let i = pos.floor() as isize; - let frac = pos - i as f64; - let (a_l, a_r) = if i <= 0 { - (last_l, last_r) - } else { - let idx = ((i - 1) as usize) * 2; - (scratch[idx], scratch[idx + 1]) - }; - let i_usize = i.max(0) as usize; - let (b_l, b_r) = if i_usize < src_frames { - let idx = i_usize * 2; - (scratch[idx], scratch[idx + 1]) - } else { - (a_l, a_r) - }; - let l = (a_l as f64 + frac * (b_l - a_l) as f64) as f32 * gain; - let r = (a_r as f64 + frac * (b_r - a_r) as f64) as f32 * gain; - let base = frame_idx * dev_channels; - if dev_channels == 1 { - // Mono output device (typical on iOS - // .voiceChat / phone-call audio path): - // downmix L+R to a single channel - // rather than dropping the R side. - // Without the downmix, anything panned - // right in the AudioHandler stereo mix - // is silently lost \u2014 on speakerphone - // this manifested as quiet remote - // speakers being inaudible. - out[base] = T::from_f32_sample((l + r) * 0.5); - } else { - out[base] = T::from_f32_sample(l); - if dev_channels >= 2 { - out[base + 1] = T::from_f32_sample(r); - } - for c in 2..dev_channels { - out[base + c] = T::from_f32_sample(0.0); - } - } - pos += resample_ratio; - } - let consumed = pos.floor() as usize; - state.pos = pos - consumed as f64; - if consumed > 0 && consumed <= src_frames { - let idx = (consumed - 1) * 2; - last_l = scratch[idx]; - last_r = scratch[idx + 1]; - state.last_l = last_l; - state.last_r = last_r; - } - } - } - - // Period-budget diagnostic. - let elapsed = cb_start.elapsed(); - let period_us = (dev_frames as u64 * 1_000_000) / dev_sample_rate as u64; - if elapsed.as_micros() as u64 > period_us / 2 - && last_slow_warn.elapsed() > std::time::Duration::from_secs(1) - { - last_slow_warn = std::time::Instant::now(); - warn!( - target: "chanora_audio", - callback_us = elapsed.as_micros() as u64, - period_us, - dev_frames, - "output callback exceeded half the period budget — possible underrun cause" - ); - } - }, - move |e| { - error!(target: "chanora_audio", error = %e, "output stream error"); - }, - None, - ) - .map_err(|e| { - error!( - target: "chanora_audio", - error = %e, - requested_channels = config.channels, - requested_sample_rate = config.sample_rate, - "build_output_stream FAILED" - ); - AudioError::Backend(format!("build_output_stream: {e}")) - })?; - Ok(stream) -} - -/// Resampler state carried across output cpal callbacks. See -/// `build_output_stream` for the rationale. -#[cfg(not(target_os = "linux"))] -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -struct PlaybackResampleState { - pos: f64, - last_l: f32, - last_r: f32, -} - -#[cfg(not(target_os = "linux"))] -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -trait FromF32 { - fn from_f32_sample(v: f32) -> Self; -} -#[cfg(not(target_os = "linux"))] -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -impl FromF32 for f32 { - fn from_f32_sample(v: f32) -> Self { - v - } -} -#[cfg(not(target_os = "linux"))] -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -impl FromF32 for i16 { - fn from_f32_sample(v: f32) -> Self { - (v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i16 - } -} -#[cfg(not(target_os = "linux"))] -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -impl FromF32 for u16 { - fn from_f32_sample(v: f32) -> Self { - let s = (v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i32; - (s + i32::from(i16::MAX) + 1) as u16 - } -} - -// ---------- Android voice-communication routing ---------- -// -// SDD-108 §1: `AudioManager.setMode(MODE_IN_COMMUNICATION)` engagement -// is the routing-level lever that tells Android "this is a voice -// call, please use the earpiece / engage hardware AEC / NS / AGC -// where the device supports it". The helpers here are the -// platform-write surface invoked by the engine's `ModeStack` -// (`crate::mode_stack`) — the stack owns refcount + snapshot -// semantics (SDD-108 §2), these helpers only perform the platform -// write / read. -// -// Placement rationale (SDD-108 §4): the JNI helpers for -// `AudioManager.getMode` / `AudioManager.setMode` live in -// `chanora_audio::engine` (not `chanora_bridge::android_init`) per -// SDD-108 §4, which assigns the AudioManager JNI surface to the Rust -// audio engine. This keeps the AudioManager interaction co-located -// with the engine state (`ModeStack`) that owns it, so the -// snapshot/restore lifecycle and the JNI calls evolve together. -// -// `AudioManager.MODE_IN_COMMUNICATION == 3` per the Android SDK. -#[cfg(target_os = "android")] -pub const ANDROID_MODE_IN_COMMUNICATION: i32 = 3; - -/// SDD-108 §5: typed error for the Android `AudioManager` JNI surface. -/// -/// Replaces the previous ad-hoc `Result<_, String>` so callers can -/// pattern-match on the failure category (attach vs. method call vs. -/// other) and log/route accordingly. `Display` renders a stable -/// human-readable form that is safe to feed into the existing -/// `tracing::warn!(error = %e, ...)` sites. -#[cfg(target_os = "android")] -#[derive(Debug, Clone)] -pub enum AudioModeError { - /// `JavaVM::from_raw` or `attach_current_thread` failed: the - /// engine could not reach the JVM at all. - JniAttachFailed(String), - /// A specific JNI method call failed (e.g. `getMode`, `setMode`, - /// `getSystemService`). `method` is a static string for grep-ability. - MethodCallFailed { - method: &'static str, - detail: String, - }, - /// Catch-all for non-JNI-method failures (null context, panic in - /// the JNI body, etc.). - Other(String), -} - -#[cfg(target_os = "android")] -impl std::fmt::Display for AudioModeError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::JniAttachFailed(d) => write!(f, "JNI attach failed: {d}"), - Self::MethodCallFailed { method, detail } => { - write!(f, "JNI method `{method}` failed: {detail}") - } - Self::Other(d) => write!(f, "{d}"), - } - } -} - -#[cfg(target_os = "android")] -impl std::error::Error for AudioModeError {} - -#[cfg(target_os = "android")] -impl From for AudioModeError { - fn from(value: jni::errors::Error) -> Self { - Self::JniAttachFailed(value.to_string()) - } -} - -/// JNI helper shared by `android_get_audio_mode` and -/// `android_set_audio_mode`: attach to the current thread and return -/// the `AudioManager` jobject. Centralised so SDD-108's two platform -/// entry points share one bootstrapping path. -#[cfg(target_os = "android")] -fn android_audio_manager_call(op: F) -> Result -where - F: for<'local> FnOnce( - &mut jni::Env<'local>, - &jni::objects::JObject<'local>, - ) -> Result - + std::panic::UnwindSafe, -{ - use jni::objects::{JObject, JString, JValue}; - - let result = std::panic::catch_unwind(|| -> Result { - let ctx = ndk_context::android_context(); - let vm_ptr = ctx.vm(); - if vm_ptr.is_null() { - return Err(AudioModeError::JniAttachFailed( - "ndk_context vm is null".to_string(), - )); - } - // SAFETY: ndk_context::android_context guarantees `vm` points - // at a live JavaVM* set by our bridge_init JNI hook. The - // unsafe block contains only the cast required by - // `JavaVM::from_raw`. - let jvm = unsafe { jni::JavaVM::from_raw(vm_ptr as *mut _) }; - jvm.attach_current_thread(|env| -> Result { - let context_obj = unsafe { JObject::from_raw(env, ctx.context() as jni::sys::jobject) }; - - let service_name: JString = - env.new_string("audio") - .map_err(|e| AudioModeError::MethodCallFailed { - method: "new_string", - detail: e.to_string(), - })?; - let service_name_obj = JObject::from(service_name); - let audio_manager = env - .call_method( - &context_obj, - jni::jni_str!("getSystemService"), - jni::jni_sig!("(Ljava/lang/String;)Ljava/lang/Object;"), - &[JValue::Object(&service_name_obj)], - ) - .map_err(|e| AudioModeError::MethodCallFailed { - method: "getSystemService", - detail: e.to_string(), - })? - .l() - .map_err(|e| AudioModeError::MethodCallFailed { - method: "getSystemService", - detail: format!("obj cast: {e}"), - })?; - if audio_manager.is_null() { - return Err(AudioModeError::Other( - "AudioManager service is null".to_string(), - )); - } - op(env, &audio_manager) - }) - }); - match result { - Ok(inner) => inner, - Err(_) => Err(AudioModeError::Other( - "panic in JNI audio_manager call".to_string(), - )), - } -} - -/// SDD-108 §1 platform-read: `AudioManager.getMode()`. -/// -/// Returns the integer mode constant currently active on the system. -/// Called by the engine on first acquire (0 → 1 transition) so that -/// `ModeStack` can snapshot the prior mode for restoration on last -/// release. -#[cfg(target_os = "android")] -pub fn android_get_audio_mode() -> Result { - android_audio_manager_call(|env, audio_manager| { - env.call_method( - audio_manager, - jni::jni_str!("getMode"), - jni::jni_sig!("()I"), - &[], - ) - .map_err(|e| AudioModeError::MethodCallFailed { - method: "getMode", - detail: e.to_string(), - })? - .i() - .map_err(|e| AudioModeError::MethodCallFailed { - method: "getMode", - detail: format!("int cast: {e}"), - }) - }) -} - -/// SDD-108 §1 platform-write: `AudioManager.setMode(mode)`. -/// -/// `mode` is the Android `AudioManager.MODE_*` integer constant. -/// Use [`ANDROID_MODE_IN_COMMUNICATION`] for engagement; pass back -/// the snapshotted prior mode (from -/// [`crate::mode_stack::ModeRelease::LastRelease`]) for restoration. -#[cfg(target_os = "android")] -pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> { - use jni::objects::JValue; - android_audio_manager_call(move |env, audio_manager| { - env.call_method( - audio_manager, - jni::jni_str!("setMode"), - jni::jni_sig!("(I)V"), - &[JValue::Int(mode)], - ) - .map_err(|e| AudioModeError::MethodCallFailed { - method: "setMode", - detail: e.to_string(), - })?; - Ok(()) - }) -} - -// --------------------------------------------------------------------------- -// SDD-120 §3 bench seam. -// -// Exposes a minimal factory for `CaptureState` plus a thin `ingest_f32` -// shim so the criterion bench harness in `crates/chanora_audio/benches/` -// can drive the same realtime capture code path the production cpal -// callback uses, without re-implementing CaptureState in the bench file. -// Marked `#[doc(hidden)]` so the public API surface is unaffected; this -// is not a supported external API. Only compiled on cpal-capture targets -// because Apple and Android use native voice backends instead. -// --------------------------------------------------------------------------- -#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] -#[doc(hidden)] -pub mod bench_seam { - use super::{Arc, AtomicBool, AtomicU32, CaptureState, OutPacket}; - use tokio::sync::mpsc; - - /// Opaque handle wrapping a CaptureState plus the dummy mpsc - /// receiver that prevents the channel sender from erroring out - /// when the bench drives `ingest`. The receiver is held inside - /// the handle so it lives for the bench's lifetime. - pub struct CaptureBenchHandle { - state: CaptureState, - // Keep the receiver alive so sends from CaptureState::encode_and_send - // do not fail; the bench discards what would be transmitted. - _rx: mpsc::Receiver, - transmit_active: Arc, - } - - impl CaptureBenchHandle { - /// Construct a CaptureState wired to a private mpsc + a - /// pre-asserted transmit-active flag so `ingest` exercises - /// the full down-mix → resample → encode → send pipeline. - /// - /// `in_sample_rate` selects the input rate (48000 for the - /// passthrough path, 44100 / 16000 to exercise the linear - /// resampler). `in_channels` selects the channel layout - /// (typically 1 or 2). - pub fn new(in_sample_rate: u32, in_channels: usize) -> Self { - let encoder = - crate::opus_voice::new_voip_encoder("cpal bench").expect("opus encoder init"); - let (tx, rx) = mpsc::channel::(64); - let transmit_active = Arc::new(AtomicBool::new(true)); - let frames_sent = Arc::new(AtomicU32::new(0)); - // Bridge the bench's private mpsc to the - // realtime-thread-safe EncodedVoiceFrameSender that - // CaptureState expects. The worker task forwards - // encoded Opus frames to `tx` via `frames_sent`. - let voice_out_tx = - crate::opus_voice::start_out_packet_worker(tx, frames_sent, "cpal-bench") - .expect("start_out_packet_worker"); - let state = CaptureState::new( - encoder, - in_sample_rate, - in_channels, - 1.0, - voice_out_tx, - transmit_active.clone(), - None, - Arc::new(std::sync::Mutex::new( - crate::AudioProcessingConfig::default(), - )), - Arc::new(std::sync::Mutex::new(None)), - Arc::new(crate::SharedAudioProcessingStats::default()), - ); - Self { - state, - _rx: rx, - transmit_active, - } - } - - /// Drive one cpal-callback-equivalent buffer through the - /// realtime capture pipeline. - #[inline] - pub fn ingest_f32(&mut self, buf: &[f32]) { - self.state.ingest(buf); - } - - /// Set the PTT gate. Defaults to true (bench measures the - /// transmitting path). - pub fn set_transmit_active(&self, active: bool) { - self.transmit_active - .store(active, std::sync::atomic::Ordering::Relaxed); - } - } -} diff --git a/crates/chanora_audio/src/engine/capture.rs b/crates/chanora_audio/src/engine/capture.rs new file mode 100644 index 0000000..4942508 --- /dev/null +++ b/crates/chanora_audio/src/engine/capture.rs @@ -0,0 +1,515 @@ +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; + +use cpal::traits::StreamTrait; +use cpal::{SampleFormat, SizedSample}; +use tracing::{debug, error, warn, info}; + +use chanora_protocol::OutPacket; + +use audiopus::coder::Encoder as OpusEncoder; + +use crate::AudioError; + +use super::SAMPLE_RATE; +use super::FRAME_SAMPLES; + +const LEVEL_METER_INTERVAL: std::time::Duration = std::time::Duration::from_millis(33); + +pub(super) fn try_open_capture( + in_dev: &cpal::Device, + voice_out_tx: tokio::sync::mpsc::Sender, + transmit_active: Arc, + frames_sent: Arc, + mic_gain: f32, + voice_activity_selector: Option>, + audio_processing_config: Arc>, + silero_vad_worker: Arc>>, + audio_processing_stats: Arc, +) -> Result { + let in_cfg = in_dev + .default_input_config() + .map_err(|e| AudioError::StreamConfig(format!("input default: {e}")))?; + let in_sample_rate = in_cfg.sample_rate(); + let in_channels = in_cfg.channels() as usize; + let in_format = in_cfg.sample_format(); + let mut in_stream_cfg: cpal::StreamConfig = in_cfg.into(); + #[cfg(target_os = "windows")] + { + in_stream_cfg.buffer_size = cpal::BufferSize::Fixed(2048); + } + #[cfg(not(target_os = "windows"))] + { + in_stream_cfg.buffer_size = cpal::BufferSize::Default; + } + + let opus_enc = crate::opus_voice::new_voip_encoder("cpal capture")?; + + let capture_state = Arc::new(Mutex::new(CaptureState::new( + opus_enc, + in_sample_rate, + in_channels, + mic_gain, + crate::opus_voice::start_out_packet_worker( + voice_out_tx, + frames_sent.clone(), + "cpal-capture", + )?, + transmit_active, + voice_activity_selector, + audio_processing_config, + silero_vad_worker, + audio_processing_stats, + ))); + + let stream = match in_format { + SampleFormat::F32 => build_input_stream::(in_dev, &in_stream_cfg, capture_state)?, + SampleFormat::I16 => build_input_stream::(in_dev, &in_stream_cfg, capture_state)?, + SampleFormat::U16 => build_input_stream::(in_dev, &in_stream_cfg, capture_state)?, + other => { + return Err(AudioError::StreamConfig(format!( + "unsupported input format: {other:?}" + ))) + } + }; + Ok(stream) +} + +pub(super) struct CaptureState { + encoder: OpusEncoder, + pub(super) pcm_accum: Vec, + pub(super) pending_10ms: [f32; crate::frame::FRAME_10MS_SAMPLES], + pub(super) pending_10ms_len: usize, + pub(super) capture_frame_seq: u64, + in_sample_rate: u32, + in_channels: usize, + mic_gain: f32, + resample_pos: f64, + resample_last: f32, + opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME], + voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender, + transmit_active: Arc, + voice_activity_selector: Option>, + vad_detector: crate::vad::WebRtcFallbackVad, + silero_vad_worker: Arc>>, + silero_model_epoch: u64, + current_vad_backend: crate::VadBackend, + fallback_warned_backend: Option, + vad_state: crate::voice_activity::VoiceActivityStateMachine, + audio_processing_config: Arc>, + mono_scratch: Vec, + frame_scratch: Vec, + audio_processing_stats: Arc, + last_level_emit: std::time::Instant, +} + +impl CaptureState { + pub(super) fn new( + encoder: OpusEncoder, + in_sample_rate: u32, + in_channels: usize, + mic_gain: f32, + voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender, + transmit_active: Arc, + voice_activity_selector: Option>, + audio_processing_config: Arc>, + silero_vad_worker: Arc>>, + audio_processing_stats: Arc, + ) -> Self { + Self { + encoder, + in_sample_rate, + in_channels, + mic_gain, + pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2), + resample_pos: 0.0, + resample_last: 0.0, + opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME], + voice_out_tx, + transmit_active, + voice_activity_selector, + vad_detector: crate::vad::WebRtcFallbackVad::default(), + silero_vad_worker, + silero_model_epoch: crate::vad::silero_model_epoch(), + current_vad_backend: crate::VadBackend::Disabled, + fallback_warned_backend: None, + capture_frame_seq: 0, + vad_state: crate::voice_activity::VoiceActivityStateMachine::default(), + audio_processing_config, + pending_10ms: [0.0; crate::frame::FRAME_10MS_SAMPLES], + pending_10ms_len: 0, + mono_scratch: Vec::with_capacity(4096), + frame_scratch: Vec::with_capacity(FRAME_SAMPLES), + audio_processing_stats, + last_level_emit: std::time::Instant::now() + .checked_sub(LEVEL_METER_INTERVAL) + .unwrap_or_else(std::time::Instant::now), + } + } + + pub(super) fn ingest(&mut self, buf: &[T]) { + let in_channels = self.in_channels; + let mic_gain = self.mic_gain; + self.mono_scratch.clear(); + let frame_count = buf.len() / in_channels.max(1); + self.mono_scratch.reserve(frame_count); + for frame in buf.chunks(in_channels) { + let sum: f32 = frame.iter().map(|s| s.to_f32_sample()).sum(); + self.mono_scratch.push(sum / frame.len() as f32); + } + + let now = std::time::Instant::now(); + if now.duration_since(self.last_level_emit) >= LEVEL_METER_INTERVAL { + self.last_level_emit = now; + self.audio_processing_stats + .set_input_dbfs(crate::frame::dbfs(&self.mono_scratch)); + } + + if mic_gain != 1.0 { + for s in &mut self.mono_scratch { + *s *= mic_gain; + } + } + + let vad_start_offset = self.pcm_accum.len(); + if self.in_sample_rate == SAMPLE_RATE { + let (src, dst) = (&self.mono_scratch, &mut self.pcm_accum); + dst.extend_from_slice(src); + } else { + let mono = std::mem::take(&mut self.mono_scratch); + self.resample_into_accum(&mono); + self.mono_scratch = mono; + } + + self.process_pending_vad_frames(vad_start_offset); + + if !self.transmit_active.load(Ordering::Relaxed) { + self.pcm_accum.clear(); + return; + } + + while self.pcm_accum.len() >= FRAME_SAMPLES { + let frame = &mut self.frame_scratch; + frame.clear(); + frame.extend(self.pcm_accum.drain(..FRAME_SAMPLES)); + for s in frame.iter_mut() { + if *s > 1.0 { + *s = 1.0; + } else if *s < -1.0 { + *s = -1.0; + } + } + match self + .encoder + .encode_float(&frame[..], &mut self.opus_out[..]) + { + Ok(len) => { + crate::opus_voice::send_voip_frame( + &self.voice_out_tx, + &self.opus_out, + len, + || { + warn!( + target: "chanora_audio", + "voice_out queue full; dropping frame" + ); + }, + || { + warn!(target: "chanora_audio", "voice_out closed; stopping send"); + }, + ); + } + Err(e) => { + error!(target: "chanora_audio", error = %e, "opus encode failed"); + } + } + } + } + + pub(super) fn process_pending_vad_frames(&mut self, start_offset: usize) { + let mut offset = start_offset.min(self.pcm_accum.len()); + while offset < self.pcm_accum.len() { + let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len; + let take = remaining.min(self.pcm_accum.len() - offset); + self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take] + .copy_from_slice(&self.pcm_accum[offset..offset + take]); + self.pending_10ms_len += take; + offset += take; + + if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES { + let frame = self.pending_10ms; + self.process_10ms_capture_frame(&frame); + self.pending_10ms_len = 0; + } + } + } + + fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) { + self.fallback_warned_backend = Some(failed_backend); + } + + fn sync_vad_backend(&mut self, voice_activity_mode: bool, vad_backend: crate::VadBackend) { + if !voice_activity_mode { + self.current_vad_backend = crate::VadBackend::Disabled; + self.fallback_warned_backend = None; + self.audio_processing_stats.set_vad_fallback_active(false); + return; + } + + let silero_epoch = crate::vad::silero_model_epoch(); + let silero_changed = + vad_backend == crate::VadBackend::SileroOnnx && silero_epoch != self.silero_model_epoch; + if vad_backend == self.current_vad_backend && !silero_changed { + return; + } + + self.current_vad_backend = vad_backend; + self.silero_model_epoch = silero_epoch; + self.fallback_warned_backend = None; + self.vad_state.reset(); + + match vad_backend { + crate::VadBackend::SileroOnnx => { + let worker_available = self + .silero_vad_worker + .try_lock() + .map(|worker| worker.is_some()) + .unwrap_or(false); + if worker_available { + self.audio_processing_stats.set_vad_fallback_active(false); + } else { + self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx); + self.audio_processing_stats.set_vad_fallback_active(true); + } + } + crate::VadBackend::WebrtcVad => { + self.audio_processing_stats.set_vad_fallback_active(false); + } + crate::VadBackend::EnergyDebug => { + self.audio_processing_stats.set_vad_fallback_active(true); + } + crate::VadBackend::Disabled => { + self.audio_processing_stats.set_vad_fallback_active(false); + } + } + } + + pub(super) fn process_10ms_capture_frame(&mut self, frame: &[f32; crate::frame::FRAME_10MS_SAMPLES]) { + let input_dbfs = crate::frame::dbfs(frame); + let (vad_backend, vad_hangover) = self + .audio_processing_config + .try_lock() + .map(|cfg| (cfg.vad_backend, cfg.vad_hangover_ms)) + .unwrap_or(( + crate::VadBackend::WebrtcVad, + crate::voice_activity::VAD_HANGOVER_MS, + )); + let voice_activity_mode = self + .voice_activity_selector + .as_ref() + .map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity) + .unwrap_or(false); + + if voice_activity_mode { + self.sync_vad_backend(true, vad_backend); + self.vad_state.configure( + crate::voice_activity::VAD_OPEN_AFTER_MS, + vad_hangover, + crate::voice_activity::VAD_MIN_TX_MS, + ); + } else { + self.sync_vad_backend(false, vad_backend); + } + + let (vad_probability, gate_open, used_fallback_vad) = if voice_activity_mode { + self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1); + let capture_seq = self.capture_frame_seq; + let mut used_fallback_vad = false; + let vad = match vad_backend { + crate::VadBackend::Disabled => crate::vad::VadOutput { + probability: 1.0, + speech: true, + }, + crate::VadBackend::SileroOnnx => { + let worker_output = { + let guard = self.silero_vad_worker.try_lock().ok(); + guard.and_then(|guard| { + let worker = guard.as_ref()?; + if worker.try_send(capture_seq, frame) && !worker.is_stale(capture_seq) + { + let p = worker.latest_probability(); + Some(crate::vad::VadOutput { + probability: p, + speech: p >= 0.5, + }) + } else { + None + } + }) + }; + if let Some(output) = worker_output { + output + } else { + used_fallback_vad = true; + self.mark_vad_fallback_active(vad_backend); + crate::vad::VoiceActivityDetector::process_10ms( + &mut self.vad_detector, + frame, + ) + } + } + crate::VadBackend::WebrtcVad | crate::VadBackend::EnergyDebug => { + used_fallback_vad = vad_backend == crate::VadBackend::EnergyDebug; + crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, frame) + } + }; + ( + vad.probability, + self.vad_state.update(vad.speech), + used_fallback_vad, + ) + } else { + (0.0, false, false) + }; + self.audio_processing_stats + .set_vad_fallback_active(used_fallback_vad); + + let vad_active = voice_activity_mode && gate_open; + if let Some(selector) = &self.voice_activity_selector { + selector.set_voice_activity_open(vad_active); + } + self.audio_processing_stats.update_capture( + input_dbfs, + input_dbfs, + vad_probability, + vad_active, + self.transmit_active.load(Ordering::Relaxed), + ); + self.audio_processing_stats + .record_capture_frame(frame.iter().all(|sample| sample.abs() <= 0.000_001)); + } + + fn resample_into_accum(&mut self, mono: &[f32]) { + if mono.is_empty() { + return; + } + let ratio = self.in_sample_rate as f64 / SAMPLE_RATE as f64; + let mut pos = self.resample_pos; + while pos < mono.len() as f64 { + let i = pos.floor() as isize; + let frac = pos - i as f64; + let a = if i <= 0 { + self.resample_last + } else { + mono[(i - 1) as usize] + }; + let b = if i < mono.len() as isize { + mono[i as usize] + } else { + a + }; + self.pcm_accum + .push((a as f64 + frac * (b - a) as f64) as f32); + pos += ratio; + } + self.resample_pos = pos - mono.len() as f64; + self.resample_last = *mono.last().unwrap(); + } +} + +trait ToF32 { + fn to_f32_sample(self) -> f32; +} +impl ToF32 for f32 { + fn to_f32_sample(self) -> f32 { + self + } +} +impl ToF32 for i16 { + fn to_f32_sample(self) -> f32 { + f32::from(self) / f32::from(i16::MAX) + } +} +impl ToF32 for u16 { + fn to_f32_sample(self) -> f32 { + (f32::from(self) - f32::from(i16::MAX) - 1.0) / f32::from(i16::MAX) + } +} + +fn build_input_stream( + device: &cpal::Device, + config: &cpal::StreamConfig, + state: Arc>, +) -> Result +where + T: SizedSample + ToF32 + Send + 'static, +{ + let stream = device + .build_input_stream( + *config, + move |data: &[T], _: &cpal::InputCallbackInfo| { + let mut s = state.lock().unwrap_or_else(|e| e.into_inner()); + s.ingest(data); + }, + move |e| { + error!(target: "chanora_audio", error = %e, "input stream error"); + }, + None, + ) + .map_err(|e| AudioError::Backend(format!("build_input_stream: {e}")))?; + Ok(stream) +} + +#[doc(hidden)] +pub mod bench_seam { + use super::{Arc, AtomicBool, AtomicU32, CaptureState, OutPacket}; + use tokio::sync::mpsc; + + pub struct CaptureBenchHandle { + pub(crate) state: CaptureState, + _rx: mpsc::Receiver, + transmit_active: Arc, + } + + impl CaptureBenchHandle { + pub fn new(in_sample_rate: u32, in_channels: usize) -> Self { + let encoder = + crate::opus_voice::new_voip_encoder("cpal bench").expect("opus encoder init"); + let (tx, rx) = mpsc::channel::(64); + let transmit_active = Arc::new(AtomicBool::new(true)); + let frames_sent = Arc::new(AtomicU32::new(0)); + let voice_out_tx = + crate::opus_voice::start_out_packet_worker(tx, frames_sent, "cpal-bench") + .expect("start_out_packet_worker"); + let state = CaptureState::new( + encoder, + in_sample_rate, + in_channels, + 1.0, + voice_out_tx, + transmit_active.clone(), + None, + Arc::new(std::sync::Mutex::new( + crate::AudioProcessingConfig::default(), + )), + Arc::new(std::sync::Mutex::new(None)), + Arc::new(crate::SharedAudioProcessingStats::default()), + ); + Self { + state, + _rx: rx, + transmit_active, + } + } + + #[inline] + pub fn ingest_f32(&mut self, buf: &[f32]) { + self.state.ingest(buf); + } + + pub fn set_transmit_active(&self, active: bool) { + self.transmit_active + .store(active, std::sync::atomic::Ordering::Relaxed); + } + } +} diff --git a/crates/chanora_audio/src/engine/lifecycle.rs b/crates/chanora_audio/src/engine/lifecycle.rs new file mode 100644 index 0000000..cf8f552 --- /dev/null +++ b/crates/chanora_audio/src/engine/lifecycle.rs @@ -0,0 +1,1149 @@ +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; + +use tokio::sync::mpsc; +use tracing::{debug, info}; + +#[cfg(any(target_os = "android", target_os = "ios"))] +use tracing::warn; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android"), +))] +use tracing::warn; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android"), +))] +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android"), +))] +use cpal::SampleFormat; + +use tsclientlib::audio::AudioHandler; + +use chanora_protocol::{InboundVoice, OutPacket}; + +#[cfg(target_os = "android")] +use crate::audio_event_queue::{AudioEventQueue, AudioPacket}; + +use crate::AudioError; + +use super::{AudioEngine, AudioEngineConfig, SessionAudioId}; + +#[cfg(any(target_os = "ios", target_os = "macos"))] +pub(super) enum IosVoiceBackend { + Vpio(crate::ios_voice_unit::IosVoiceUnit), +} + +#[cfg(any(target_os = "ios", target_os = "macos"))] +impl IosVoiceBackend { + pub(super) fn restart(&mut self) -> Result<(), AudioError> { + #[cfg(target_os = "ios")] + { + match self { + Self::Vpio(unit) => unit.restart(), + } + } + #[cfg(target_os = "macos")] + { + let _ = self; + Ok(()) + } + } + + pub(super) fn pause(&mut self) -> Result<(), AudioError> { + #[cfg(target_os = "ios")] + { + match self { + Self::Vpio(unit) => unit.pause(), + } + } + #[cfg(target_os = "macos")] + { + let _ = self; + Ok(()) + } + } + + pub(super) fn resume(&mut self) -> Result<(), AudioError> { + #[cfg(target_os = "ios")] + { + match self { + Self::Vpio(unit) => unit.resume(), + } + } + #[cfg(target_os = "macos")] + { + let _ = self; + Ok(()) + } + } +} + +#[cfg(any(target_os = "ios", target_os = "macos"))] +fn open_ios_voice_backend( + params: crate::mobile_voice_backend::VoiceAudioParams, +) -> Result { + let unit = crate::ios_voice_unit::IosVoiceUnit::start(params)?; + Ok(IosVoiceBackend::Vpio(unit)) +} + +pub(super) fn stop_audio_engine(engine: &mut AudioEngine) { + if let Some(tx) = engine.shutdown_tx.take() { + let _ = tx.send(()); + } + #[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") + ))] + { + let _ = engine._input_stream.lock().unwrap_or_else(|e| e.into_inner()).take(); + let _ = engine._output_stream.lock().unwrap_or_else(|e| e.into_inner()).take(); + } + #[cfg(any(target_os = "ios", target_os = "macos"))] + { + let _ = engine._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()).take(); + } + #[cfg(target_os = "android")] + { + crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); + crate::android_voice_unit::chanora_android_abandon_audio_focus(); + crate::android_voice_unit::clear_global_event_sender(); + if let Some(mut unit) = engine._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() { + use crate::mobile_voice_backend::MobileVoiceAudioBackend; + if let Err(e) = unit.close() { + warn!( + target: "chanora_audio", + error = %e, + "android: AndroidVoiceUnit::close failed (SDD-115)" + ); + } + } + let release = engine + .audio_mode_stack + .lock() + .unwrap_or_else(|e| e.into_inner()) + .release(); + match release { + crate::mode_stack::ModeRelease::LastRelease { prior } => { + match super::android_set_audio_mode(prior) { + Ok(()) => info!( + target: "chanora_audio", + restored_mode = prior, + "android: AudioManager mode restored (SDD-108)" + ), + Err(e) => warn!( + target: "chanora_audio", + error = %e, + restored_mode = prior, + "android: failed to restore prior AudioManager mode (SDD-108)" + ), + } + } + crate::mode_stack::ModeRelease::StillHeld => { + info!( + target: "chanora_audio", + "android: audio mode still held by composed session (SDD-108)" + ); + } + crate::mode_stack::ModeRelease::AlreadyReleased => {} + } + if crate::android_voice_unit::chanora_android_stop_voice_service() { + info!( + target: "chanora_audio", + "android: voice foreground service stop dispatched (SDD-115)" + ); + } + } + info!(target: "chanora_audio", "audio engine stopped"); +} + +impl AudioEngine { + #[cfg(target_os = "android")] + fn spawn_android_backend_event_task( + android_voice_unit: Arc>>, + android_voice_stream_config: Arc< + Mutex, + >, + voice_out_tx: mpsc::Sender, + transmit_gate: crate::ptt::AudioTransmitGate, + frames_sent: Arc, + event_producer: crate::audio_event_queue::AudioEventProducer, + output_gain: Arc, + output_muted: Arc, + voice_activity_selector: Option>, + audio_processing_config: Arc>, + audio_processing_stats: Arc, + mic_gain: f32, + audio_mode_stack: Arc>, + mut event_rx: crate::mobile_voice_backend::BackendEventRx, + ) { + use crate::mobile_voice_backend::BackendEvent; + + tokio::spawn(async move { + while let Some(event) = event_rx.recv().await { + match event { + BackendEvent::Disconnected => { + warn!(target: "chanora_audio", "android: backend disconnected; reopening voice unit"); + crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); + crate::android_voice_unit::chanora_android_abandon_audio_focus(); + crate::android_voice_unit::clear_global_event_sender(); + + if let Some(mut unit) = android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() { + use crate::mobile_voice_backend::MobileVoiceAudioBackend; + if let Err(e) = unit.close() { + warn!( + target: "chanora_audio", + error = %e, + "android: failed to close disconnected voice unit before reopen" + ); + } + } + + if crate::android_voice_unit::chanora_android_start_voice_service() { + info!( + target: "chanora_audio", + "android: voice foreground service restart dispatched after backend disconnect" + ); + } + + match super::android_get_audio_mode() { + Ok(current_mode) => { + if current_mode != super::ANDROID_MODE_IN_COMMUNICATION { + match super::android_set_audio_mode(super::ANDROID_MODE_IN_COMMUNICATION) { + Ok(()) => info!( + target: "chanora_audio", + prior_mode = current_mode, + "android: AudioManager mode re-engaged after backend disconnect" + ), + Err(e) => warn!( + target: "chanora_audio", + error = %e, + prior_mode = current_mode, + "android: failed to re-engage MODE_IN_COMMUNICATION after backend disconnect" + ), + } + } + } + Err(e) => warn!( + target: "chanora_audio", + error = %e, + "android: AudioManager.getMode failed during backend reopen" + ), + } + + let cfg_av = android_voice_stream_config.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let params = crate::mobile_voice_backend::VoiceAudioParams { + voice_out_tx: voice_out_tx.clone(), + transmit_active: transmit_gate.flag_arc(), + frames_sent: frames_sent.clone(), + mic_gain, + handler: AudioHandler::new(), + event_producer: event_producer.clone(), + output_gain: output_gain.clone(), + output_muted: output_muted.clone(), + voice_activity_selector: voice_activity_selector.clone(), + audio_processing_config: audio_processing_config.clone(), + audio_processing_stats: audio_processing_stats.clone(), + }; + + match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) { + Ok(mut reopened) => { + use crate::mobile_voice_backend::MobileVoiceAudioBackend; + match reopened.start() { + Ok(()) => { + if let Some(next_rx) = reopened.take_event_rx() { + Self::spawn_android_backend_event_task( + android_voice_unit.clone(), + android_voice_stream_config.clone(), + voice_out_tx.clone(), + transmit_gate.clone(), + frames_sent.clone(), + event_producer.clone(), + output_gain.clone(), + output_muted.clone(), + voice_activity_selector.clone(), + audio_processing_config.clone(), + audio_processing_stats.clone(), + mic_gain, + audio_mode_stack.clone(), + next_rx, + ); + } + crate::android_voice_unit::register_global_event_sender( + reopened.event_sender(), + ); + if crate::android_voice_unit::chanora_android_request_audio_focus() { + info!( + target: "chanora_audio", + "android: audio focus re-requested after backend disconnect" + ); + } + if crate::android_voice_unit::chanora_android_start_bluetooth_sco() { + info!( + target: "chanora_audio", + "android: bluetooth route re-engaged after backend disconnect" + ); + } + *android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()) = Some(reopened); + } + Err(e) => warn!( + target: "chanora_audio", + error = %e, + "android: reopened voice unit failed to start after backend disconnect" + ), + } + } + Err(e) => warn!( + target: "chanora_audio", + error = %e, + "android: failed to reopen voice unit after backend disconnect" + ), + } + } + BackendEvent::FocusLost => { + warn!( + target: "chanora_audio", + "android: audio focus lost permanently (SDD-109); engine should leave session" + ); + } + BackendEvent::FocusTransient => { + info!( + target: "chanora_audio", + "android: transient audio focus loss (SDD-109); pausing capture" + ); + } + BackendEvent::FocusTransientCanDuck => { + info!( + target: "chanora_audio", + "android: transient audio focus loss with ducking (SDD-109); continuing" + ); + } + BackendEvent::FocusGain => { + info!( + target: "chanora_audio", + "android: audio focus regained (SDD-109); resuming capture" + ); + } + BackendEvent::BluetoothScoStateChanged(s) => { + info!( + target: "chanora_audio", + sco_state = s, + "android: Bluetooth SCO state changed (SDD-110)" + ); + } + } + } + }); + } + + /// Start the engine: open capture + playback streams, spawn the + /// inbound-voice forwarder, return a handle. + pub fn start( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + voice_in_rx: mpsc::Receiver, + ) -> Result { + let gate = crate::ptt::AudioTransmitGate::new(cfg.ptt_initial); + Self::start_with_gate(cfg, voice_out_tx, voice_in_rx, gate) + } + + /// Start the engine using an externally-owned + /// [`AudioTransmitGate`]. + pub fn start_with_gate( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + Self::start_with_gate_platform(cfg, voice_out_tx, voice_in_rx, transmit_gate) + } + + #[cfg(any(target_os = "ios", target_os = "macos"))] + fn start_with_gate_platform( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + Self::start_with_gate_ios(cfg, voice_out_tx, voice_in_rx, transmit_gate) + } + + #[cfg(target_os = "android")] + fn start_with_gate_platform( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + Self::start_with_gate_android(cfg, voice_out_tx, voice_in_rx, transmit_gate) + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + fn start_with_gate_platform( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate) + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + fn start_with_gate_cpal( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + mut voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + let host = cpal::default_host(); + info!( + target: "chanora_audio", + host_id = ?host.id(), + "starting audio engine: cpal host selected" + ); + + fn find_device( + host: &cpal::Host, + default_fn: DefaultFn, + all_fn: AllFn, + prefer: Option<&str>, + ) -> Option + where + DefaultFn: Fn(&cpal::Host) -> Option, + AllFn: Fn(&cpal::Host) -> Result, + Devices: IntoIterator, + { + if let Some(id) = prefer { + if let Ok(devices) = all_fn(host) { + for d in devices { + if super::desktop_device_id(&d).as_deref() == Some(id) { + return Some(d); + } + } + } + } + default_fn(host) + } + + let in_dev = find_device( + &host, + cpal::Host::default_input_device, + cpal::Host::input_devices, + cfg.input_device_id.as_deref(), + ) + .ok_or(AudioError::NoInputDevice)?; + + let out_dev = find_device( + &host, + cpal::Host::default_output_device, + cpal::Host::output_devices, + cfg.output_device_id.as_deref(), + ) + .ok_or(AudioError::NoOutputDevice)?; + + info!( + target: "chanora_audio", + in_device = %in_dev.description().map(|d| d.name().to_owned()).unwrap_or_default(), + out_device = %out_dev.description().map(|d| d.name().to_owned()).unwrap_or_default(), + "starting audio engine" + ); + + match in_dev.default_input_config() { + Ok(c) => info!( + target: "chanora_audio", + channels = c.channels(), + sample_rate = c.sample_rate(), + sample_format = ?c.sample_format(), + "default_input_config reported" + ), + Err(e) => { + super::warn_desktop!( + target: "chanora_audio", + error = %e, + "default_input_config FAILED — capture will be disabled" + ); + } + } + match out_dev.default_output_config() { + Ok(c) => info!( + target: "chanora_audio", + channels = c.channels(), + sample_rate = c.sample_rate(), + sample_format = ?c.sample_format(), + "default_output_config reported" + ), + Err(e) => { + super::warn_desktop!( + target: "chanora_audio", + error = %e, + "default_output_config FAILED — output stream will fail to build" + ); + } + } + + let transmit_flag_for_capture = transmit_gate.flag_arc(); + let frames_sent = Arc::new(AtomicU32::new(0)); + let frames_received = Arc::new(AtomicU32::new(0)); + let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits())); + let output_muted = Arc::new(AtomicBool::new(false)); + let (audio_processing_config, audio_processing_stats, silero_vad_worker) = + super::new_desktop_audio_processing_state(); + + let capture_result = super::try_open_capture( + &in_dev, + voice_out_tx, + transmit_flag_for_capture, + frames_sent.clone(), + cfg.mic_gain, + cfg.voice_activity_selector.clone(), + audio_processing_config.clone(), + silero_vad_worker.clone(), + audio_processing_stats.clone(), + ); + let (input_stream, capture_active) = match capture_result { + Ok(s) => (Some(s), true), + Err(e) => { + super::warn_desktop!( + target: "chanora_audio", + error = %e, + "capture stream unavailable; continuing with playback only" + ); + (None, false) + } + }; + if let Some(s) = &input_stream { + s.play() + .map_err(|e| AudioError::Backend(format!("input play: {e}")))?; + } + + let audio_handler: Arc>> = + Arc::new(Mutex::new(AudioHandler::new())); + + #[cfg(target_os = "linux")] + let output_stream = crate::sdl_output::SdlOutput::start( + audio_handler.clone(), + output_gain.clone(), + output_muted.clone(), + )?; + + #[cfg(not(target_os = "linux"))] + let output_stream = { + let out_cfg = out_dev + .default_output_config() + .map_err(|e| AudioError::StreamConfig(format!("output default: {e}")))?; + let out_format = out_cfg.sample_format(); + let dev_sample_rate = out_cfg.sample_rate(); + let dev_channels = out_cfg.channels() as usize; + #[cfg(target_os = "windows")] + let buffer_size = cpal::BufferSize::Fixed(2048); + #[cfg(not(target_os = "windows"))] + let buffer_size = cpal::BufferSize::Default; + let out_stream_cfg = cpal::StreamConfig { + channels: out_cfg.channels(), + sample_rate: out_cfg.sample_rate(), + buffer_size, + }; + info!( + target: "chanora_audio", + dev_sample_rate, + dev_channels, + buffer_size = ?buffer_size, + "output stream using device native config (no 48k force)" + ); + + let stream = match out_format { + SampleFormat::F32 => super::build_output_stream::( + &out_dev, + &out_stream_cfg, + audio_handler.clone(), + output_gain.clone(), + output_muted.clone(), + dev_sample_rate, + dev_channels, + )?, + SampleFormat::I16 => super::build_output_stream::( + &out_dev, + &out_stream_cfg, + audio_handler.clone(), + output_gain.clone(), + output_muted.clone(), + dev_sample_rate, + dev_channels, + )?, + SampleFormat::U16 => super::build_output_stream::( + &out_dev, + &out_stream_cfg, + audio_handler.clone(), + output_gain.clone(), + output_muted.clone(), + dev_sample_rate, + dev_channels, + )?, + other => { + return Err(AudioError::StreamConfig(format!( + "unsupported output format: {other:?}" + ))) + } + }; + stream + .play() + .map_err(|e| AudioError::Backend(format!("output play: {e}")))?; + stream + }; + + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + let handler_for_task = audio_handler.clone(); + let frames_received_for_task = frames_received.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = &mut shutdown_rx => { + debug!(target: "chanora_audio", "inbound forwarder shutting down"); + break; + } + item = voice_in_rx.recv() => { + match item { + Some(v) => { + let id = SessionAudioId(v.from_client); + let mut h = handler_for_task.lock().unwrap_or_else(|e| e.into_inner()); + if let Err(e) = h.handle_packet(id, v.packet) { + debug!(target: "chanora_audio", error = %e, "decode failed"); + } else { + frames_received_for_task.fetch_add(1, Ordering::Relaxed); + } + } + None => break, + } + } + } + } + }); + + Ok(Self { + transmit_gate, + frames_sent, + frames_received, + output_gain, + output_muted, + audio_processing_config, + audio_processing_stats, + silero_vad_worker, + audio_handler, + _input_stream: Mutex::new(input_stream), + _output_stream: Mutex::new(Some(output_stream)), + shutdown_tx: Some(shutdown_tx), + capture_active, + }) + } + + #[cfg(target_os = "android")] + fn start_with_gate_android( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + mut voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + use crate::mobile_voice_backend::MobileVoiceAudioBackend; + + info!(target: "chanora_audio", "starting audio engine: Android Oboe backend"); + + let transmit_flag_for_capture = transmit_gate.flag_arc(); + let frames_sent = Arc::new(AtomicU32::new(0)); + let frames_received = Arc::new(AtomicU32::new(0)); + let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits())); + let output_muted = Arc::new(AtomicBool::new(false)); + let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default())); + let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); + + let event_queue = AudioEventQueue::new(); + let event_producer = AudioEventQueue::producer(&event_queue); + + if !cfg.mobile_voice_preset { + return Err(AudioError::Backend( + "android: mobile_voice_preset=false is unsupported for Oboe-only backend" + .to_string(), + )); + } + + let mut audio_mode_stack = crate::mode_stack::ModeStack::new(); + + if crate::android_voice_unit::chanora_android_start_voice_service() { + info!( + target: "chanora_audio", + "android: voice foreground service start dispatched (SDD-115)" + ); + } else { + warn!( + target: "chanora_audio", + "android: foreground service start failed; capture may be denied in background (SDD-115)" + ); + } + + match super::android_get_audio_mode() { + Ok(prior_now) => { + let outcome = audio_mode_stack.acquire(prior_now); + if let crate::mode_stack::ModeAcquire::FirstAcquire { prior } = outcome { + match super::android_set_audio_mode(super::ANDROID_MODE_IN_COMMUNICATION) { + Ok(()) => info!( + target: "chanora_audio", + prior_mode = prior, + "android: AudioManager mode set to MODE_IN_COMMUNICATION (SDD-108)" + ), + Err(e) => { + warn!( + target: "chanora_audio", + error = %e, + prior_mode = prior, + "android: setMode failed; rolling back ModeStack acquire (SDD-108 §5)" + ); + let _ = audio_mode_stack.release(); + } + } + } + } + Err(e) => warn!( + target: "chanora_audio", + error = %e, + "android: AudioManager.getMode failed; skipping mode engage (SDD-108)" + ), + } + + let cfg_av = crate::mobile_voice_backend::AndroidVoiceStreamConfig { + effects: cfg.effects, + ..Default::default() + }; + let params = crate::mobile_voice_backend::VoiceAudioParams { + voice_out_tx: voice_out_tx.clone(), + transmit_active: transmit_flag_for_capture, + frames_sent: frames_sent.clone(), + mic_gain: cfg.mic_gain, + handler: AudioHandler::new(), + event_producer: event_producer.clone(), + output_gain: output_gain.clone(), + output_muted: output_muted.clone(), + voice_activity_selector: cfg.voice_activity_selector.clone(), + audio_processing_config: audio_processing_config.clone(), + audio_processing_stats: audio_processing_stats.clone(), + }; + let mut android_voice_unit = + match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) { + Ok(unit) => unit, + Err(e) => { + Self::rollback_android_startup_resources(&mut audio_mode_stack); + return Err(AudioError::Backend(format!( + "android: failed to open Oboe voice unit: {e}" + ))); + } + }; + + if let Err(e) = android_voice_unit.start() { + if let Err(close_err) = android_voice_unit.close() { + warn!( + target: "chanora_audio", + error = %close_err, + "android: AndroidVoiceUnit::close failed during startup rollback" + ); + } + Self::rollback_android_startup_resources(&mut audio_mode_stack); + return Err(AudioError::Backend(format!( + "android: failed to start Oboe voice unit: {e}" + ))); + } + + let android_voice_unit = Arc::new(Mutex::new(Some(android_voice_unit))); + let android_voice_stream_config = Arc::new(Mutex::new(cfg_av.clone())); + let audio_mode_stack = Arc::new(Mutex::new(audio_mode_stack)); + + if let Some(event_rx) = android_voice_unit + .lock() + .unwrap() + .as_mut() + .and_then(|unit| unit.take_event_rx()) + { + Self::spawn_android_backend_event_task( + android_voice_unit.clone(), + android_voice_stream_config.clone(), + voice_out_tx.clone(), + transmit_gate.clone(), + frames_sent.clone(), + event_producer.clone(), + output_gain.clone(), + output_muted.clone(), + cfg.voice_activity_selector.clone(), + audio_processing_config.clone(), + audio_processing_stats.clone(), + cfg.mic_gain, + audio_mode_stack.clone(), + event_rx, + ); + } + + crate::android_voice_unit::register_global_event_sender( + android_voice_unit + .lock() + .unwrap() + .as_ref() + .expect("android voice unit installed") + .event_sender(), + ); + + if crate::android_voice_unit::chanora_android_request_audio_focus() { + info!( + target: "chanora_audio", + "android: audio focus requested (SDD-109)" + ); + } else { + warn!( + target: "chanora_audio", + "android: audio focus request failed; engine operates without focus (SDD-109)" + ); + } + + if crate::android_voice_unit::chanora_android_start_bluetooth_sco() { + info!( + target: "chanora_audio", + "android: Bluetooth SCO started (SDD-110)" + ); + } else { + warn!( + target: "chanora_audio", + "android: Bluetooth SCO start failed; BT HFP may not route correctly (SDD-110)" + ); + } + + let capture_active = true; + + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + let event_producer_for_task = event_producer.clone(); + let frames_received_for_task = frames_received.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = &mut shutdown_rx => { + debug!(target: "chanora_audio", "inbound forwarder shutting down"); + break; + } + item = voice_in_rx.recv() => { + match item { + Some(v) => { + let id = SessionAudioId(v.from_client); + let packet = AudioPacket { client_id: id, data: v.packet }; + if event_producer_for_task.push_packet(packet).is_ok() { + frames_received_for_task.fetch_add(1, Ordering::Relaxed); + } + } + None => break, + } + } + } + } + }); + + Ok(Self { + transmit_gate, + frames_sent, + frames_received, + output_gain, + output_muted, + audio_processing_config, + audio_processing_stats, + audio_event_producer: event_producer, + voice_out_tx, + voice_activity_selector: cfg.voice_activity_selector.clone(), + mic_gain: cfg.mic_gain, + _android_voice_unit: android_voice_unit, + android_voice_stream_config, + audio_mode_stack, + shutdown_tx: Some(shutdown_tx), + capture_active, + }) + } + + #[cfg(any(target_os = "ios", target_os = "macos"))] + fn start_with_gate_ios( + cfg: AudioEngineConfig, + voice_out_tx: mpsc::Sender, + mut voice_in_rx: mpsc::Receiver, + transmit_gate: crate::ptt::AudioTransmitGate, + ) -> Result { + info!( + target: "chanora_audio", + "starting audio engine: Apple VoiceProcessingIO backend" + ); + + if cfg.mobile_voice_preset { + info!( + target: "chanora_audio", + "ios: voice-chat session mode requested — AVAudioSession configured in AppDelegate" + ); + } + + let transmit_flag_for_capture = transmit_gate.flag_arc(); + let frames_sent = Arc::new(AtomicU32::new(0)); + let frames_received = Arc::new(AtomicU32::new(0)); + let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits())); + let output_muted = Arc::new(AtomicBool::new(false)); + let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default())); + let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); + + #[cfg(any(target_os = "ios", target_os = "macos"))] + let audio_handler: Arc>> = + Arc::new(Mutex::new(AudioHandler::new())); + let voice_out_tx_for_backend = voice_out_tx.clone(); + + let ios_voice_backend = + open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams { + handler: audio_handler.clone(), + output_gain: output_gain.clone(), + output_muted: output_muted.clone(), + voice_out_tx: voice_out_tx_for_backend, + transmit_active: transmit_flag_for_capture, + frames_sent: frames_sent.clone(), + mic_gain: cfg.mic_gain, + voice_activity_selector: cfg.voice_activity_selector.clone(), + audio_processing_config: audio_processing_config.clone(), + audio_processing_stats: audio_processing_stats.clone(), + })?; + + let capture_active = true; + + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + let frames_received_for_task = frames_received.clone(); + let handler_for_task = audio_handler.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = &mut shutdown_rx => { + debug!(target: "chanora_audio", "inbound forwarder shutting down"); + break; + } + item = voice_in_rx.recv() => { + match item { + Some(v) => { + let id = SessionAudioId(v.from_client); + let mut h = handler_for_task.lock().unwrap_or_else(|e| e.into_inner()); + let res = h.handle_packet(id, v.packet); + drop(h); + match res { + Ok(_) => { + frames_received_for_task.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + debug!(target: "chanora_audio", error = %e, "decode failed"); + } + } + } + None => break, + } + } + } + } + }); + + Ok(Self { + transmit_gate, + frames_sent, + frames_received, + output_gain, + output_muted, + audio_processing_config, + audio_processing_stats, + audio_handler, + _ios_voice_backend: Mutex::new(Some(ios_voice_backend)), + shutdown_tx: Some(shutdown_tx), + capture_active, + }) + } + + #[cfg(target_os = "android")] + fn rollback_android_startup_resources(audio_mode_stack: &mut crate::mode_stack::ModeStack) { + crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); + crate::android_voice_unit::chanora_android_abandon_audio_focus(); + match super::release_android_audio_mode_for_startup_rollback(audio_mode_stack) { + crate::mode_stack::ModeRelease::LastRelease { prior } => { + match super::android_set_audio_mode(prior) { + Ok(()) => info!( + target: "chanora_audio", + restored_mode = prior, + "android: AudioManager mode restored during startup rollback" + ), + Err(e) => warn!( + target: "chanora_audio", + error = %e, + restored_mode = prior, + "android: failed to restore AudioManager mode during startup rollback" + ), + } + } + crate::mode_stack::ModeRelease::StillHeld => { + info!( + target: "chanora_audio", + "android: audio mode still held during startup rollback" + ); + } + crate::mode_stack::ModeRelease::AlreadyReleased => {} + } + + if crate::android_voice_unit::chanora_android_stop_voice_service() { + info!( + target: "chanora_audio", + "android: voice foreground service stopped during startup rollback" + ); + } + } + + /// Stop the engine. Idempotent. + pub fn stop(&mut self) { + stop_audio_engine(self); + } + + /// iOS-only: restart the underlying VoiceProcessingIO unit after + /// route changes. + pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> { + #[cfg(any(target_os = "ios", target_os = "macos"))] + { + let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()); + let backend = guard + .as_mut() + .ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?; + backend.restart() + } + #[cfg(not(any(target_os = "ios", target_os = "macos")))] + { + Ok(()) + } + } + + /// Android-only: reopen the underlying Oboe voice backend after a + /// route/device change while preserving the engine-owned state. + pub fn android_restart_voice_unit(&self) -> Result<(), AudioError> { + #[cfg(target_os = "android")] + { + use crate::mobile_voice_backend::MobileVoiceAudioBackend; + + crate::android_voice_unit::chanora_android_stop_bluetooth_sco(); + crate::android_voice_unit::chanora_android_abandon_audio_focus(); + crate::android_voice_unit::clear_global_event_sender(); + + if let Some(mut unit) = self._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() { + unit.close().map_err(|e| { + AudioError::Backend(format!( + "android: failed to close Oboe voice unit during route restart: {e}" + )) + })?; + } + + if crate::android_voice_unit::chanora_android_start_voice_service() { + info!( + target: "chanora_audio", + "android: voice foreground service restart dispatched after route change" + ); + } + + if crate::android_voice_unit::chanora_android_request_audio_focus() { + info!( + target: "chanora_audio", + "android: audio focus re-requested after route change" + ); + } + + let cfg_av = self.android_voice_stream_config.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let params = crate::mobile_voice_backend::VoiceAudioParams { + voice_out_tx: self.voice_out_tx.clone(), + transmit_active: self.transmit_gate.flag_arc(), + frames_sent: self.frames_sent.clone(), + mic_gain: self.mic_gain, + handler: AudioHandler::new(), + event_producer: self.audio_event_producer.clone(), + output_gain: self.output_gain.clone(), + output_muted: self.output_muted.clone(), + voice_activity_selector: self.voice_activity_selector.clone(), + audio_processing_config: self.audio_processing_config.clone(), + audio_processing_stats: self.audio_processing_stats.clone(), + }; + let mut reopened = crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) + .map_err(|e| { + AudioError::Backend(format!("android: failed to reopen Oboe voice unit: {e}")) + })?; + reopened.start().map_err(|e| { + AudioError::Backend(format!("android: failed to restart Oboe voice unit: {e}")) + })?; + if let Some(event_rx) = reopened.take_event_rx() { + Self::spawn_android_backend_event_task( + self._android_voice_unit.clone(), + self.android_voice_stream_config.clone(), + self.voice_out_tx.clone(), + self.transmit_gate.clone(), + self.frames_sent.clone(), + self.audio_event_producer.clone(), + self.output_gain.clone(), + self.output_muted.clone(), + self.voice_activity_selector.clone(), + self.audio_processing_config.clone(), + self.audio_processing_stats.clone(), + self.mic_gain, + self.audio_mode_stack.clone(), + event_rx, + ); + } + crate::android_voice_unit::register_global_event_sender(reopened.event_sender()); + + if crate::android_voice_unit::chanora_android_start_bluetooth_sco() { + info!( + target: "chanora_audio", + "android: bluetooth route re-engaged after route change" + ); + } + + let mut guard = self._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()); + *guard = Some(reopened); + Ok(()) + } + #[cfg(not(target_os = "android"))] + { + Ok(()) + } + } + + /// iOS-only: pause the underlying VoiceProcessingIO unit. + pub fn ios_pause_voice_unit(&self) -> Result<(), AudioError> { + #[cfg(any(target_os = "ios", target_os = "macos"))] + { + let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()); + let unit = guard + .as_mut() + .ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?; + unit.pause() + } + #[cfg(not(any(target_os = "ios", target_os = "macos")))] + { + Ok(()) + } + } + + /// iOS-only: resume the underlying VoiceProcessingIO unit. + pub fn ios_resume_voice_unit(&self) -> Result<(), AudioError> { + #[cfg(any(target_os = "ios", target_os = "macos"))] + { + let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()); + let unit = guard + .as_mut() + .ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?; + unit.resume() + } + #[cfg(not(any(target_os = "ios", target_os = "macos")))] + { + Ok(()) + } + } +} diff --git a/crates/chanora_audio/src/engine/mod.rs b/crates/chanora_audio/src/engine/mod.rs new file mode 100644 index 0000000..ec50dc2 --- /dev/null +++ b/crates/chanora_audio/src/engine/mod.rs @@ -0,0 +1,1106 @@ +//! Audio engine — owns platform-native capture/playback, the Opus +//! encoder, and the tsclientlib `AudioHandler` for decode+mix. +//! +//! The engine is started after a protocol connection is established +//! and stopped before disconnect. It does not retry on device +//! change. + +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +use cpal::{SampleFormat, SizedSample}; +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +use std::collections::hash_map::DefaultHasher; +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +use std::hash::{Hash, Hasher}; +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +use tracing::{error, warn}; + +use tsclientlib::audio::AudioHandler; + +#[cfg(target_os = "android")] +use crate::audio_event_queue::{AudioCommand, AudioEventQueue, AudioPacket}; +use crate::AudioError; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +use audiopus::coder::Encoder as OpusEncoder; + +// --------------------------------------------------------------------------- +// Sub-modules +// --------------------------------------------------------------------------- + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android"), +))] +mod capture; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android"), +))] +use capture::{CaptureState, try_open_capture}; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android"), + not(target_os = "linux"), +))] +mod render; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android"), + not(target_os = "linux"), +))] +use render::{FromF32, build_output_stream}; + +#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] +#[doc(hidden)] +pub use capture::bench_seam; + +mod lifecycle; +#[cfg(any(target_os = "ios", target_os = "macos"))] +use lifecycle::IosVoiceBackend; + +// --------------------------------------------------------------------------- +// Desktop-only helpers used by lifecycle (desktop path) +// --------------------------------------------------------------------------- + +/// Audio framing: 48 kHz mono, 20 ms = 960 samples per frame. +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +const SAMPLE_RATE: u32 = 48_000; +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +const FRAME_SAMPLES: usize = 48_000 / 50; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +fn desktop_device_id(device: &cpal::Device) -> Option { + device.id().ok().map(|id| id.to_string()) +} + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +fn short_device_id(id: &str) -> String { + let mut hasher = DefaultHasher::new(); + id.hash(&mut hasher); + format!("{:016x}", hasher.finish()) +} + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +fn describe_device(device: &cpal::Device) -> Option { + let id = desktop_device_id(device)?; + let description = device.description().ok(); + let name = description + .as_ref() + .map(|d| d.name().trim().to_owned()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| format!("Device {}", &short_device_id(&id)[..8])); + + let mut details = Vec::new(); + if let Some(description) = description.as_ref() { + if let Some(manufacturer) = description.manufacturer() { + details.push(manufacturer.to_owned()); + } + if let Some(driver) = description.driver() { + details.push(driver.to_owned()); + } + let device_type = description.device_type(); + if device_type != cpal::device_description::DeviceType::Unknown { + details.push(format!("{device_type}")); + } + let interface_type = description.interface_type(); + if interface_type != cpal::device_description::InterfaceType::Unknown { + details.push(format!("{interface_type}")); + } + } + details.push(format!("id={}", short_device_id(&id))); + + Some(AudioDeviceInfo { + id, + name, + details: details.join(" · "), + is_default: false, + }) +} + +// Macro used by lifecycle.rs for desktop warn! calls. +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +macro_rules! warn_desktop { + ($($tt:tt)*) => { warn!($($tt)*) }; +} +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +pub(crate) use warn_desktop; + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +fn new_desktop_audio_processing_state() -> ( + Arc>, + Arc, + Arc>>, +) { + let mut config = crate::AudioProcessingConfig::default(); + config.processing_backend = crate::AudioBackend::Noop; + let audio_processing_config = Arc::new(Mutex::new(config.clone())); + let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); + let silero_vad_worker = Arc::new(Mutex::new(None)); + apply_desktop_vad_backend_to_worker( + &config, + &silero_vad_worker, + audio_processing_stats.as_ref(), + ); + ( + audio_processing_config, + audio_processing_stats, + silero_vad_worker, + ) +} + +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +fn apply_desktop_vad_backend_to_worker( + config: &crate::AudioProcessingConfig, + silero_vad_worker: &Arc>>, + audio_processing_stats: &crate::SharedAudioProcessingStats, +) { + if config.vad_backend != crate::VadBackend::SileroOnnx { + let mut worker_guard = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner()); + if worker_guard.is_some() { + info!( + target: "chanora_audio", + backend = config.vad_backend.as_str(), + "desktop: Silero ONNX VAD worker cleared because another VAD backend is selected" + ); + } + *worker_guard = None; + audio_processing_stats.set_vad_fallback_active(false); + return; + } + + let model_path = crate::vad::silero_model_bundle_path(); + info!( + target: "chanora_audio", + path = %model_path, + "desktop: Silero ONNX VAD selected; loading model worker" + ); + let new_worker = crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path); + { + let mut worker_guard = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner()); + *worker_guard = new_worker; + } + let worker_installed = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner()).is_some(); + if worker_installed { + info!( + target: "chanora_audio", + path = %model_path, + "desktop: Silero ONNX VAD worker loaded" + ); + audio_processing_stats.set_vad_fallback_active(false); + } else { + warn!( + target: "chanora_audio", + path = %model_path, + "desktop: Silero ONNX VAD worker unavailable; WebRTC fallback will be used" + ); + audio_processing_stats.set_vad_fallback_active(true); + } +} + +#[cfg(any(test, target_os = "android"))] +pub(crate) fn release_android_audio_mode_for_startup_rollback( + audio_mode_stack: &mut crate::mode_stack::ModeStack, +) -> crate::mode_stack::ModeRelease { + audio_mode_stack.release() +} + +// --------------------------------------------------------------------------- +// Public types +// --------------------------------------------------------------------------- + +/// Stable Chanora-side identifier for AudioHandler bookkeeping. +/// We only ever have one connection at a time (DEC-006), so this is +/// trivially unique. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct SessionAudioId(pub u64); + +/// List of available audio devices from the platform. +#[derive(Debug, Clone)] +pub struct AudioDeviceList { + /// Available input (capture) devices. + pub input_devices: Vec, + /// Available output (playback) devices. + pub output_devices: Vec, +} + +/// Info about a single audio device. +#[derive(Debug, Clone)] +pub struct AudioDeviceInfo { + /// Stable platform-reported device identifier. + pub id: String, + /// Human-readable device name from the OS. + pub name: String, + /// Additional device details useful for disambiguation. + pub details: String, + /// True if the OS reports this as the default device. + pub is_default: bool, +} + +/// Enumerate available audio input and output devices. +/// On mobile platforms (iOS, Android) returns an empty list because +/// device selection is managed by the OS audio session. +#[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") +))] +pub fn list_audio_devices() -> AudioDeviceList { + use cpal::traits::HostTrait; + let mut list = AudioDeviceList { + input_devices: Vec::new(), + output_devices: Vec::new(), + }; + let host = cpal::default_host(); + let default_in = host + .default_input_device() + .and_then(|device| desktop_device_id(&device)); + let default_out = host + .default_output_device() + .and_then(|device| desktop_device_id(&device)); + if let Ok(devices) = host.input_devices() { + for d in devices { + if let Some(mut device) = describe_device(&d) { + device.is_default = default_in + .as_ref() + .is_some_and(|default_id| default_id == &device.id); + list.input_devices.push(device); + } + } + } + if let Ok(devices) = host.output_devices() { + for d in devices { + if let Some(mut device) = describe_device(&d) { + device.is_default = default_out + .as_ref() + .is_some_and(|default_id| default_id == &device.id); + list.output_devices.push(device); + } + } + } + list +} + +#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))] +/// Enumerate available audio input and output devices. +pub fn list_audio_devices() -> AudioDeviceList { + AudioDeviceList { + input_devices: Vec::new(), + output_devices: Vec::new(), + } +} + +/// Engine configuration. +#[derive(Clone)] +pub struct AudioEngineConfig { + /// Input gain applied before encoding (1.0 = pass-through). + pub mic_gain: f32, + /// Initial PTT state. When false the encoder is bypassed and no + /// outbound packets are produced. + pub ptt_initial: bool, + /// Audio-effect toggles. The struct is honoured by *naming* but + /// the filters themselves are still no-op in Beta — see the + /// crate-level docs and DEC-007/008/009/010. + pub effects: crate::AudioEffects, + /// A.5 mobile: prefer the OS-provided "voice communication" + /// audio source on mobile platforms (Android + /// `MediaRecorder.AudioSource.VOICE_COMMUNICATION`, iOS + /// `AVAudioSession.Mode.voiceChat`). On Linux desktop this is + /// ignored — the DSP chain stays a no-op and we use the + /// default ALSA/PipeWire source. + /// + /// Android status: this is enforced by the Oboe-only backend, + /// which opens input with `VoiceCommunication` and output with + /// voice-communication usage / speech content. Setting `false` + /// is rejected on Android because the P0 path intentionally has + /// no generic mobile-audio fallback. + pub mobile_voice_preset: bool, + /// Optional input device id override. When `None`, the system + /// default input device is used. Set to a device id from + /// [`list_audio_devices`] to pin a specific microphone. + pub input_device_id: Option, + /// Optional output device id override. + pub output_device_id: Option, + /// Optional selector used by P1 VoiceActivity to publish VAD state. + #[doc(hidden)] + pub voice_activity_selector: Option>, +} + +impl std::fmt::Debug for AudioEngineConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AudioEngineConfig") + .field("mic_gain", &self.mic_gain) + .field("ptt_initial", &self.ptt_initial) + .field("effects", &self.effects) + .field("mobile_voice_preset", &self.mobile_voice_preset) + .field("input_device_id", &self.input_device_id) + .field("output_device_id", &self.output_device_id) + .field( + "voice_activity_selector", + &self.voice_activity_selector.as_ref().map(|_| "present"), + ) + .finish() + } +} + +impl Default for AudioEngineConfig { + fn default() -> Self { + Self { + mic_gain: 1.0, + ptt_initial: false, + effects: crate::AudioEffects::default(), + mobile_voice_preset: true, + input_device_id: None, + output_device_id: None, + voice_activity_selector: None, + } + } +} + +/// Running audio engine. Drop = stop. +pub struct AudioEngine { + /// `transmit_active` is the authoritative gate for outbound + /// voice — the Opus encoder feed consults this flag once per + /// 20 ms frame. PTT subsystems (focused widget, future + /// Windows / macOS / Linux global backends) drive this flag + /// through [`Self::set_transmit_active`]; nothing else is + /// permitted to flip it (SAD-075 / SDD-089). + transmit_gate: crate::ptt::AudioTransmitGate, + frames_sent: Arc, + frames_received: Arc, + /// Master output gain as f32 bits in an AtomicU32. Default 1.0. + /// Adjusted via [`Self::set_output_gain`] from the bridge. + output_gain: Arc, + /// Master output mute. When true the output callback fills the + /// device buffer with silence regardless of incoming voice + /// frames. Used for self-output-mute on the local device, + /// independent of the server-side mute the protocol layer + /// broadcasts. + output_muted: Arc, + audio_processing_config: Arc>, + audio_processing_stats: Arc, + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + silero_vad_worker: Arc>>, + #[cfg(not(target_os = "android"))] + audio_handler: Arc>>, + #[cfg(target_os = "android")] + audio_event_producer: crate::audio_event_queue::AudioEventProducer, + #[cfg(target_os = "android")] + voice_out_tx: mpsc::Sender, + #[cfg(target_os = "android")] + voice_activity_selector: Option>, + #[cfg(target_os = "android")] + mic_gain: f32, + #[cfg(all( + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") + ))] + _input_stream: Mutex>, + #[cfg(target_os = "linux")] + _output_stream: Mutex>, + #[cfg(all( + not(target_os = "linux"), + not(target_os = "ios"), + not(target_os = "macos"), + not(target_os = "android") + ))] + _output_stream: Mutex>, + #[cfg(any(target_os = "ios", target_os = "macos"))] + _ios_voice_backend: Mutex>, + #[cfg(target_os = "android")] + _android_voice_unit: Arc>>, + #[cfg(target_os = "android")] + android_voice_stream_config: Arc>, + #[cfg(target_os = "android")] + audio_mode_stack: Arc>, + shutdown_tx: Option>, + capture_active: bool, +} + +unsafe impl Send for AudioEngine {} +unsafe impl Sync for AudioEngine {} + +// --------------------------------------------------------------------------- +// Public API methods +// --------------------------------------------------------------------------- + +impl AudioEngine { + /// Set the **transmission gate** (SRS-201). When true the + /// encoder feed is allowed to emit Opus frames; when false the + /// captured audio is discarded before encoding. This is the + /// only writer permitted on `transmit_active` (SAD-075 / + /// SDD-089). Push-to-Talk subsystems — focused PTT today, + /// per-platform global backends in a follow-up — call this + /// method exclusively. No-op when capture is inactive. + pub fn set_transmit_active(&self, active: bool) { + self.transmit_gate.set(active); + } + + /// Current transmit gate state. + pub fn transmit_active(&self) -> bool { + self.transmit_gate.load() + } + + /// Shared handle to the underlying transmit gate (SAD-075 / + /// SDD-089). Returned for diagnostics and integration tests + /// only; never mutate the underlying atomic directly — use + /// [`Self::set_transmit_active`] instead. + pub fn transmit_gate(&self) -> &crate::ptt::AudioTransmitGate { + &self.transmit_gate + } + + /// True if the capture stream opened. When false, the engine + /// runs in playback-only mode and the transmit gate is a + /// no-op (no frames will ever be encoded). + pub fn capture_active(&self) -> bool { + self.capture_active + } + + /// Number of Opus frames sent since the engine started. + pub fn frames_sent(&self) -> u32 { + self.frames_sent.load(Ordering::Relaxed) + } + + /// Number of inbound voice packets received and decoded. + pub fn frames_received(&self) -> u32 { + self.frames_received.load(Ordering::Relaxed) + } + + /// Current microphone input level in dBFS (-120.0 = silence, 0.0 = clipping). + pub fn input_level(&self) -> f32 { + self.audio_processing_stats.input_dbfs() + } + + /// Current audio-processing config snapshot. + pub fn audio_processing_config_snapshot(&self) -> crate::AudioProcessingConfig { + self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// Apply a voice-processing config after validating iOS invariants. + pub fn set_audio_processing_config( + &self, + mut config: crate::AudioProcessingConfig, + ) -> Result<(), AudioError> { + #[cfg(target_os = "ios")] + config.validate_for_ios()?; + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + { + config.processing_backend = crate::AudioBackend::Noop; + } + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + self.apply_desktop_vad_backend(&config); + let mut guard = self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner()); + *guard = config; + Ok(()) + } + + /// Re-apply the current audio processing config. + pub fn reload_audio_processing_config(&self) -> Result<(), AudioError> { + let config = self.audio_processing_config_snapshot(); + self.set_audio_processing_config(config) + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + fn apply_desktop_vad_backend(&self, config: &crate::AudioProcessingConfig) { + apply_desktop_vad_backend_to_worker( + config, + &self.silero_vad_worker, + self.audio_processing_stats.as_ref(), + ); + } + + /// Current voice-processing stats snapshot. + pub fn audio_processing_stats(&self) -> crate::AudioProcessingStats { + let config = self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner()).clone(); + self.audio_processing_stats.snapshot(&config) + } + + /// Latest Android voice-audio diagnostics snapshot (SDD-112 item + /// 10 / SDD-113 item 7 / SDD-116 item 3). + pub fn android_diagnostics( + &self, + ) -> Option { + crate::mobile_voice_backend::current_android_audio_diagnostics() + } + + /// Set master output mute. + pub fn set_output_muted(&self, muted: bool) { + self.output_muted.store(muted, Ordering::Relaxed); + } + + /// True if the master output is currently muted locally. + pub fn output_muted(&self) -> bool { + self.output_muted.load(Ordering::Relaxed) + } + + /// Set master output gain. + pub fn set_output_gain(&self, gain: f32) { + let clamped = gain.clamp(0.0, 4.0); + self.output_gain.store(clamped.to_bits(), Ordering::Relaxed); + } + + /// Current master output gain. + pub fn output_gain(&self) -> f32 { + f32::from_bits(self.output_gain.load(Ordering::Relaxed)) + } + + /// Set per-client output volume (SRS-075). + pub fn set_client_volume(&self, client_id: u64, volume: f32) { + let clamped = volume.clamp(0.0, 4.0); + #[cfg(target_os = "android")] + { + let mut cmd = AudioCommand::SetVolume(SessionAudioId(client_id), clamped); + for _ in 0..64 { + match self.audio_event_producer.push_control(cmd) { + Ok(()) => return, + Err(returned) => { + cmd = returned; + std::thread::yield_now(); + } + } + } + tracing::warn!( + target: "chanora_audio", + client_id, + volume = clamped, + "set_client_volume: control queue full after 64 retries — volume not applied" + ); + } + #[cfg(not(target_os = "android"))] + { + match self.audio_handler.lock() { + Ok(mut h) => { + if let Some(q) = h.get_mut_queues().get_mut(&SessionAudioId(client_id)) { + q.volume = clamped; + } + } + Err(e) => { + tracing::warn!( + target: "chanora_audio", + client_id, + volume = clamped, + error = %e, + "set_client_volume: audio_handler lock poisoned — volume not applied" + ); + } + } + } + } +} + +impl Drop for AudioEngine { + fn drop(&mut self) { + lifecycle::stop_audio_engine(self); + } +} + +// --------------------------------------------------------------------------- +// Android voice-communication routing +// --------------------------------------------------------------------------- +#[cfg(target_os = "android")] +pub const ANDROID_MODE_IN_COMMUNICATION: i32 = 3; + +#[cfg(target_os = "android")] +#[derive(Debug, Clone)] +pub enum AudioModeError { + JniAttachFailed(String), + MethodCallFailed { + method: &'static str, + detail: String, + }, + Other(String), +} + +#[cfg(target_os = "android")] +impl std::fmt::Display for AudioModeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::JniAttachFailed(d) => write!(f, "JNI attach failed: {d}"), + Self::MethodCallFailed { method, detail } => { + write!(f, "JNI method `{method}` failed: {detail}") + } + Self::Other(d) => write!(f, "{d}"), + } + } +} + +#[cfg(target_os = "android")] +impl std::error::Error for AudioModeError {} + +#[cfg(target_os = "android")] +impl From for AudioModeError { + fn from(value: jni::errors::Error) -> Self { + Self::JniAttachFailed(value.to_string()) + } +} + +#[cfg(target_os = "android")] +fn android_audio_manager_call(op: F) -> Result +where + F: for<'local> FnOnce( + &mut jni::Env<'local>, + &jni::objects::JObject<'local>, + ) -> Result + + std::panic::UnwindSafe, +{ + use jni::objects::{JObject, JString, JValue}; + + let result = std::panic::catch_unwind(|| -> Result { + let ctx = ndk_context::android_context(); + let vm_ptr = ctx.vm(); + if vm_ptr.is_null() { + return Err(AudioModeError::JniAttachFailed( + "ndk_context vm is null".to_string(), + )); + } + let jvm = unsafe { jni::JavaVM::from_raw(vm_ptr as *mut _) }; + jvm.attach_current_thread(|env| -> Result { + let context_obj = unsafe { JObject::from_raw(env, ctx.context() as jni::sys::jobject) }; + + let service_name: JString = + env.new_string("audio") + .map_err(|e| AudioModeError::MethodCallFailed { + method: "new_string", + detail: e.to_string(), + })?; + let service_name_obj = JObject::from(service_name); + let audio_manager = env + .call_method( + &context_obj, + jni::jni_str!("getSystemService"), + jni::jni_sig!("(Ljava/lang/String;)Ljava/lang/Object;"), + &[JValue::Object(&service_name_obj)], + ) + .map_err(|e| AudioModeError::MethodCallFailed { + method: "getSystemService", + detail: e.to_string(), + })? + .l() + .map_err(|e| AudioModeError::MethodCallFailed { + method: "getSystemService", + detail: format!("obj cast: {e}"), + })?; + if audio_manager.is_null() { + return Err(AudioModeError::Other( + "AudioManager service is null".to_string(), + )); + } + op(env, &audio_manager) + }) + }); + match result { + Ok(inner) => inner, + Err(_) => Err(AudioModeError::Other( + "panic in JNI audio_manager call".to_string(), + )), + } +} + +#[cfg(target_os = "android")] +pub fn android_get_audio_mode() -> Result { + android_audio_manager_call(|env, audio_manager| { + env.call_method( + audio_manager, + jni::jni_str!("getMode"), + jni::jni_sig!("()I"), + &[], + ) + .map_err(|e| AudioModeError::MethodCallFailed { + method: "getMode", + detail: e.to_string(), + })? + .i() + .map_err(|e| AudioModeError::MethodCallFailed { + method: "getMode", + detail: format!("int cast: {e}"), + }) + }) +} + +#[cfg(target_os = "android")] +pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> { + use jni::objects::JValue; + android_audio_manager_call(move |env, audio_manager| { + env.call_method( + audio_manager, + jni::jni_str!("setMode"), + jni::jni_sig!("(I)V"), + &[JValue::Int(mode)], + ) + .map_err(|e| AudioModeError::MethodCallFailed { + method: "setMode", + detail: e.to_string(), + })?; + Ok(()) + }) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + #[test] + fn desktop_capture_voice_activity_opens_selector_from_speech() { + let gate = crate::ptt::AudioTransmitGate::new(false); + let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone())); + selector.set_mode(crate::TransmitMode::VoiceActivity); + selector.set_in_channel(true); + + let encoder = crate::opus_voice::new_voip_encoder("desktop VAD test").unwrap(); + let (voice_out_tx, _voice_out_rx) = mpsc::channel::(16); + let frames_sent = Arc::new(AtomicU32::new(0)); + let voice_out_tx = crate::opus_voice::start_out_packet_worker( + voice_out_tx, + frames_sent, + "desktop-vad-test", + ) + .unwrap(); + let stats = Arc::new(crate::SharedAudioProcessingStats::default()); + let mut capture = CaptureState::new( + encoder, + SAMPLE_RATE, + 1, + 1.0, + voice_out_tx, + gate.flag_arc(), + Some(selector.clone()), + Arc::new(Mutex::new(crate::AudioProcessingConfig { + vad_backend: crate::VadBackend::WebrtcVad, + ..crate::AudioProcessingConfig::default() + })), + Arc::new(Mutex::new(None)), + stats.clone(), + ); + + let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; + for (idx, sample) in voiced.iter_mut().enumerate() { + let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32; + *sample = phase.sin() * 0.4; + } + for _ in 0..6 { + capture.ingest(&voiced); + } + + assert!( + selector.voice_activity_open(), + "desktop capture must feed VAD and open VoiceActivity selector before transmit is already active" + ); + assert!( + gate.load(), + "VoiceActivity selector should publish transmit gate" + ); + let snapshot = stats.snapshot(&crate::AudioProcessingConfig::default()); + assert!(snapshot.vad_active, "stats should expose active VAD"); + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + #[test] + fn desktop_startup_audio_processing_state_applies_default_silero_fallback() { + let _guard = crate::vad::SILERO_MODEL_PATH_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + crate::vad::clear_silero_model_path_for_test(); + + let (config, stats, worker) = new_desktop_audio_processing_state(); + + assert_eq!( + config.lock().unwrap_or_else(|e| e.into_inner()).vad_backend, + crate::VadBackend::SileroOnnx, + "desktop startup config should keep the default Silero backend selected" + ); + assert!( + worker.lock().unwrap_or_else(|e| e.into_inner()).is_none(), + "missing startup model should not create an ONNX worker" + ); + let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner())); + assert!( + snapshot.vad_fallback_active, + "desktop startup should mark WebRTC fallback active when the default Silero worker cannot load" + ); + crate::vad::clear_silero_model_path_for_test(); + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + #[test] + fn desktop_set_audio_processing_config_normalizes_default_backend_to_noop() { + let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default())); + let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default()); + let engine = AudioEngine { + transmit_gate: crate::ptt::AudioTransmitGate::new(false), + frames_sent: Arc::new(AtomicU32::new(0)), + frames_received: Arc::new(AtomicU32::new(0)), + output_gain: Arc::new(AtomicU32::new(1.0_f32.to_bits())), + output_muted: Arc::new(AtomicBool::new(false)), + audio_processing_config: audio_processing_config.clone(), + audio_processing_stats, + silero_vad_worker: Arc::new(Mutex::new(None)), + audio_handler: Arc::new(Mutex::new(AudioHandler::new())), + _input_stream: Mutex::new(None), + _output_stream: Mutex::new(None), + shutdown_tx: None, + capture_active: false, + }; + + engine + .set_audio_processing_config(crate::AudioProcessingConfig::default()) + .unwrap(); + + assert_eq!( + engine.audio_processing_config_snapshot().processing_backend, + crate::AudioBackend::Noop, + "desktop setter should report the actual no-op processing backend for default configs" + ); + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + #[test] + fn desktop_capture_silero_selection_reports_fallback_when_worker_unavailable() { + let _guard = crate::vad::SILERO_MODEL_PATH_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + crate::vad::clear_silero_model_path_for_test(); + + let gate = crate::ptt::AudioTransmitGate::new(false); + let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone())); + selector.set_mode(crate::TransmitMode::VoiceActivity); + selector.set_in_channel(true); + + let encoder = crate::opus_voice::new_voip_encoder("desktop Silero fallback test").unwrap(); + let (voice_out_tx, _voice_out_rx) = mpsc::channel::(16); + let frames_sent = Arc::new(AtomicU32::new(0)); + let voice_out_tx = crate::opus_voice::start_out_packet_worker( + voice_out_tx, + frames_sent, + "desktop-silero-fallback-test", + ) + .unwrap(); + let stats = Arc::new(crate::SharedAudioProcessingStats::default()); + let config = Arc::new(Mutex::new(crate::AudioProcessingConfig { + vad_backend: crate::VadBackend::SileroOnnx, + ..crate::AudioProcessingConfig::default() + })); + let mut capture = CaptureState::new( + encoder, + SAMPLE_RATE, + 1, + 1.0, + voice_out_tx, + gate.flag_arc(), + Some(selector), + config.clone(), + Arc::new(Mutex::new(None)), + stats.clone(), + ); + + let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; + for (idx, sample) in voiced.iter_mut().enumerate() { + let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32; + *sample = phase.sin() * 0.4; + } + capture.ingest(&voiced); + + let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner())); + assert_eq!(snapshot.vad_backend, crate::VadBackend::SileroOnnx); + assert!( + snapshot.vad_fallback_active, + "desktop Silero selection should make WebRTC fallback visible when ONNX worker cannot load" + ); + crate::vad::clear_silero_model_path_for_test(); + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + #[test] + fn desktop_capture_vad_consumes_only_new_pcm_while_transmitting() { + let gate = crate::ptt::AudioTransmitGate::new(true); + let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone())); + selector.set_mode(crate::TransmitMode::VoiceActivity); + selector.set_in_channel(true); + + let encoder = crate::opus_voice::new_voip_encoder("desktop VAD duplicate test").unwrap(); + let (voice_out_tx, _voice_out_rx) = mpsc::channel::(16); + let frames_sent = Arc::new(AtomicU32::new(0)); + let voice_out_tx = crate::opus_voice::start_out_packet_worker( + voice_out_tx, + frames_sent, + "desktop-vad-duplicate-test", + ) + .unwrap(); + let stats = Arc::new(crate::SharedAudioProcessingStats::default()); + let mut capture = CaptureState::new( + encoder, + SAMPLE_RATE, + 1, + 1.0, + voice_out_tx, + gate.flag_arc(), + Some(selector), + Arc::new(Mutex::new(crate::AudioProcessingConfig { + vad_backend: crate::VadBackend::Disabled, + ..crate::AudioProcessingConfig::default() + })), + Arc::new(Mutex::new(None)), + stats, + ); + + capture.pcm_accum.extend(std::iter::repeat_n( + 0.1_f32, + crate::frame::FRAME_10MS_SAMPLES + 240, + )); + capture.pending_10ms[..240].fill(0.1); + capture.pending_10ms_len = 240; + capture.capture_frame_seq = 1; + capture.pcm_accum.extend(std::iter::repeat_n(0.2_f32, 240)); + + capture.process_pending_vad_frames(crate::frame::FRAME_10MS_SAMPLES + 240); + + assert_eq!( + capture.capture_frame_seq, 2, + "VAD should consume only the 240 samples appended by the current ingest and complete one pending 10 ms frame" + ); + } + + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))] + #[test] + fn desktop_capture_silero_stale_enqueued_worker_uses_fallback() { + let gate = crate::ptt::AudioTransmitGate::new(false); + let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone())); + selector.set_mode(crate::TransmitMode::VoiceActivity); + selector.set_in_channel(true); + + let encoder = + crate::opus_voice::new_voip_encoder("desktop stale Silero fallback test").unwrap(); + let (voice_out_tx, _voice_out_rx) = mpsc::channel::(16); + let frames_sent = Arc::new(AtomicU32::new(0)); + let voice_out_tx = crate::opus_voice::start_out_packet_worker( + voice_out_tx, + frames_sent, + "desktop-stale-silero-fallback-test", + ) + .unwrap(); + let stats = Arc::new(crate::SharedAudioProcessingStats::default()); + let config = Arc::new(Mutex::new(crate::AudioProcessingConfig { + vad_backend: crate::VadBackend::SileroOnnx, + ..crate::AudioProcessingConfig::default() + })); + let worker = Arc::new(Mutex::new(Some( + crate::vad::silero_onnx::SileroOnnxVadWorker::stale_test_worker(), + ))); + let mut capture = CaptureState::new( + encoder, + SAMPLE_RATE, + 1, + 1.0, + voice_out_tx, + gate.flag_arc(), + Some(selector), + config.clone(), + worker, + stats.clone(), + ); + + let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES]; + for (idx, sample) in voiced.iter_mut().enumerate() { + let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32; + *sample = phase.sin() * 0.4; + } + + capture.process_10ms_capture_frame(&voiced); + + let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner())); + assert!( + snapshot.vad_fallback_active, + "stale Silero worker output should report active WebRTC fallback" + ); + assert!( + snapshot.vad_probability > 0.5, + "stale Silero worker output should use WebRTC fallback probability instead of forced silence" + ); + } + + #[test] + fn android_startup_rollback_releases_acquired_mode_snapshot() { + let mut stack = crate::mode_stack::ModeStack::new(); + let _ = stack.acquire(7); + + let release = release_android_audio_mode_for_startup_rollback(&mut stack); + + assert_eq!( + release, + crate::mode_stack::ModeRelease::LastRelease { prior: 7 } + ); + assert_eq!(stack.refcount(), 0); + assert_eq!(stack.snapshot(), None); + } +} diff --git a/crates/chanora_audio/src/engine/render.rs b/crates/chanora_audio/src/engine/render.rs new file mode 100644 index 0000000..fbb201e --- /dev/null +++ b/crates/chanora_audio/src/engine/render.rs @@ -0,0 +1,176 @@ +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; + +use cpal::traits::StreamTrait; +use cpal::SampleFormat; +use tracing::{error, warn}; +use tsclientlib::audio::AudioHandler; + +use crate::AudioError; + +use super::SessionAudioId; +use super::SAMPLE_RATE; + +pub(super) fn build_output_stream( + device: &cpal::Device, + config: &cpal::StreamConfig, + handler: Arc>>, + output_gain: Arc, + output_muted: Arc, + dev_sample_rate: u32, + dev_channels: usize, +) -> Result +where + T: cpal::SizedSample + FromF32 + Send + 'static, +{ + let resample_ratio = SAMPLE_RATE as f64 / dev_sample_rate as f64; + let same_rate = dev_sample_rate == SAMPLE_RATE; + let resample_state: Arc> = + Arc::new(Mutex::new(PlaybackResampleState { + pos: 0.0, + last_l: 0.0, + last_r: 0.0, + })); + let mut scratch: Vec = Vec::with_capacity(8192); + let mut last_slow_warn = std::time::Instant::now() + .checked_sub(std::time::Duration::from_secs(2)) + .unwrap_or_else(std::time::Instant::now); + let stream = device + .build_output_stream( + *config, + move |out: &mut [T], _: &cpal::OutputCallbackInfo| { + let cb_start = std::time::Instant::now(); + let muted = output_muted.load(Ordering::Relaxed); + let dev_frames = out.len() / dev_channels.max(1); + let src_frames = if same_rate { + dev_frames + } else { + ((dev_frames as f64 * resample_ratio).ceil() as usize) + 2 + }; + let needed = src_frames * 2; + if scratch.len() < needed { + scratch.resize(needed, 0.0); + } + scratch[..needed].fill(0.0); + { + let mut h = handler.lock().unwrap_or_else(|e| e.into_inner()); + h.fill_buffer(&mut scratch[..needed]); + } + + if muted { + for dst in out.iter_mut() { + *dst = T::from_f32_sample(0.0); + } + } else { + let gain = f32::from_bits(output_gain.load(Ordering::Relaxed)); + if same_rate && dev_channels == 2 { + for (dst, s) in out.iter_mut().zip(scratch[..needed].iter().copied()) { + *dst = T::from_f32_sample(s * gain); + } + } else { + let mut state = resample_state.lock().unwrap_or_else(|e| e.into_inner()); + let mut pos = state.pos; + let mut last_l = state.last_l; + let mut last_r = state.last_r; + for frame_idx in 0..dev_frames { + let i = pos.floor() as isize; + let frac = pos - i as f64; + let (a_l, a_r) = if i <= 0 { + (last_l, last_r) + } else { + let idx = ((i - 1) as usize) * 2; + (scratch[idx], scratch[idx + 1]) + }; + let i_usize = i.max(0) as usize; + let (b_l, b_r) = if i_usize < src_frames { + let idx = i_usize * 2; + (scratch[idx], scratch[idx + 1]) + } else { + (a_l, a_r) + }; + let l = (a_l as f64 + frac * (b_l - a_l) as f64) as f32 * gain; + let r = (a_r as f64 + frac * (b_r - a_r) as f64) as f32 * gain; + let base = frame_idx * dev_channels; + if dev_channels == 1 { + out[base] = T::from_f32_sample((l + r) * 0.5); + } else { + out[base] = T::from_f32_sample(l); + if dev_channels >= 2 { + out[base + 1] = T::from_f32_sample(r); + } + for c in 2..dev_channels { + out[base + c] = T::from_f32_sample(0.0); + } + } + pos += resample_ratio; + } + let consumed = pos.floor() as usize; + state.pos = pos - consumed as f64; + if consumed > 0 && consumed <= src_frames { + let idx = (consumed - 1) * 2; + last_l = scratch[idx]; + last_r = scratch[idx + 1]; + state.last_l = last_l; + state.last_r = last_r; + } + } + } + + let elapsed = cb_start.elapsed(); + let period_us = (dev_frames as u64 * 1_000_000) / dev_sample_rate as u64; + if elapsed.as_micros() as u64 > period_us / 2 + && last_slow_warn.elapsed() > std::time::Duration::from_secs(1) + { + last_slow_warn = std::time::Instant::now(); + warn!( + target: "chanora_audio", + callback_us = elapsed.as_micros() as u64, + period_us, + dev_frames, + "output callback exceeded half the period budget — possible underrun cause" + ); + } + }, + move |e| { + error!(target: "chanora_audio", error = %e, "output stream error"); + }, + None, + ) + .map_err(|e| { + error!( + target: "chanora_audio", + error = %e, + requested_channels = config.channels, + requested_sample_rate = config.sample_rate, + "build_output_stream FAILED" + ); + AudioError::Backend(format!("build_output_stream: {e}")) + })?; + Ok(stream) +} + +struct PlaybackResampleState { + pos: f64, + last_l: f32, + last_r: f32, +} + +pub(super) trait FromF32 { + fn from_f32_sample(v: f32) -> Self; +} +impl FromF32 for f32 { + fn from_f32_sample(v: f32) -> Self { + v + } +} +impl FromF32 for i16 { + fn from_f32_sample(v: f32) -> Self { + (v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i16 + } +} +impl FromF32 for u16 { + fn from_f32_sample(v: f32) -> Self { + let s = (v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i32; + (s + i32::from(i16::MAX) + 1) as u16 + } +}