Edison Jwa
b20e6b663a
Replace 42 .lock().unwrap() calls with .unwrap_or_else(|e| e.into_inner()) across 7 files. Poisoned mutex recovery prevents panics in realtime audio callbacks. Add SAFETY comment to WebRtcFallbackVad Send impl (TODO-007).
3245 lines
134 KiB
Rust
3245 lines
134 KiB
Rust
//! Audio engine — owns platform-native capture/playback, the Opus
|
||
//! encoder, and the tsclientlib `AudioHandler` for decode+mix.
|
||
//!
|
||
//! The engine is started after a protocol connection is established
|
||
//! and stopped before disconnect. It does not retry on device
|
||
//! change.
|
||
|
||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||
use std::sync::{Arc, Mutex};
|
||
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
use cpal::{SampleFormat, SizedSample};
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
use std::collections::hash_map::DefaultHasher;
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
use std::hash::{Hash, Hasher};
|
||
use tokio::sync::mpsc;
|
||
use tracing::{debug, info};
|
||
// `error!` and `warn!` are used only inside the cpal capture /
|
||
// playback paths (`build_input_stream`, `build_output_stream`,
|
||
// `try_open_capture` log lines). Cfg-gate the imports too so iOS
|
||
// builds don't carry an unused-imports warning.
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
use tracing::{error, warn};
|
||
|
||
#[cfg(any(target_os = "ios", target_os = "android"))]
|
||
use tracing::warn;
|
||
|
||
use tsclientlib::audio::AudioHandler;
|
||
|
||
use chanora_protocol::{InboundVoice, OutPacket};
|
||
|
||
#[cfg(target_os = "android")]
|
||
use crate::audio_event_queue::{AudioCommand, AudioEventQueue, AudioPacket};
|
||
use crate::AudioError;
|
||
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
use audiopus::coder::Encoder as OpusEncoder;
|
||
/// Stable Chanora-side identifier for AudioHandler bookkeeping.
|
||
/// We only ever have one connection at a time (DEC-006), so this is
|
||
/// trivially unique.
|
||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||
pub struct SessionAudioId(pub u64);
|
||
|
||
/// Audio framing: 48 kHz mono, 20 ms = 960 samples per frame.
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
const SAMPLE_RATE: u32 = 48_000;
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
const FRAME_SAMPLES: usize = 48_000 / 50; // 960
|
||
|
||
/// List of available audio devices from the platform.
|
||
#[derive(Debug, Clone)]
|
||
pub struct AudioDeviceList {
|
||
/// Available input (capture) devices.
|
||
pub input_devices: Vec<AudioDeviceInfo>,
|
||
/// Available output (playback) devices.
|
||
pub output_devices: Vec<AudioDeviceInfo>,
|
||
}
|
||
|
||
/// Info about a single audio device.
|
||
#[derive(Debug, Clone)]
|
||
pub struct AudioDeviceInfo {
|
||
/// Stable platform-reported device identifier.
|
||
pub id: String,
|
||
/// Human-readable device name from the OS.
|
||
pub name: String,
|
||
/// Additional device details useful for disambiguation.
|
||
pub details: String,
|
||
/// True if the OS reports this as the default device.
|
||
pub is_default: bool,
|
||
}
|
||
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
fn desktop_device_id(device: &cpal::Device) -> Option<String> {
|
||
device.id().ok().map(|id| id.to_string())
|
||
}
|
||
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
fn short_device_id(id: &str) -> String {
|
||
let mut hasher = DefaultHasher::new();
|
||
id.hash(&mut hasher);
|
||
format!("{:016x}", hasher.finish())
|
||
}
|
||
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
fn describe_device(device: &cpal::Device) -> Option<AudioDeviceInfo> {
|
||
let id = desktop_device_id(device)?;
|
||
let description = device.description().ok();
|
||
let name = description
|
||
.as_ref()
|
||
.map(|d| d.name().trim().to_owned())
|
||
.filter(|name| !name.is_empty())
|
||
.unwrap_or_else(|| format!("Device {}", &short_device_id(&id)[..8]));
|
||
|
||
let mut details = Vec::new();
|
||
if let Some(description) = description.as_ref() {
|
||
if let Some(manufacturer) = description.manufacturer() {
|
||
details.push(manufacturer.to_owned());
|
||
}
|
||
if let Some(driver) = description.driver() {
|
||
details.push(driver.to_owned());
|
||
}
|
||
let device_type = description.device_type();
|
||
if device_type != cpal::device_description::DeviceType::Unknown {
|
||
details.push(format!("{device_type}"));
|
||
}
|
||
let interface_type = description.interface_type();
|
||
if interface_type != cpal::device_description::InterfaceType::Unknown {
|
||
details.push(format!("{interface_type}"));
|
||
}
|
||
}
|
||
details.push(format!("id={}", short_device_id(&id)));
|
||
|
||
Some(AudioDeviceInfo {
|
||
id,
|
||
name,
|
||
details: details.join(" · "),
|
||
is_default: false,
|
||
})
|
||
}
|
||
|
||
/// Enumerate available audio input and output devices.
|
||
/// On mobile platforms (iOS, Android) returns an empty list because
|
||
/// device selection is managed by the OS audio session.
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
pub fn list_audio_devices() -> AudioDeviceList {
|
||
use cpal::traits::HostTrait;
|
||
let mut list = AudioDeviceList {
|
||
input_devices: Vec::new(),
|
||
output_devices: Vec::new(),
|
||
};
|
||
let host = cpal::default_host();
|
||
let default_in = host
|
||
.default_input_device()
|
||
.and_then(|device| desktop_device_id(&device));
|
||
let default_out = host
|
||
.default_output_device()
|
||
.and_then(|device| desktop_device_id(&device));
|
||
if let Ok(devices) = host.input_devices() {
|
||
for d in devices {
|
||
if let Some(mut device) = describe_device(&d) {
|
||
device.is_default = default_in
|
||
.as_ref()
|
||
.is_some_and(|default_id| default_id == &device.id);
|
||
list.input_devices.push(device);
|
||
}
|
||
}
|
||
}
|
||
if let Ok(devices) = host.output_devices() {
|
||
for d in devices {
|
||
if let Some(mut device) = describe_device(&d) {
|
||
device.is_default = default_out
|
||
.as_ref()
|
||
.is_some_and(|default_id| default_id == &device.id);
|
||
list.output_devices.push(device);
|
||
}
|
||
}
|
||
}
|
||
list
|
||
}
|
||
|
||
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
|
||
/// Enumerate available audio input and output devices.
|
||
pub fn list_audio_devices() -> AudioDeviceList {
|
||
AudioDeviceList {
|
||
input_devices: Vec::new(),
|
||
output_devices: Vec::new(),
|
||
}
|
||
}
|
||
|
||
/// Engine configuration.
|
||
#[derive(Clone)]
|
||
pub struct AudioEngineConfig {
|
||
/// Input gain applied before encoding (1.0 = pass-through).
|
||
pub mic_gain: f32,
|
||
/// Initial PTT state. When false the encoder is bypassed and no
|
||
/// outbound packets are produced.
|
||
pub ptt_initial: bool,
|
||
/// Audio-effect toggles. The struct is honoured by *naming* but
|
||
/// the filters themselves are still no-op in Beta — see the
|
||
/// crate-level docs and DEC-007/008/009/010.
|
||
pub effects: crate::AudioEffects,
|
||
/// A.5 mobile: prefer the OS-provided "voice communication"
|
||
/// audio source on mobile platforms (Android
|
||
/// `MediaRecorder.AudioSource.VOICE_COMMUNICATION`, iOS
|
||
/// `AVAudioSession.Mode.voiceChat`). On Linux desktop this is
|
||
/// ignored — the DSP chain stays a no-op and we use the
|
||
/// default ALSA/PipeWire source.
|
||
///
|
||
/// Android status: this is enforced by the Oboe-only backend,
|
||
/// which opens input with `VoiceCommunication` and output with
|
||
/// voice-communication usage / speech content. Setting `false`
|
||
/// is rejected on Android because the P0 path intentionally has
|
||
/// no generic mobile-audio fallback.
|
||
pub mobile_voice_preset: bool,
|
||
/// Optional input device id override. When `None`, the system
|
||
/// default input device is used. Set to a device id from
|
||
/// [`list_audio_devices`] to pin a specific microphone.
|
||
pub input_device_id: Option<String>,
|
||
/// Optional output device id override.
|
||
pub output_device_id: Option<String>,
|
||
/// Optional selector used by P1 VoiceActivity to publish VAD state.
|
||
#[doc(hidden)]
|
||
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||
}
|
||
|
||
impl std::fmt::Debug for AudioEngineConfig {
|
||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
f.debug_struct("AudioEngineConfig")
|
||
.field("mic_gain", &self.mic_gain)
|
||
.field("ptt_initial", &self.ptt_initial)
|
||
.field("effects", &self.effects)
|
||
.field("mobile_voice_preset", &self.mobile_voice_preset)
|
||
.field("input_device_id", &self.input_device_id)
|
||
.field("output_device_id", &self.output_device_id)
|
||
.field(
|
||
"voice_activity_selector",
|
||
&self.voice_activity_selector.as_ref().map(|_| "present"),
|
||
)
|
||
.finish()
|
||
}
|
||
}
|
||
|
||
impl Default for AudioEngineConfig {
|
||
fn default() -> Self {
|
||
Self {
|
||
mic_gain: 1.0,
|
||
ptt_initial: false,
|
||
effects: crate::AudioEffects::default(),
|
||
mobile_voice_preset: true,
|
||
input_device_id: None,
|
||
output_device_id: None,
|
||
voice_activity_selector: None,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Running audio engine. Drop = stop.
|
||
pub struct AudioEngine {
|
||
/// `transmit_active` is the authoritative gate for outbound
|
||
/// voice — the Opus encoder feed consults this flag once per
|
||
/// 20 ms frame. PTT subsystems (focused widget, future
|
||
/// Windows / macOS / Linux global backends) drive this flag
|
||
/// through [`Self::set_transmit_active`]; nothing else is
|
||
/// permitted to flip it (SAD-075 / SDD-089).
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
frames_sent: Arc<AtomicU32>,
|
||
frames_received: Arc<AtomicU32>,
|
||
/// Master output gain as f32 bits in an AtomicU32. Default 1.0.
|
||
/// Adjusted via [`Self::set_output_gain`] from the bridge.
|
||
output_gain: Arc<AtomicU32>,
|
||
/// Master output mute. When true the output callback fills the
|
||
/// device buffer with silence regardless of incoming voice
|
||
/// frames. Used for self-output-mute on the local device,
|
||
/// independent of the server-side mute the protocol layer
|
||
/// broadcasts.
|
||
output_muted: Arc<AtomicBool>,
|
||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
silero_vad_worker: Arc<Mutex<Option<crate::vad::silero_onnx::SileroOnnxVadWorker>>>,
|
||
#[cfg(not(target_os = "android"))]
|
||
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||
#[cfg(target_os = "android")]
|
||
audio_event_producer: crate::audio_event_queue::AudioEventProducer,
|
||
#[cfg(target_os = "android")]
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
#[cfg(target_os = "android")]
|
||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||
#[cfg(target_os = "android")]
|
||
mic_gain: f32,
|
||
// Streams must be dropped to stop audio. Both are `!Send` because
|
||
// cpal's Stream isn't Send on some backends; we keep them in an
|
||
// Option wrapped by Mutex so stop() can move them out. On Linux
|
||
// the output side is `crate::sdl_output::SdlOutput` instead of a
|
||
// cpal Stream (see the SDD note inside `sdl_output.rs`); the
|
||
// same unsafe Send/Sync impl below covers both. On iOS both
|
||
// sides collapse into a single `IosVoiceUnit` (one
|
||
// VoiceProcessingIO AudioUnit hosts mic + speaker) — cpal is
|
||
// unused on iOS for the reasons documented in
|
||
// `ios_voice_unit.rs`.
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
_input_stream: Mutex<Option<cpal::Stream>>,
|
||
#[cfg(target_os = "linux")]
|
||
_output_stream: Mutex<Option<crate::sdl_output::SdlOutput>>,
|
||
#[cfg(all(
|
||
not(target_os = "linux"),
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
_output_stream: Mutex<Option<cpal::Stream>>,
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
_ios_voice_backend: Mutex<Option<IosVoiceBackend>>,
|
||
/// SDD-111..SDD-115: Android Oboe voice backend. Owns the input
|
||
/// and output streams, SDD-113 hardware-effect handles, and the
|
||
/// foreground-service lifecycle; tearing it down on engine drop
|
||
/// unbinds effects and stops the service in SDD-115 reverse
|
||
/// order.
|
||
#[cfg(target_os = "android")]
|
||
_android_voice_unit: Arc<Mutex<Option<crate::android_voice_unit::AndroidVoiceUnit>>>,
|
||
/// Persist the Android stream request so route-change reopen uses the
|
||
/// same effect and latency policy as the original session start.
|
||
#[cfg(target_os = "android")]
|
||
android_voice_stream_config: Arc<Mutex<crate::mobile_voice_backend::AndroidVoiceStreamConfig>>,
|
||
/// SDD-108 §1/§2: refcount-composable audio-mode controller.
|
||
/// Snapshots `AudioManager.getMode()` on the 0 → 1 transition and
|
||
/// restores it on the 1 → 0 transition. Held in a `Mutex` so the
|
||
/// snapshot/restore critical section is serialized across
|
||
/// composed callers (SDD-108 §2). Engine-scoped because the
|
||
/// mode lifecycle is bound to the voice-session lifecycle
|
||
/// (SDD-108 §3).
|
||
#[cfg(target_os = "android")]
|
||
audio_mode_stack: Arc<Mutex<crate::mode_stack::ModeStack>>,
|
||
// Hand the inbound-voice forwarder task a shutdown signal.
|
||
shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
|
||
/// True if the capture stream actually opened. If false (typical
|
||
/// in headless environments with null sources, or where the user
|
||
/// denied microphone permission), PTT becomes a no-op and
|
||
/// `frames_sent` stays at 0.
|
||
capture_active: bool,
|
||
}
|
||
|
||
// cpal::Stream is not Send. We keep the engine pinned to the thread
|
||
// it was constructed on — `chanora_core` spawns it inside a
|
||
// `tokio::task::spawn_blocking` so the streams stay on that worker.
|
||
// This `unsafe impl Send` is necessary because the outer Arc<AudioEngine>
|
||
// is stored in core's session and must move into a task. The streams
|
||
// themselves are only mutated through the Mutex and are dropped on
|
||
// the same thread that owns them.
|
||
//
|
||
// SAFETY: cpal's Stream is not Send because the underlying audio API
|
||
// callback thread may not be transferable. We never invoke methods on
|
||
// the streams from any thread but the owning one; we only ever *drop*
|
||
// them, which cpal documents as safe from any thread for ALSA and
|
||
// PipeWire (Linux backend used here). For Windows/macOS the contract
|
||
// may differ; production Beta+ work must revisit per-platform.
|
||
unsafe impl Send for AudioEngine {}
|
||
unsafe impl Sync for AudioEngine {}
|
||
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
enum IosVoiceBackend {
|
||
Vpio(crate::ios_voice_unit::IosVoiceUnit),
|
||
}
|
||
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
impl IosVoiceBackend {
|
||
fn restart(&mut self) -> Result<(), AudioError> {
|
||
#[cfg(target_os = "ios")]
|
||
{
|
||
match self {
|
||
Self::Vpio(unit) => unit.restart(),
|
||
}
|
||
}
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
let _ = self;
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
fn pause(&mut self) -> Result<(), AudioError> {
|
||
#[cfg(target_os = "ios")]
|
||
{
|
||
match self {
|
||
Self::Vpio(unit) => unit.pause(),
|
||
}
|
||
}
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
let _ = self;
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
fn resume(&mut self) -> Result<(), AudioError> {
|
||
#[cfg(target_os = "ios")]
|
||
{
|
||
match self {
|
||
Self::Vpio(unit) => unit.resume(),
|
||
}
|
||
}
|
||
#[cfg(target_os = "macos")]
|
||
{
|
||
let _ = self;
|
||
Ok(())
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
fn open_ios_voice_backend(
|
||
params: crate::mobile_voice_backend::VoiceAudioParams,
|
||
) -> Result<IosVoiceBackend, AudioError> {
|
||
let unit = crate::ios_voice_unit::IosVoiceUnit::start(params)?;
|
||
Ok(IosVoiceBackend::Vpio(unit))
|
||
}
|
||
|
||
impl AudioEngine {
|
||
#[cfg(target_os = "android")]
|
||
fn spawn_android_backend_event_task(
|
||
android_voice_unit: Arc<Mutex<Option<crate::android_voice_unit::AndroidVoiceUnit>>>,
|
||
android_voice_stream_config: Arc<
|
||
Mutex<crate::mobile_voice_backend::AndroidVoiceStreamConfig>,
|
||
>,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
frames_sent: Arc<AtomicU32>,
|
||
event_producer: crate::audio_event_queue::AudioEventProducer,
|
||
output_gain: Arc<AtomicU32>,
|
||
output_muted: Arc<AtomicBool>,
|
||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||
mic_gain: f32,
|
||
audio_mode_stack: Arc<Mutex<crate::mode_stack::ModeStack>>,
|
||
mut event_rx: crate::mobile_voice_backend::BackendEventRx,
|
||
) {
|
||
use crate::mobile_voice_backend::BackendEvent;
|
||
|
||
tokio::spawn(async move {
|
||
while let Some(event) = event_rx.recv().await {
|
||
match event {
|
||
BackendEvent::Disconnected => {
|
||
warn!(target: "chanora_audio", "android: backend disconnected; reopening voice unit");
|
||
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
|
||
crate::android_voice_unit::chanora_android_abandon_audio_focus();
|
||
crate::android_voice_unit::clear_global_event_sender();
|
||
|
||
if let Some(mut unit) = android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() {
|
||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||
if let Err(e) = unit.close() {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"android: failed to close disconnected voice unit before reopen"
|
||
);
|
||
}
|
||
}
|
||
|
||
if crate::android_voice_unit::chanora_android_start_voice_service() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: voice foreground service restart dispatched after backend disconnect"
|
||
);
|
||
}
|
||
|
||
match android_get_audio_mode() {
|
||
Ok(current_mode) => {
|
||
if current_mode != ANDROID_MODE_IN_COMMUNICATION {
|
||
match android_set_audio_mode(ANDROID_MODE_IN_COMMUNICATION) {
|
||
Ok(()) => info!(
|
||
target: "chanora_audio",
|
||
prior_mode = current_mode,
|
||
"android: AudioManager mode re-engaged after backend disconnect"
|
||
),
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
prior_mode = current_mode,
|
||
"android: failed to re-engage MODE_IN_COMMUNICATION after backend disconnect"
|
||
),
|
||
}
|
||
}
|
||
}
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"android: AudioManager.getMode failed during backend reopen"
|
||
),
|
||
}
|
||
|
||
let cfg_av = android_voice_stream_config.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
||
let params = crate::mobile_voice_backend::VoiceAudioParams {
|
||
voice_out_tx: voice_out_tx.clone(),
|
||
transmit_active: transmit_gate.flag_arc(),
|
||
frames_sent: frames_sent.clone(),
|
||
mic_gain,
|
||
handler: AudioHandler::new(),
|
||
event_producer: event_producer.clone(),
|
||
output_gain: output_gain.clone(),
|
||
output_muted: output_muted.clone(),
|
||
voice_activity_selector: voice_activity_selector.clone(),
|
||
audio_processing_config: audio_processing_config.clone(),
|
||
audio_processing_stats: audio_processing_stats.clone(),
|
||
};
|
||
|
||
match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) {
|
||
Ok(mut reopened) => {
|
||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||
match reopened.start() {
|
||
Ok(()) => {
|
||
if let Some(next_rx) = reopened.take_event_rx() {
|
||
Self::spawn_android_backend_event_task(
|
||
android_voice_unit.clone(),
|
||
android_voice_stream_config.clone(),
|
||
voice_out_tx.clone(),
|
||
transmit_gate.clone(),
|
||
frames_sent.clone(),
|
||
event_producer.clone(),
|
||
output_gain.clone(),
|
||
output_muted.clone(),
|
||
voice_activity_selector.clone(),
|
||
audio_processing_config.clone(),
|
||
audio_processing_stats.clone(),
|
||
mic_gain,
|
||
audio_mode_stack.clone(),
|
||
next_rx,
|
||
);
|
||
}
|
||
crate::android_voice_unit::register_global_event_sender(
|
||
reopened.event_sender(),
|
||
);
|
||
if crate::android_voice_unit::chanora_android_request_audio_focus() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: audio focus re-requested after backend disconnect"
|
||
);
|
||
}
|
||
if crate::android_voice_unit::chanora_android_start_bluetooth_sco() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: bluetooth route re-engaged after backend disconnect"
|
||
);
|
||
}
|
||
*android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()) = Some(reopened);
|
||
}
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"android: reopened voice unit failed to start after backend disconnect"
|
||
),
|
||
}
|
||
}
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"android: failed to reopen voice unit after backend disconnect"
|
||
),
|
||
}
|
||
}
|
||
BackendEvent::FocusLost => {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
"android: audio focus lost permanently (SDD-109); engine should leave session"
|
||
);
|
||
}
|
||
BackendEvent::FocusTransient => {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: transient audio focus loss (SDD-109); pausing capture"
|
||
);
|
||
}
|
||
BackendEvent::FocusTransientCanDuck => {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: transient audio focus loss with ducking (SDD-109); continuing"
|
||
);
|
||
}
|
||
BackendEvent::FocusGain => {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: audio focus regained (SDD-109); resuming capture"
|
||
);
|
||
}
|
||
BackendEvent::BluetoothScoStateChanged(s) => {
|
||
info!(
|
||
target: "chanora_audio",
|
||
sco_state = s,
|
||
"android: Bluetooth SCO state changed (SDD-110)"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
});
|
||
}
|
||
|
||
/// Start the engine: open capture + playback streams, spawn the
|
||
/// inbound-voice forwarder, return a handle.
|
||
pub fn start(
|
||
cfg: AudioEngineConfig,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||
) -> Result<Self, AudioError> {
|
||
let gate = crate::ptt::AudioTransmitGate::new(cfg.ptt_initial);
|
||
Self::start_with_gate(cfg, voice_out_tx, voice_in_rx, gate)
|
||
}
|
||
|
||
/// Start the engine using an externally-owned
|
||
/// [`AudioTransmitGate`]. The gate is shared with whatever
|
||
/// upstream (typically [`crate::TransmitModeSelector`]) is the
|
||
/// authoritative writer of `transmit_active`. See SAD-083.
|
||
pub fn start_with_gate(
|
||
cfg: AudioEngineConfig,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
) -> Result<Self, AudioError> {
|
||
Self::start_with_gate_platform(cfg, voice_out_tx, voice_in_rx, transmit_gate)
|
||
}
|
||
|
||
// Apple platforms route to a separate backend (VoiceProcessingIO
|
||
// via coreaudio-rs) because cpal does not expose the native
|
||
// voice-processing AudioUnit controls Chanora needs for VoIP.
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
fn start_with_gate_platform(
|
||
cfg: AudioEngineConfig,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
) -> Result<Self, AudioError> {
|
||
Self::start_with_gate_ios(cfg, voice_out_tx, voice_in_rx, transmit_gate)
|
||
}
|
||
|
||
#[cfg(target_os = "android")]
|
||
fn start_with_gate_platform(
|
||
cfg: AudioEngineConfig,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
) -> Result<Self, AudioError> {
|
||
Self::start_with_gate_android(cfg, voice_out_tx, voice_in_rx, transmit_gate)
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
fn start_with_gate_platform(
|
||
cfg: AudioEngineConfig,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
) -> Result<Self, AudioError> {
|
||
Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate)
|
||
}
|
||
|
||
/// Non-Apple/non-Android implementation: cpal capture + (cpal | SDL2) output.
|
||
/// Kept as a separate function so the Apple and Android paths can
|
||
/// short-circuit at the top of `start_with_gate` without dragging a 200-line
|
||
/// cfg-gated block. Body is the pre-iOS-port code, unchanged
|
||
/// except for the new function name + signature.
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
fn start_with_gate_cpal(
|
||
cfg: AudioEngineConfig,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
mut voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
) -> Result<Self, AudioError> {
|
||
let host = cpal::default_host();
|
||
info!(
|
||
target: "chanora_audio",
|
||
host_id = ?host.id(),
|
||
"starting audio engine: cpal host selected"
|
||
);
|
||
|
||
/// Helper: find a device by stable id, falling back to default.
|
||
fn find_device<DefaultFn, AllFn, Devices>(
|
||
host: &cpal::Host,
|
||
default_fn: DefaultFn,
|
||
all_fn: AllFn,
|
||
prefer: Option<&str>,
|
||
) -> Option<cpal::Device>
|
||
where
|
||
DefaultFn: Fn(&cpal::Host) -> Option<cpal::Device>,
|
||
AllFn: Fn(&cpal::Host) -> Result<Devices, cpal::Error>,
|
||
Devices: IntoIterator<Item = cpal::Device>,
|
||
{
|
||
if let Some(id) = prefer {
|
||
if let Ok(devices) = all_fn(host) {
|
||
for d in devices {
|
||
if desktop_device_id(&d).as_deref() == Some(id) {
|
||
return Some(d);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
default_fn(host)
|
||
}
|
||
|
||
let in_dev = find_device(
|
||
&host,
|
||
cpal::Host::default_input_device,
|
||
cpal::Host::input_devices,
|
||
cfg.input_device_id.as_deref(),
|
||
)
|
||
.ok_or(AudioError::NoInputDevice)?;
|
||
|
||
let out_dev = find_device(
|
||
&host,
|
||
cpal::Host::default_output_device,
|
||
cpal::Host::output_devices,
|
||
cfg.output_device_id.as_deref(),
|
||
)
|
||
.ok_or(AudioError::NoOutputDevice)?;
|
||
|
||
info!(
|
||
target: "chanora_audio",
|
||
in_device = %in_dev.description().map(|d| d.name().to_owned()).unwrap_or_default(),
|
||
out_device = %out_dev.description().map(|d| d.name().to_owned()).unwrap_or_default(),
|
||
"starting audio engine"
|
||
);
|
||
|
||
// Log the cpal-reported default configs *before* trying to
|
||
// open streams, so a downstream stream-build failure can
|
||
// be cross-referenced against what the platform reported
|
||
// as its default format. Some locale / driver combinations
|
||
// on Windows have been observed to expose configs that
|
||
// accept device enumeration but reject `default_*_config`
|
||
// afterwards (reported on the ko-KR Windows 11 host as
|
||
// "Start audio button not work"). Promote what would
|
||
// otherwise be silent or laconic errors into structured
|
||
// log records the user can paste back.
|
||
match in_dev.default_input_config() {
|
||
Ok(c) => info!(
|
||
target: "chanora_audio",
|
||
channels = c.channels(),
|
||
sample_rate = c.sample_rate(),
|
||
sample_format = ?c.sample_format(),
|
||
"default_input_config reported"
|
||
),
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"default_input_config FAILED — capture will be disabled"
|
||
),
|
||
}
|
||
match out_dev.default_output_config() {
|
||
Ok(c) => info!(
|
||
target: "chanora_audio",
|
||
channels = c.channels(),
|
||
sample_rate = c.sample_rate(),
|
||
sample_format = ?c.sample_format(),
|
||
"default_output_config reported"
|
||
),
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"default_output_config FAILED — output stream will fail to build"
|
||
),
|
||
}
|
||
|
||
let transmit_flag_for_capture = transmit_gate.flag_arc();
|
||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||
let frames_received = Arc::new(AtomicU32::new(0));
|
||
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
|
||
let output_muted = Arc::new(AtomicBool::new(false));
|
||
let (audio_processing_config, audio_processing_stats, silero_vad_worker) =
|
||
new_desktop_audio_processing_state();
|
||
|
||
// ---------- Capture ----------
|
||
// Capture is best-effort. If the platform default input
|
||
// device refuses any supported config (typical for
|
||
// headless null sources or for users who deny the mic
|
||
// permission) we log and continue — playback alone is
|
||
// still useful. PTT becomes a no-op in that case.
|
||
let capture_result = try_open_capture(
|
||
&in_dev,
|
||
voice_out_tx,
|
||
transmit_flag_for_capture,
|
||
frames_sent.clone(),
|
||
cfg.mic_gain,
|
||
cfg.voice_activity_selector.clone(),
|
||
audio_processing_config.clone(),
|
||
silero_vad_worker.clone(),
|
||
audio_processing_stats.clone(),
|
||
);
|
||
let (input_stream, capture_active) = match capture_result {
|
||
Ok(s) => (Some(s), true),
|
||
Err(e) => {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"capture stream unavailable; continuing with playback only"
|
||
);
|
||
(None, false)
|
||
}
|
||
};
|
||
if let Some(s) = &input_stream {
|
||
s.play()
|
||
.map_err(|e| AudioError::Backend(format!("input play: {e}")))?;
|
||
}
|
||
|
||
// ---------- Playback ----------
|
||
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
|
||
Arc::new(Mutex::new(AudioHandler::new()));
|
||
|
||
// Linux uses SDL2 for output (Qint / upstream tsclientlib
|
||
// pattern). cpal's Linux backend opens raw ALSA which routes
|
||
// through `dmix`+`plug` and produces audible crackling /
|
||
// popping on the 48 kHz → device-rate step. SDL2 on the same
|
||
// box routes through PipeWire's PA bridge (or PulseAudio)
|
||
// whose resampler is high-quality. We keep cpal on Windows
|
||
// and macOS — both have native backends (WASAPI / CoreAudio)
|
||
// without this problem. See `crates/chanora_audio/src/sdl_output.rs`
|
||
// for the full rationale.
|
||
#[cfg(target_os = "linux")]
|
||
let output_stream = crate::sdl_output::SdlOutput::start(
|
||
audio_handler.clone(),
|
||
output_gain.clone(),
|
||
output_muted.clone(),
|
||
)?;
|
||
|
||
#[cfg(not(target_os = "linux"))]
|
||
let output_stream = {
|
||
let out_cfg = out_dev
|
||
.default_output_config()
|
||
.map_err(|e| AudioError::StreamConfig(format!("output default: {e}")))?;
|
||
let out_format = out_cfg.sample_format();
|
||
let dev_sample_rate = out_cfg.sample_rate();
|
||
let dev_channels = out_cfg.channels() as usize;
|
||
// Buffer-size rationale:
|
||
// * Windows (WASAPI via cpal): the default period is
|
||
// small enough to expose audio-thread scheduler
|
||
// jitter on shared-mode endpoints. Pinning at 2048
|
||
// frames (~46 ms @ 44.1 kHz) gives the Opus decode
|
||
// callback enough headroom while still being well
|
||
// under voice-chat latency tolerance.
|
||
// * Apple platforms do not reach this cpal path; they
|
||
// use direct VoiceProcessingIO AudioUnits.
|
||
#[cfg(target_os = "windows")]
|
||
let buffer_size = cpal::BufferSize::Fixed(2048);
|
||
#[cfg(not(target_os = "windows"))]
|
||
let buffer_size = cpal::BufferSize::Default;
|
||
let out_stream_cfg = cpal::StreamConfig {
|
||
channels: out_cfg.channels(),
|
||
sample_rate: out_cfg.sample_rate(),
|
||
buffer_size,
|
||
};
|
||
info!(
|
||
target: "chanora_audio",
|
||
dev_sample_rate,
|
||
dev_channels,
|
||
buffer_size = ?buffer_size,
|
||
"output stream using device native config (no 48k force)"
|
||
);
|
||
|
||
let stream = match out_format {
|
||
SampleFormat::F32 => build_output_stream::<f32>(
|
||
&out_dev,
|
||
&out_stream_cfg,
|
||
audio_handler.clone(),
|
||
output_gain.clone(),
|
||
output_muted.clone(),
|
||
dev_sample_rate,
|
||
dev_channels,
|
||
)?,
|
||
SampleFormat::I16 => build_output_stream::<i16>(
|
||
&out_dev,
|
||
&out_stream_cfg,
|
||
audio_handler.clone(),
|
||
output_gain.clone(),
|
||
output_muted.clone(),
|
||
dev_sample_rate,
|
||
dev_channels,
|
||
)?,
|
||
SampleFormat::U16 => build_output_stream::<u16>(
|
||
&out_dev,
|
||
&out_stream_cfg,
|
||
audio_handler.clone(),
|
||
output_gain.clone(),
|
||
output_muted.clone(),
|
||
dev_sample_rate,
|
||
dev_channels,
|
||
)?,
|
||
other => {
|
||
return Err(AudioError::StreamConfig(format!(
|
||
"unsupported output format: {other:?}"
|
||
)))
|
||
}
|
||
};
|
||
stream
|
||
.play()
|
||
.map_err(|e| AudioError::Backend(format!("output play: {e}")))?;
|
||
stream
|
||
};
|
||
|
||
// ---------- Inbound forwarder ----------
|
||
let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
|
||
let handler_for_task = audio_handler.clone();
|
||
let frames_received_for_task = frames_received.clone();
|
||
tokio::spawn(async move {
|
||
loop {
|
||
tokio::select! {
|
||
_ = &mut shutdown_rx => {
|
||
debug!(target: "chanora_audio", "inbound forwarder shutting down");
|
||
break;
|
||
}
|
||
item = voice_in_rx.recv() => {
|
||
match item {
|
||
Some(v) => {
|
||
let id = SessionAudioId(v.from_client);
|
||
let mut h = handler_for_task.lock().unwrap_or_else(|e| e.into_inner());
|
||
if let Err(e) = h.handle_packet(id, v.packet) {
|
||
debug!(target: "chanora_audio", error = %e, "decode failed");
|
||
} else {
|
||
frames_received_for_task.fetch_add(1, Ordering::Relaxed);
|
||
}
|
||
}
|
||
None => break,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
Ok(Self {
|
||
transmit_gate,
|
||
frames_sent,
|
||
frames_received,
|
||
output_gain,
|
||
output_muted,
|
||
audio_processing_config,
|
||
audio_processing_stats,
|
||
silero_vad_worker,
|
||
audio_handler,
|
||
_input_stream: Mutex::new(input_stream),
|
||
_output_stream: Mutex::new(Some(output_stream)),
|
||
shutdown_tx: Some(shutdown_tx),
|
||
capture_active,
|
||
})
|
||
}
|
||
|
||
#[cfg(target_os = "android")]
|
||
fn start_with_gate_android(
|
||
cfg: AudioEngineConfig,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
mut voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
) -> Result<Self, AudioError> {
|
||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||
|
||
info!(target: "chanora_audio", "starting audio engine: Android Oboe backend");
|
||
|
||
let transmit_flag_for_capture = transmit_gate.flag_arc();
|
||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||
let frames_received = Arc::new(AtomicU32::new(0));
|
||
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
|
||
let output_muted = Arc::new(AtomicBool::new(false));
|
||
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
|
||
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||
|
||
let event_queue = AudioEventQueue::new();
|
||
let event_producer = AudioEventQueue::producer(&event_queue);
|
||
|
||
if !cfg.mobile_voice_preset {
|
||
return Err(AudioError::Backend(
|
||
"android: mobile_voice_preset=false is unsupported for Oboe-only backend"
|
||
.to_string(),
|
||
));
|
||
}
|
||
|
||
let mut audio_mode_stack = crate::mode_stack::ModeStack::new();
|
||
|
||
if crate::android_voice_unit::chanora_android_start_voice_service() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: voice foreground service start dispatched (SDD-115)"
|
||
);
|
||
} else {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
"android: foreground service start failed; capture may be denied in background (SDD-115)"
|
||
);
|
||
}
|
||
|
||
match android_get_audio_mode() {
|
||
Ok(prior_now) => {
|
||
let outcome = audio_mode_stack.acquire(prior_now);
|
||
if let crate::mode_stack::ModeAcquire::FirstAcquire { prior } = outcome {
|
||
match android_set_audio_mode(ANDROID_MODE_IN_COMMUNICATION) {
|
||
Ok(()) => info!(
|
||
target: "chanora_audio",
|
||
prior_mode = prior,
|
||
"android: AudioManager mode set to MODE_IN_COMMUNICATION (SDD-108)"
|
||
),
|
||
Err(e) => {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
prior_mode = prior,
|
||
"android: setMode failed; rolling back ModeStack acquire (SDD-108 §5)"
|
||
);
|
||
let _ = audio_mode_stack.release();
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"android: AudioManager.getMode failed; skipping mode engage (SDD-108)"
|
||
),
|
||
}
|
||
|
||
let cfg_av = crate::mobile_voice_backend::AndroidVoiceStreamConfig {
|
||
effects: cfg.effects,
|
||
..Default::default()
|
||
};
|
||
let params = crate::mobile_voice_backend::VoiceAudioParams {
|
||
voice_out_tx: voice_out_tx.clone(),
|
||
transmit_active: transmit_flag_for_capture,
|
||
frames_sent: frames_sent.clone(),
|
||
mic_gain: cfg.mic_gain,
|
||
handler: AudioHandler::new(),
|
||
event_producer: event_producer.clone(),
|
||
output_gain: output_gain.clone(),
|
||
output_muted: output_muted.clone(),
|
||
voice_activity_selector: cfg.voice_activity_selector.clone(),
|
||
audio_processing_config: audio_processing_config.clone(),
|
||
audio_processing_stats: audio_processing_stats.clone(),
|
||
};
|
||
let mut android_voice_unit =
|
||
match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params) {
|
||
Ok(unit) => unit,
|
||
Err(e) => {
|
||
Self::rollback_android_startup_resources(&mut audio_mode_stack);
|
||
return Err(AudioError::Backend(format!(
|
||
"android: failed to open Oboe voice unit: {e}"
|
||
)));
|
||
}
|
||
};
|
||
|
||
if let Err(e) = android_voice_unit.start() {
|
||
if let Err(close_err) = android_voice_unit.close() {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
error = %close_err,
|
||
"android: AndroidVoiceUnit::close failed during startup rollback"
|
||
);
|
||
}
|
||
Self::rollback_android_startup_resources(&mut audio_mode_stack);
|
||
return Err(AudioError::Backend(format!(
|
||
"android: failed to start Oboe voice unit: {e}"
|
||
)));
|
||
}
|
||
|
||
let android_voice_unit = Arc::new(Mutex::new(Some(android_voice_unit)));
|
||
let android_voice_stream_config = Arc::new(Mutex::new(cfg_av.clone()));
|
||
let audio_mode_stack = Arc::new(Mutex::new(audio_mode_stack));
|
||
|
||
if let Some(event_rx) = android_voice_unit
|
||
.lock()
|
||
.unwrap()
|
||
.as_mut()
|
||
.and_then(|unit| unit.take_event_rx())
|
||
{
|
||
Self::spawn_android_backend_event_task(
|
||
android_voice_unit.clone(),
|
||
android_voice_stream_config.clone(),
|
||
voice_out_tx.clone(),
|
||
transmit_gate.clone(),
|
||
frames_sent.clone(),
|
||
event_producer.clone(),
|
||
output_gain.clone(),
|
||
output_muted.clone(),
|
||
cfg.voice_activity_selector.clone(),
|
||
audio_processing_config.clone(),
|
||
audio_processing_stats.clone(),
|
||
cfg.mic_gain,
|
||
audio_mode_stack.clone(),
|
||
event_rx,
|
||
);
|
||
}
|
||
|
||
crate::android_voice_unit::register_global_event_sender(
|
||
android_voice_unit
|
||
.lock()
|
||
.unwrap()
|
||
.as_ref()
|
||
.expect("android voice unit installed")
|
||
.event_sender(),
|
||
);
|
||
|
||
if crate::android_voice_unit::chanora_android_request_audio_focus() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: audio focus requested (SDD-109)"
|
||
);
|
||
} else {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
"android: audio focus request failed; engine operates without focus (SDD-109)"
|
||
);
|
||
}
|
||
|
||
if crate::android_voice_unit::chanora_android_start_bluetooth_sco() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: Bluetooth SCO started (SDD-110)"
|
||
);
|
||
} else {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
"android: Bluetooth SCO start failed; BT HFP may not route correctly (SDD-110)"
|
||
);
|
||
}
|
||
|
||
let capture_active = true;
|
||
|
||
let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
|
||
let event_producer_for_task = event_producer.clone();
|
||
let frames_received_for_task = frames_received.clone();
|
||
tokio::spawn(async move {
|
||
loop {
|
||
tokio::select! {
|
||
_ = &mut shutdown_rx => {
|
||
debug!(target: "chanora_audio", "inbound forwarder shutting down");
|
||
break;
|
||
}
|
||
item = voice_in_rx.recv() => {
|
||
match item {
|
||
Some(v) => {
|
||
let id = SessionAudioId(v.from_client);
|
||
let packet = AudioPacket { client_id: id, data: v.packet };
|
||
if event_producer_for_task.push_packet(packet).is_ok() {
|
||
frames_received_for_task.fetch_add(1, Ordering::Relaxed);
|
||
}
|
||
}
|
||
None => break,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
Ok(Self {
|
||
transmit_gate,
|
||
frames_sent,
|
||
frames_received,
|
||
output_gain,
|
||
output_muted,
|
||
audio_processing_config,
|
||
audio_processing_stats,
|
||
audio_event_producer: event_producer,
|
||
voice_out_tx,
|
||
voice_activity_selector: cfg.voice_activity_selector.clone(),
|
||
mic_gain: cfg.mic_gain,
|
||
_android_voice_unit: android_voice_unit,
|
||
android_voice_stream_config,
|
||
audio_mode_stack,
|
||
shutdown_tx: Some(shutdown_tx),
|
||
capture_active,
|
||
})
|
||
}
|
||
|
||
/// Apple implementation: a single VoiceProcessingIO AudioUnit
|
||
/// drives mic capture + speaker playback (see
|
||
/// `ios_voice_unit.rs` for why cpal is unsuitable on iOS).
|
||
/// This mirrors `start_with_gate_cpal` in scaffolding —
|
||
/// atomics, audio handler, inbound forwarder task — but
|
||
/// replaces the two cpal stream constructions with a single
|
||
/// `IosVoiceUnit::start` call.
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
fn start_with_gate_ios(
|
||
cfg: AudioEngineConfig,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
mut voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||
) -> Result<Self, AudioError> {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"starting audio engine: Apple VoiceProcessingIO backend"
|
||
);
|
||
|
||
// Apple audio-session configuration is performed Swift-side
|
||
// in `apps/chanora_flutter/ios/Runner/AppDelegate.swift`
|
||
// BEFORE Flutter starts its audio pipeline. The category +
|
||
// mode pair set there (`.playAndRecord` + `.default`) is
|
||
// what VPIO binds against. Logging here just records that
|
||
// the engine-start path acknowledges the request; the
|
||
// actual session mutation lives in Swift because it must
|
||
// happen before Dart loads.
|
||
if cfg.mobile_voice_preset {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"ios: voice-chat session mode requested — AVAudioSession configured in AppDelegate"
|
||
);
|
||
}
|
||
|
||
let transmit_flag_for_capture = transmit_gate.flag_arc();
|
||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||
let frames_received = Arc::new(AtomicU32::new(0));
|
||
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
|
||
let output_muted = Arc::new(AtomicBool::new(false));
|
||
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
|
||
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
|
||
Arc::new(Mutex::new(AudioHandler::new()));
|
||
let voice_out_tx_for_backend = voice_out_tx.clone();
|
||
|
||
// Construct the live iOS voice backend. Platform VPIO stays
|
||
// the default shipping path; Sonora/RemoteIO remains opt-in.
|
||
let ios_voice_backend =
|
||
open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams {
|
||
handler: audio_handler.clone(),
|
||
output_gain: output_gain.clone(),
|
||
output_muted: output_muted.clone(),
|
||
voice_out_tx: voice_out_tx_for_backend,
|
||
transmit_active: transmit_flag_for_capture,
|
||
frames_sent: frames_sent.clone(),
|
||
mic_gain: cfg.mic_gain,
|
||
voice_activity_selector: cfg.voice_activity_selector.clone(),
|
||
audio_processing_config: audio_processing_config.clone(),
|
||
audio_processing_stats: audio_processing_stats.clone(),
|
||
})?;
|
||
|
||
// Capture is always considered active on iOS — VPIO's
|
||
// input element is wired up by the AudioUnit itself, no
|
||
// separate "did the capture stream open" question to
|
||
// answer. If the user denied mic permission VPIO will
|
||
// simply hand us silence buffers.
|
||
let capture_active = true;
|
||
|
||
// Inbound forwarder: same shape as the non-iOS path. Pumps
|
||
// Opus packets from `voice_in_rx` into AudioHandler so the
|
||
// VPIO render callback (commit 4) finds decoded frames
|
||
// waiting.
|
||
let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
|
||
let frames_received_for_task = frames_received.clone();
|
||
let handler_for_task = audio_handler.clone();
|
||
tokio::spawn(async move {
|
||
loop {
|
||
tokio::select! {
|
||
_ = &mut shutdown_rx => {
|
||
debug!(target: "chanora_audio", "inbound forwarder shutting down");
|
||
break;
|
||
}
|
||
item = voice_in_rx.recv() => {
|
||
match item {
|
||
Some(v) => {
|
||
let id = SessionAudioId(v.from_client);
|
||
let mut h = handler_for_task.lock().unwrap_or_else(|e| e.into_inner());
|
||
let res = h.handle_packet(id, v.packet);
|
||
drop(h);
|
||
match res {
|
||
Ok(_) => {
|
||
frames_received_for_task.fetch_add(1, Ordering::Relaxed);
|
||
}
|
||
Err(e) => {
|
||
debug!(target: "chanora_audio", error = %e, "decode failed");
|
||
}
|
||
}
|
||
}
|
||
None => break,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
Ok(Self {
|
||
transmit_gate,
|
||
frames_sent,
|
||
frames_received,
|
||
output_gain,
|
||
output_muted,
|
||
audio_processing_config,
|
||
audio_processing_stats,
|
||
audio_handler,
|
||
_ios_voice_backend: Mutex::new(Some(ios_voice_backend)),
|
||
shutdown_tx: Some(shutdown_tx),
|
||
capture_active,
|
||
})
|
||
}
|
||
|
||
#[cfg(target_os = "android")]
|
||
fn rollback_android_startup_resources(audio_mode_stack: &mut crate::mode_stack::ModeStack) {
|
||
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
|
||
crate::android_voice_unit::chanora_android_abandon_audio_focus();
|
||
match release_android_audio_mode_for_startup_rollback(audio_mode_stack) {
|
||
crate::mode_stack::ModeRelease::LastRelease { prior } => {
|
||
match android_set_audio_mode(prior) {
|
||
Ok(()) => info!(
|
||
target: "chanora_audio",
|
||
restored_mode = prior,
|
||
"android: AudioManager mode restored during startup rollback"
|
||
),
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
restored_mode = prior,
|
||
"android: failed to restore AudioManager mode during startup rollback"
|
||
),
|
||
}
|
||
}
|
||
crate::mode_stack::ModeRelease::StillHeld => {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: audio mode still held during startup rollback"
|
||
);
|
||
}
|
||
crate::mode_stack::ModeRelease::AlreadyReleased => {}
|
||
}
|
||
|
||
if crate::android_voice_unit::chanora_android_stop_voice_service() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: voice foreground service stopped during startup rollback"
|
||
);
|
||
}
|
||
}
|
||
|
||
/// Stop the engine. Idempotent.
|
||
pub fn stop(&mut self) {
|
||
if let Some(tx) = self.shutdown_tx.take() {
|
||
let _ = tx.send(());
|
||
}
|
||
// Drop the streams, which stops their callback threads.
|
||
// Each platform has a slightly different backend; the
|
||
// common contract is that dropping the wrapper stops
|
||
// audio. iOS collapses input + output into one
|
||
// `IosVoiceUnit` (see `ios_voice_unit.rs`); every other
|
||
// platform has separate cpal input + cpal/SDL output.
|
||
#[cfg(all(
|
||
not(target_os = "ios"),
|
||
not(target_os = "macos"),
|
||
not(target_os = "android")
|
||
))]
|
||
{
|
||
let _ = self._input_stream.lock().unwrap_or_else(|e| e.into_inner()).take();
|
||
let _ = self._output_stream.lock().unwrap_or_else(|e| e.into_inner()).take();
|
||
}
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
{
|
||
let _ = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner()).take();
|
||
}
|
||
// SDD-115 reverse-order teardown on Android:
|
||
// 1) stop Bluetooth SCO + abandon audio focus;
|
||
// 2) close the voice unit (releases SDD-113 hardware
|
||
// effects then stops + closes the Oboe streams);
|
||
// 3) restore the prior audio mode (SDD-108 §1 on 1 → 0
|
||
// transition);
|
||
// 4) stop the foreground service.
|
||
#[cfg(target_os = "android")]
|
||
{
|
||
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
|
||
crate::android_voice_unit::chanora_android_abandon_audio_focus();
|
||
crate::android_voice_unit::clear_global_event_sender();
|
||
if let Some(mut unit) = self._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() {
|
||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||
if let Err(e) = unit.close() {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
"android: AndroidVoiceUnit::close failed (SDD-115)"
|
||
);
|
||
}
|
||
}
|
||
// SDD-108 §1/§2: release the audio-mode stack. Only the
|
||
// 1 → 0 transition writes the platform; mid-stack
|
||
// releases stay engaged. Underflow (release without a
|
||
// matching acquire) is clamped without panic per
|
||
// SDD-108 §1.
|
||
// SDD-108 §5: tolerate a poisoned mutex on the teardown
|
||
// path — if a panicking thread held the lock, we still
|
||
// need to drive the release to completion (otherwise the
|
||
// platform stays in MODE_IN_COMMUNICATION).
|
||
let release = self
|
||
.audio_mode_stack
|
||
.lock()
|
||
.unwrap_or_else(|e| e.into_inner())
|
||
.release();
|
||
match release {
|
||
crate::mode_stack::ModeRelease::LastRelease { prior } => {
|
||
match android_set_audio_mode(prior) {
|
||
Ok(()) => info!(
|
||
target: "chanora_audio",
|
||
restored_mode = prior,
|
||
"android: AudioManager mode restored (SDD-108)"
|
||
),
|
||
Err(e) => warn!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
restored_mode = prior,
|
||
"android: failed to restore prior AudioManager mode (SDD-108)"
|
||
),
|
||
}
|
||
}
|
||
crate::mode_stack::ModeRelease::StillHeld => {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: audio mode still held by composed session (SDD-108)"
|
||
);
|
||
}
|
||
crate::mode_stack::ModeRelease::AlreadyReleased => {
|
||
// No engage ever happened (e.g. mobile_voice_preset
|
||
// was false, or getMode failed). Silent no-op.
|
||
}
|
||
}
|
||
if crate::android_voice_unit::chanora_android_stop_voice_service() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: voice foreground service stop dispatched (SDD-115)"
|
||
);
|
||
}
|
||
}
|
||
info!(target: "chanora_audio", "audio engine stopped");
|
||
}
|
||
|
||
/// iOS-only: restart the underlying VoiceProcessingIO unit after
|
||
/// route changes.
|
||
pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> {
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
{
|
||
let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner());
|
||
let backend = guard
|
||
.as_mut()
|
||
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
|
||
backend.restart()
|
||
}
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||
{
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// Android-only: reopen the underlying Oboe voice backend after a
|
||
/// route/device change while preserving the engine-owned state.
|
||
pub fn android_restart_voice_unit(&self) -> Result<(), AudioError> {
|
||
#[cfg(target_os = "android")]
|
||
{
|
||
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
|
||
|
||
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
|
||
crate::android_voice_unit::chanora_android_abandon_audio_focus();
|
||
crate::android_voice_unit::clear_global_event_sender();
|
||
|
||
if let Some(mut unit) = self._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner()).take() {
|
||
unit.close().map_err(|e| {
|
||
AudioError::Backend(format!(
|
||
"android: failed to close Oboe voice unit during route restart: {e}"
|
||
))
|
||
})?;
|
||
}
|
||
|
||
if crate::android_voice_unit::chanora_android_start_voice_service() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: voice foreground service restart dispatched after route change"
|
||
);
|
||
}
|
||
|
||
if crate::android_voice_unit::chanora_android_request_audio_focus() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: audio focus re-requested after route change"
|
||
);
|
||
}
|
||
|
||
let cfg_av = self.android_voice_stream_config.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
||
let params = crate::mobile_voice_backend::VoiceAudioParams {
|
||
voice_out_tx: self.voice_out_tx.clone(),
|
||
transmit_active: self.transmit_gate.flag_arc(),
|
||
frames_sent: self.frames_sent.clone(),
|
||
mic_gain: self.mic_gain,
|
||
handler: AudioHandler::new(),
|
||
event_producer: self.audio_event_producer.clone(),
|
||
output_gain: self.output_gain.clone(),
|
||
output_muted: self.output_muted.clone(),
|
||
voice_activity_selector: self.voice_activity_selector.clone(),
|
||
audio_processing_config: self.audio_processing_config.clone(),
|
||
audio_processing_stats: self.audio_processing_stats.clone(),
|
||
};
|
||
let mut reopened = crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params)
|
||
.map_err(|e| {
|
||
AudioError::Backend(format!("android: failed to reopen Oboe voice unit: {e}"))
|
||
})?;
|
||
reopened.start().map_err(|e| {
|
||
AudioError::Backend(format!("android: failed to restart Oboe voice unit: {e}"))
|
||
})?;
|
||
if let Some(event_rx) = reopened.take_event_rx() {
|
||
Self::spawn_android_backend_event_task(
|
||
self._android_voice_unit.clone(),
|
||
self.android_voice_stream_config.clone(),
|
||
self.voice_out_tx.clone(),
|
||
self.transmit_gate.clone(),
|
||
self.frames_sent.clone(),
|
||
self.audio_event_producer.clone(),
|
||
self.output_gain.clone(),
|
||
self.output_muted.clone(),
|
||
self.voice_activity_selector.clone(),
|
||
self.audio_processing_config.clone(),
|
||
self.audio_processing_stats.clone(),
|
||
self.mic_gain,
|
||
self.audio_mode_stack.clone(),
|
||
event_rx,
|
||
);
|
||
}
|
||
crate::android_voice_unit::register_global_event_sender(reopened.event_sender());
|
||
|
||
if crate::android_voice_unit::chanora_android_start_bluetooth_sco() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
"android: bluetooth route re-engaged after route change"
|
||
);
|
||
}
|
||
|
||
let mut guard = self._android_voice_unit.lock().unwrap_or_else(|e| e.into_inner());
|
||
*guard = Some(reopened);
|
||
Ok(())
|
||
}
|
||
#[cfg(not(target_os = "android"))]
|
||
{
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// iOS-only: pause the underlying VoiceProcessingIO unit.
|
||
pub fn ios_pause_voice_unit(&self) -> Result<(), AudioError> {
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
{
|
||
let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner());
|
||
let unit = guard
|
||
.as_mut()
|
||
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
|
||
unit.pause()
|
||
}
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||
{
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// iOS-only: resume the underlying VoiceProcessingIO unit.
|
||
pub fn ios_resume_voice_unit(&self) -> Result<(), AudioError> {
|
||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||
{
|
||
let mut guard = self._ios_voice_backend.lock().unwrap_or_else(|e| e.into_inner());
|
||
let unit = guard
|
||
.as_mut()
|
||
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
|
||
unit.resume()
|
||
}
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||
{
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
/// Set the **transmission gate** (SRS-201). When true the
|
||
/// encoder feed is allowed to emit Opus frames; when false the
|
||
/// captured audio is discarded before encoding. This is the
|
||
/// only writer permitted on `transmit_active` (SAD-075 /
|
||
/// SDD-089). Push-to-Talk subsystems — focused PTT today,
|
||
/// per-platform global backends in a follow-up — call this
|
||
/// method exclusively. No-op when capture is inactive.
|
||
pub fn set_transmit_active(&self, active: bool) {
|
||
self.transmit_gate.set(active);
|
||
}
|
||
|
||
/// Current transmit gate state.
|
||
pub fn transmit_active(&self) -> bool {
|
||
self.transmit_gate.load()
|
||
}
|
||
|
||
/// Shared handle to the underlying transmit gate (SAD-075 /
|
||
/// SDD-089). Returned for diagnostics and integration tests
|
||
/// only; never mutate the underlying atomic directly — use
|
||
/// [`Self::set_transmit_active`] instead.
|
||
pub fn transmit_gate(&self) -> &crate::ptt::AudioTransmitGate {
|
||
&self.transmit_gate
|
||
}
|
||
|
||
/// True if the capture stream opened. When false, the engine
|
||
/// runs in playback-only mode and the transmit gate is a
|
||
/// no-op (no frames will ever be encoded).
|
||
pub fn capture_active(&self) -> bool {
|
||
self.capture_active
|
||
}
|
||
|
||
/// Number of Opus frames sent since the engine started.
|
||
pub fn frames_sent(&self) -> u32 {
|
||
self.frames_sent.load(Ordering::Relaxed)
|
||
}
|
||
|
||
/// Number of inbound voice packets received and decoded.
|
||
pub fn frames_received(&self) -> u32 {
|
||
self.frames_received.load(Ordering::Relaxed)
|
||
}
|
||
|
||
/// Current microphone input level in dBFS (-120.0 = silence, 0.0 = clipping).
|
||
pub fn input_level(&self) -> f32 {
|
||
self.audio_processing_stats.input_dbfs()
|
||
}
|
||
|
||
/// Current audio-processing config snapshot.
|
||
pub fn audio_processing_config_snapshot(&self) -> crate::AudioProcessingConfig {
|
||
self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner()).clone()
|
||
}
|
||
|
||
/// Apply a voice-processing config after validating iOS invariants.
|
||
pub fn set_audio_processing_config(
|
||
&self,
|
||
mut config: crate::AudioProcessingConfig,
|
||
) -> Result<(), AudioError> {
|
||
#[cfg(target_os = "ios")]
|
||
config.validate_for_ios()?;
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
{
|
||
config.processing_backend = crate::AudioBackend::Noop;
|
||
}
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
self.apply_desktop_vad_backend(&config);
|
||
let mut guard = self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner());
|
||
*guard = config;
|
||
Ok(())
|
||
}
|
||
|
||
/// Re-apply the current audio processing config.
|
||
///
|
||
/// Used by the core layer to trigger VAD worker reload after a
|
||
/// model-path change (the epoch increments but the worker is only
|
||
/// reconstructed when `set_audio_processing_config` is called).
|
||
pub fn reload_audio_processing_config(&self) -> Result<(), AudioError> {
|
||
let config = self.audio_processing_config_snapshot();
|
||
self.set_audio_processing_config(config)
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
fn apply_desktop_vad_backend(&self, config: &crate::AudioProcessingConfig) {
|
||
apply_desktop_vad_backend_to_worker(
|
||
config,
|
||
&self.silero_vad_worker,
|
||
self.audio_processing_stats.as_ref(),
|
||
);
|
||
}
|
||
|
||
/// Current voice-processing stats snapshot.
|
||
pub fn audio_processing_stats(&self) -> crate::AudioProcessingStats {
|
||
let config = self.audio_processing_config.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
||
self.audio_processing_stats.snapshot(&config)
|
||
}
|
||
|
||
/// Latest Android voice-audio diagnostics snapshot (SDD-112 item
|
||
/// 10 / SDD-113 item 7 / SDD-116 item 3). On non-Android targets
|
||
/// this always returns `None`. On Android it returns `Some(...)`
|
||
/// once `AndroidVoiceUnit::open()` has published a snapshot; the
|
||
/// slot is cleared on `close()` / `Drop`. Per SDD-090 the
|
||
/// snapshot contains only device-side technical scalars — no PII.
|
||
pub fn android_diagnostics(
|
||
&self,
|
||
) -> Option<crate::mobile_voice_backend::AndroidAudioDiagnostics> {
|
||
crate::mobile_voice_backend::current_android_audio_diagnostics()
|
||
}
|
||
|
||
/// Set master output mute. When true the output stream emits
|
||
/// silence regardless of incoming voice frames.
|
||
pub fn set_output_muted(&self, muted: bool) {
|
||
self.output_muted.store(muted, Ordering::Relaxed);
|
||
}
|
||
|
||
/// True if the master output is currently muted locally.
|
||
pub fn output_muted(&self) -> bool {
|
||
self.output_muted.load(Ordering::Relaxed)
|
||
}
|
||
|
||
/// Set master output gain. 1.0 is unity; 0.0 is silent. Values
|
||
/// above 1.0 amplify (and may clip downstream). Clamped to a
|
||
/// sensible range internally.
|
||
pub fn set_output_gain(&self, gain: f32) {
|
||
let clamped = gain.clamp(0.0, 4.0);
|
||
self.output_gain.store(clamped.to_bits(), Ordering::Relaxed);
|
||
}
|
||
|
||
/// Current master output gain.
|
||
pub fn output_gain(&self) -> f32 {
|
||
f32::from_bits(self.output_gain.load(Ordering::Relaxed))
|
||
}
|
||
|
||
/// Set per-client output volume (SRS-075). `1.0` is unity, `0.0`
|
||
/// mutes. Values above `1.0` amplify and may clip. Clamped to
|
||
/// `0.0..4.0`.
|
||
pub fn set_client_volume(&self, client_id: u64, volume: f32) {
|
||
let clamped = volume.clamp(0.0, 4.0);
|
||
#[cfg(target_os = "android")]
|
||
{
|
||
let mut cmd = AudioCommand::SetVolume(SessionAudioId(client_id), clamped);
|
||
for _ in 0..64 {
|
||
match self.audio_event_producer.push_control(cmd) {
|
||
Ok(()) => return,
|
||
Err(returned) => {
|
||
cmd = returned;
|
||
std::thread::yield_now();
|
||
}
|
||
}
|
||
}
|
||
tracing::warn!(
|
||
target: "chanora_audio",
|
||
client_id,
|
||
volume = clamped,
|
||
"set_client_volume: control queue full after 64 retries — volume not applied"
|
||
);
|
||
}
|
||
#[cfg(not(target_os = "android"))]
|
||
{
|
||
match self.audio_handler.lock() {
|
||
Ok(mut h) => {
|
||
if let Some(q) = h.get_mut_queues().get_mut(&SessionAudioId(client_id)) {
|
||
q.volume = clamped;
|
||
}
|
||
}
|
||
Err(e) => {
|
||
tracing::warn!(
|
||
target: "chanora_audio",
|
||
client_id,
|
||
volume = clamped,
|
||
error = %e,
|
||
"set_client_volume: audio_handler lock poisoned — volume not applied"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
impl Drop for AudioEngine {
|
||
fn drop(&mut self) {
|
||
self.stop();
|
||
}
|
||
}
|
||
|
||
#[cfg(any(test, target_os = "android"))]
|
||
fn release_android_audio_mode_for_startup_rollback(
|
||
audio_mode_stack: &mut crate::mode_stack::ModeStack,
|
||
) -> crate::mode_stack::ModeRelease {
|
||
audio_mode_stack.release()
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
fn new_desktop_audio_processing_state() -> (
|
||
Arc<Mutex<crate::AudioProcessingConfig>>,
|
||
Arc<crate::SharedAudioProcessingStats>,
|
||
Arc<Mutex<Option<crate::vad::silero_onnx::SileroOnnxVadWorker>>>,
|
||
) {
|
||
let mut config = crate::AudioProcessingConfig::default();
|
||
// Desktop cpal capture does not use a platform voice-processing API.
|
||
// The global default (PlatformVoiceProcessing) is correct for iOS/macOS
|
||
// VPIO but would mislabel the desktop path in bridge diagnostics and
|
||
// set `platform_voice_processing_enabled = true` when no such
|
||
// processing exists. Override to Noop; the bridge/UI stats layer
|
||
// will then report the accurate backend.
|
||
config.processing_backend = crate::AudioBackend::Noop;
|
||
let audio_processing_config = Arc::new(Mutex::new(config.clone()));
|
||
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||
let silero_vad_worker = Arc::new(Mutex::new(None));
|
||
apply_desktop_vad_backend_to_worker(
|
||
&config,
|
||
&silero_vad_worker,
|
||
audio_processing_stats.as_ref(),
|
||
);
|
||
(
|
||
audio_processing_config,
|
||
audio_processing_stats,
|
||
silero_vad_worker,
|
||
)
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
fn apply_desktop_vad_backend_to_worker(
|
||
config: &crate::AudioProcessingConfig,
|
||
silero_vad_worker: &Arc<Mutex<Option<crate::vad::silero_onnx::SileroOnnxVadWorker>>>,
|
||
audio_processing_stats: &crate::SharedAudioProcessingStats,
|
||
) {
|
||
if config.vad_backend != crate::VadBackend::SileroOnnx {
|
||
let mut worker_guard = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner());
|
||
if worker_guard.is_some() {
|
||
info!(
|
||
target: "chanora_audio",
|
||
backend = config.vad_backend.as_str(),
|
||
"desktop: Silero ONNX VAD worker cleared because another VAD backend is selected"
|
||
);
|
||
}
|
||
*worker_guard = None;
|
||
audio_processing_stats.set_vad_fallback_active(false);
|
||
return;
|
||
}
|
||
|
||
// Load model and spawn worker BEFORE taking the lock so the
|
||
// realtime capture callback is not blocked on try_lock() during
|
||
// model I/O + thread spawn. The old worker (if any) is dropped
|
||
// after the new one is installed under the short lock hold.
|
||
let model_path = crate::vad::silero_model_bundle_path();
|
||
info!(
|
||
target: "chanora_audio",
|
||
path = %model_path,
|
||
"desktop: Silero ONNX VAD selected; loading model worker"
|
||
);
|
||
let new_worker = crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path);
|
||
{
|
||
let mut worker_guard = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner());
|
||
*worker_guard = new_worker;
|
||
}
|
||
// Check result after releasing the lock. Re-acquire is cheap and
|
||
// ensures we log the correct state without holding the mutex.
|
||
let worker_installed = silero_vad_worker.lock().unwrap_or_else(|e| e.into_inner()).is_some();
|
||
if worker_installed {
|
||
info!(
|
||
target: "chanora_audio",
|
||
path = %model_path,
|
||
"desktop: Silero ONNX VAD worker loaded"
|
||
);
|
||
audio_processing_stats.set_vad_fallback_active(false);
|
||
} else {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
path = %model_path,
|
||
"desktop: Silero ONNX VAD worker unavailable; WebRTC fallback will be used"
|
||
);
|
||
audio_processing_stats.set_vad_fallback_active(true);
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
#[test]
|
||
fn desktop_capture_voice_activity_opens_selector_from_speech() {
|
||
let gate = crate::ptt::AudioTransmitGate::new(false);
|
||
let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone()));
|
||
selector.set_mode(crate::TransmitMode::VoiceActivity);
|
||
selector.set_in_channel(true);
|
||
|
||
let encoder = crate::opus_voice::new_voip_encoder("desktop VAD test").unwrap();
|
||
let (voice_out_tx, _voice_out_rx) = mpsc::channel::<OutPacket>(16);
|
||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||
let voice_out_tx = crate::opus_voice::start_out_packet_worker(
|
||
voice_out_tx,
|
||
frames_sent,
|
||
"desktop-vad-test",
|
||
)
|
||
.unwrap();
|
||
let stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||
let mut capture = CaptureState::new(
|
||
encoder,
|
||
SAMPLE_RATE,
|
||
1,
|
||
1.0,
|
||
voice_out_tx,
|
||
gate.flag_arc(),
|
||
Some(selector.clone()),
|
||
Arc::new(Mutex::new(crate::AudioProcessingConfig {
|
||
vad_backend: crate::VadBackend::WebrtcVad,
|
||
..crate::AudioProcessingConfig::default()
|
||
})),
|
||
Arc::new(Mutex::new(None)),
|
||
stats.clone(),
|
||
);
|
||
|
||
let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||
for (idx, sample) in voiced.iter_mut().enumerate() {
|
||
let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32;
|
||
*sample = phase.sin() * 0.4;
|
||
}
|
||
for _ in 0..6 {
|
||
capture.ingest(&voiced);
|
||
}
|
||
|
||
assert!(
|
||
selector.voice_activity_open(),
|
||
"desktop capture must feed VAD and open VoiceActivity selector before transmit is already active"
|
||
);
|
||
assert!(
|
||
gate.load(),
|
||
"VoiceActivity selector should publish transmit gate"
|
||
);
|
||
let snapshot = stats.snapshot(&crate::AudioProcessingConfig::default());
|
||
assert!(snapshot.vad_active, "stats should expose active VAD");
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
#[test]
|
||
fn desktop_startup_audio_processing_state_applies_default_silero_fallback() {
|
||
let _guard = crate::vad::SILERO_MODEL_PATH_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||
crate::vad::clear_silero_model_path_for_test();
|
||
|
||
let (config, stats, worker) = new_desktop_audio_processing_state();
|
||
|
||
assert_eq!(
|
||
config.lock().unwrap_or_else(|e| e.into_inner()).vad_backend,
|
||
crate::VadBackend::SileroOnnx,
|
||
"desktop startup config should keep the default Silero backend selected"
|
||
);
|
||
assert!(
|
||
worker.lock().unwrap_or_else(|e| e.into_inner()).is_none(),
|
||
"missing startup model should not create an ONNX worker"
|
||
);
|
||
let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner()));
|
||
assert!(
|
||
snapshot.vad_fallback_active,
|
||
"desktop startup should mark WebRTC fallback active when the default Silero worker cannot load"
|
||
);
|
||
crate::vad::clear_silero_model_path_for_test();
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
#[test]
|
||
fn desktop_set_audio_processing_config_normalizes_default_backend_to_noop() {
|
||
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
|
||
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||
let engine = AudioEngine {
|
||
transmit_gate: crate::ptt::AudioTransmitGate::new(false),
|
||
frames_sent: Arc::new(AtomicU32::new(0)),
|
||
frames_received: Arc::new(AtomicU32::new(0)),
|
||
output_gain: Arc::new(AtomicU32::new(1.0_f32.to_bits())),
|
||
output_muted: Arc::new(AtomicBool::new(false)),
|
||
audio_processing_config: audio_processing_config.clone(),
|
||
audio_processing_stats,
|
||
silero_vad_worker: Arc::new(Mutex::new(None)),
|
||
audio_handler: Arc::new(Mutex::new(AudioHandler::new())),
|
||
_input_stream: Mutex::new(None),
|
||
_output_stream: Mutex::new(None),
|
||
shutdown_tx: None,
|
||
capture_active: false,
|
||
};
|
||
|
||
engine
|
||
.set_audio_processing_config(crate::AudioProcessingConfig::default())
|
||
.unwrap();
|
||
|
||
assert_eq!(
|
||
engine.audio_processing_config_snapshot().processing_backend,
|
||
crate::AudioBackend::Noop,
|
||
"desktop setter should report the actual no-op processing backend for default configs"
|
||
);
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
#[test]
|
||
fn desktop_capture_silero_selection_reports_fallback_when_worker_unavailable() {
|
||
let _guard = crate::vad::SILERO_MODEL_PATH_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||
crate::vad::clear_silero_model_path_for_test();
|
||
|
||
let gate = crate::ptt::AudioTransmitGate::new(false);
|
||
let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone()));
|
||
selector.set_mode(crate::TransmitMode::VoiceActivity);
|
||
selector.set_in_channel(true);
|
||
|
||
let encoder = crate::opus_voice::new_voip_encoder("desktop Silero fallback test").unwrap();
|
||
let (voice_out_tx, _voice_out_rx) = mpsc::channel::<OutPacket>(16);
|
||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||
let voice_out_tx = crate::opus_voice::start_out_packet_worker(
|
||
voice_out_tx,
|
||
frames_sent,
|
||
"desktop-silero-fallback-test",
|
||
)
|
||
.unwrap();
|
||
let stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||
let config = Arc::new(Mutex::new(crate::AudioProcessingConfig {
|
||
vad_backend: crate::VadBackend::SileroOnnx,
|
||
..crate::AudioProcessingConfig::default()
|
||
}));
|
||
let mut capture = CaptureState::new(
|
||
encoder,
|
||
SAMPLE_RATE,
|
||
1,
|
||
1.0,
|
||
voice_out_tx,
|
||
gate.flag_arc(),
|
||
Some(selector),
|
||
config.clone(),
|
||
Arc::new(Mutex::new(None)),
|
||
stats.clone(),
|
||
);
|
||
|
||
let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||
for (idx, sample) in voiced.iter_mut().enumerate() {
|
||
let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32;
|
||
*sample = phase.sin() * 0.4;
|
||
}
|
||
capture.ingest(&voiced);
|
||
|
||
let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner()));
|
||
assert_eq!(snapshot.vad_backend, crate::VadBackend::SileroOnnx);
|
||
assert!(
|
||
snapshot.vad_fallback_active,
|
||
"desktop Silero selection should make WebRTC fallback visible when ONNX worker cannot load"
|
||
);
|
||
crate::vad::clear_silero_model_path_for_test();
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
#[test]
|
||
fn desktop_capture_vad_consumes_only_new_pcm_while_transmitting() {
|
||
let gate = crate::ptt::AudioTransmitGate::new(true);
|
||
let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone()));
|
||
selector.set_mode(crate::TransmitMode::VoiceActivity);
|
||
selector.set_in_channel(true);
|
||
|
||
let encoder = crate::opus_voice::new_voip_encoder("desktop VAD duplicate test").unwrap();
|
||
let (voice_out_tx, _voice_out_rx) = mpsc::channel::<OutPacket>(16);
|
||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||
let voice_out_tx = crate::opus_voice::start_out_packet_worker(
|
||
voice_out_tx,
|
||
frames_sent,
|
||
"desktop-vad-duplicate-test",
|
||
)
|
||
.unwrap();
|
||
let stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||
let mut capture = CaptureState::new(
|
||
encoder,
|
||
SAMPLE_RATE,
|
||
1,
|
||
1.0,
|
||
voice_out_tx,
|
||
gate.flag_arc(),
|
||
Some(selector),
|
||
Arc::new(Mutex::new(crate::AudioProcessingConfig {
|
||
vad_backend: crate::VadBackend::Disabled,
|
||
..crate::AudioProcessingConfig::default()
|
||
})),
|
||
Arc::new(Mutex::new(None)),
|
||
stats,
|
||
);
|
||
|
||
capture.pcm_accum.extend(std::iter::repeat_n(
|
||
0.1_f32,
|
||
crate::frame::FRAME_10MS_SAMPLES + 240,
|
||
));
|
||
capture.pending_10ms[..240].fill(0.1);
|
||
capture.pending_10ms_len = 240;
|
||
capture.capture_frame_seq = 1;
|
||
capture.pcm_accum.extend(std::iter::repeat_n(0.2_f32, 240));
|
||
|
||
capture.process_pending_vad_frames(crate::frame::FRAME_10MS_SAMPLES + 240);
|
||
|
||
assert_eq!(
|
||
capture.capture_frame_seq, 2,
|
||
"VAD should consume only the 240 samples appended by the current ingest and complete one pending 10 ms frame"
|
||
);
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
#[test]
|
||
fn desktop_capture_silero_stale_enqueued_worker_uses_fallback() {
|
||
let gate = crate::ptt::AudioTransmitGate::new(false);
|
||
let selector = Arc::new(crate::TransmitModeSelector::new(gate.clone()));
|
||
selector.set_mode(crate::TransmitMode::VoiceActivity);
|
||
selector.set_in_channel(true);
|
||
|
||
let encoder =
|
||
crate::opus_voice::new_voip_encoder("desktop stale Silero fallback test").unwrap();
|
||
let (voice_out_tx, _voice_out_rx) = mpsc::channel::<OutPacket>(16);
|
||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||
let voice_out_tx = crate::opus_voice::start_out_packet_worker(
|
||
voice_out_tx,
|
||
frames_sent,
|
||
"desktop-stale-silero-fallback-test",
|
||
)
|
||
.unwrap();
|
||
let stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||
let config = Arc::new(Mutex::new(crate::AudioProcessingConfig {
|
||
vad_backend: crate::VadBackend::SileroOnnx,
|
||
..crate::AudioProcessingConfig::default()
|
||
}));
|
||
let worker = Arc::new(Mutex::new(Some(
|
||
crate::vad::silero_onnx::SileroOnnxVadWorker::stale_test_worker(),
|
||
)));
|
||
let mut capture = CaptureState::new(
|
||
encoder,
|
||
SAMPLE_RATE,
|
||
1,
|
||
1.0,
|
||
voice_out_tx,
|
||
gate.flag_arc(),
|
||
Some(selector),
|
||
config.clone(),
|
||
worker,
|
||
stats.clone(),
|
||
);
|
||
|
||
let mut voiced = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||
for (idx, sample) in voiced.iter_mut().enumerate() {
|
||
let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / SAMPLE_RATE as f32;
|
||
*sample = phase.sin() * 0.4;
|
||
}
|
||
|
||
capture.process_10ms_capture_frame(&voiced);
|
||
|
||
let snapshot = stats.snapshot(&config.lock().unwrap_or_else(|e| e.into_inner()));
|
||
assert!(
|
||
snapshot.vad_fallback_active,
|
||
"stale Silero worker output should report active WebRTC fallback"
|
||
);
|
||
assert!(
|
||
snapshot.vad_probability > 0.5,
|
||
"stale Silero worker output should use WebRTC fallback probability instead of forced silence"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn android_startup_rollback_releases_acquired_mode_snapshot() {
|
||
let mut stack = crate::mode_stack::ModeStack::new();
|
||
let _ = stack.acquire(7);
|
||
|
||
let release = release_android_audio_mode_for_startup_rollback(&mut stack);
|
||
|
||
assert_eq!(
|
||
release,
|
||
crate::mode_stack::ModeRelease::LastRelease { prior: 7 }
|
||
);
|
||
assert_eq!(stack.refcount(), 0);
|
||
assert_eq!(stack.snapshot(), None);
|
||
}
|
||
}
|
||
|
||
// ---------- Capture pipeline ----------
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
fn try_open_capture(
|
||
in_dev: &cpal::Device,
|
||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||
transmit_active: Arc<AtomicBool>,
|
||
frames_sent: Arc<AtomicU32>,
|
||
mic_gain: f32,
|
||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||
silero_vad_worker: Arc<Mutex<Option<crate::vad::silero_onnx::SileroOnnxVadWorker>>>,
|
||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||
) -> Result<cpal::Stream, AudioError> {
|
||
let in_cfg = in_dev
|
||
.default_input_config()
|
||
.map_err(|e| AudioError::StreamConfig(format!("input default: {e}")))?;
|
||
let in_sample_rate = in_cfg.sample_rate();
|
||
let in_channels = in_cfg.channels() as usize;
|
||
let in_format = in_cfg.sample_format();
|
||
// Buffer-size rationale (same shape as the output path):
|
||
// * Windows: pin to 2048 frames to avoid the small-period
|
||
// jitter of WASAPI shared mode.
|
||
// * macOS / iOS: CoreAudio picks a HAL-friendly default;
|
||
// iOS RemoteIO rejects arbitrary buffer-size requests.
|
||
// * Linux: same SDL2-vs-cpal split as the output path; we
|
||
// still use cpal for capture but leave Default since
|
||
// PipeWire's ALSA shim works well there.
|
||
let mut in_stream_cfg: cpal::StreamConfig = in_cfg.into();
|
||
#[cfg(target_os = "windows")]
|
||
{
|
||
in_stream_cfg.buffer_size = cpal::BufferSize::Fixed(2048);
|
||
}
|
||
#[cfg(not(target_os = "windows"))]
|
||
{
|
||
in_stream_cfg.buffer_size = cpal::BufferSize::Default;
|
||
}
|
||
|
||
let opus_enc = crate::opus_voice::new_voip_encoder("cpal capture")?;
|
||
|
||
let capture_state = Arc::new(Mutex::new(CaptureState::new(
|
||
opus_enc,
|
||
in_sample_rate,
|
||
in_channels,
|
||
mic_gain,
|
||
crate::opus_voice::start_out_packet_worker(
|
||
voice_out_tx,
|
||
frames_sent.clone(),
|
||
"cpal-capture",
|
||
)?,
|
||
transmit_active,
|
||
voice_activity_selector,
|
||
audio_processing_config,
|
||
silero_vad_worker,
|
||
audio_processing_stats,
|
||
)));
|
||
|
||
let stream = match in_format {
|
||
SampleFormat::F32 => build_input_stream::<f32>(in_dev, &in_stream_cfg, capture_state)?,
|
||
SampleFormat::I16 => build_input_stream::<i16>(in_dev, &in_stream_cfg, capture_state)?,
|
||
SampleFormat::U16 => build_input_stream::<u16>(in_dev, &in_stream_cfg, capture_state)?,
|
||
other => {
|
||
return Err(AudioError::StreamConfig(format!(
|
||
"unsupported input format: {other:?}"
|
||
)))
|
||
}
|
||
};
|
||
Ok(stream)
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
struct CaptureState {
|
||
encoder: OpusEncoder,
|
||
in_sample_rate: u32,
|
||
in_channels: usize,
|
||
mic_gain: f32,
|
||
/// 48 kHz mono buffer accumulated to FRAME_SAMPLES before each encode.
|
||
pcm_accum: Vec<f32>,
|
||
/// Resampling state for non-48k sources (simple linear resampler).
|
||
resample_pos: f64,
|
||
/// Last input sample carried over from the previous cpal callback
|
||
/// so the resampler can interpolate across the buffer boundary
|
||
/// without dropping continuity. Without this, every cpal period
|
||
/// boundary produces a discontinuity → audible buzz / popping
|
||
/// roughly at the period rate (~100 Hz for a 10 ms period on
|
||
/// Linux ALSA defaults).
|
||
resample_last: f32,
|
||
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||
voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender,
|
||
/// The PTT transmission gate. Read once per outbound frame; the
|
||
/// CaptureState never mutates this flag.
|
||
transmit_active: Arc<AtomicBool>,
|
||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||
vad_detector: crate::vad::WebRtcFallbackVad,
|
||
silero_vad_worker: Arc<Mutex<Option<crate::vad::silero_onnx::SileroOnnxVadWorker>>>,
|
||
silero_model_epoch: u64,
|
||
current_vad_backend: crate::VadBackend,
|
||
fallback_warned_backend: Option<crate::VadBackend>,
|
||
capture_frame_seq: u64,
|
||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||
pending_10ms: [f32; crate::frame::FRAME_10MS_SAMPLES],
|
||
pending_10ms_len: usize,
|
||
/// Pre-allocated mono downmix buffer. Resized in-place each
|
||
/// callback; `clear()` retains capacity. SDD-094 realtime-thread
|
||
/// invariant: this avoids the heap allocation that the prior fix
|
||
/// at engine.rs:1389-1397 (output-side `scratch` buffer)
|
||
/// addressed; the capture side has the same pattern and the same
|
||
/// user-perceptible cost on glibc malloc when the callback runs
|
||
/// on a realtime SCHED_FIFO thread.
|
||
mono_scratch: Vec<f32>,
|
||
/// Pre-allocated Opus-frame buffer. Sized to FRAME_SAMPLES (960)
|
||
/// on construction; reused across frames. `clear()` retains
|
||
/// capacity so the drain-into-frame path skips the allocator
|
||
/// after warmup. Same precedent as `mono_scratch` above.
|
||
frame_scratch: Vec<f32>,
|
||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||
/// Time-based decimation for the level meter. The cpal callback
|
||
/// cadence depends on platform (256 frames at 48 kHz ≈ 187 Hz,
|
||
/// 512 frames ≈ 94 Hz, 1024 frames ≈ 47 Hz) and can change at
|
||
/// runtime on device or sample-rate switch. The bridge consumer
|
||
/// (`input_level_stream`) only reads at ~30 Hz, so computing
|
||
/// `sqrt()` + `log10()` on every callback wastes real-time budget
|
||
/// and caused buffer underruns on macOS CoreAudio with small
|
||
/// buffer sizes. We only emit a new dBFS sample after at least
|
||
/// [LEVEL_METER_INTERVAL] has elapsed since the previous emit,
|
||
/// which is platform-cadence-independent.
|
||
last_level_emit: std::time::Instant,
|
||
}
|
||
|
||
/// Minimum interval between input-level dBFS samples sent to the
|
||
/// bridge. Matches the consumer rate (`input_level_stream` at ~30 Hz).
|
||
/// Time-based gating is robust to cpal buffer-size and sample-rate
|
||
/// changes that a fixed callback-count would not be.
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
const LEVEL_METER_INTERVAL: std::time::Duration = std::time::Duration::from_millis(33);
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
impl CaptureState {
|
||
fn new(
|
||
encoder: OpusEncoder,
|
||
in_sample_rate: u32,
|
||
in_channels: usize,
|
||
mic_gain: f32,
|
||
voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender,
|
||
transmit_active: Arc<AtomicBool>,
|
||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||
silero_vad_worker: Arc<Mutex<Option<crate::vad::silero_onnx::SileroOnnxVadWorker>>>,
|
||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||
) -> Self {
|
||
Self {
|
||
encoder,
|
||
in_sample_rate,
|
||
in_channels,
|
||
mic_gain,
|
||
pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2),
|
||
resample_pos: 0.0,
|
||
resample_last: 0.0,
|
||
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||
voice_out_tx,
|
||
transmit_active,
|
||
voice_activity_selector,
|
||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||
silero_vad_worker,
|
||
silero_model_epoch: crate::vad::silero_model_epoch(),
|
||
current_vad_backend: crate::VadBackend::Disabled,
|
||
fallback_warned_backend: None,
|
||
capture_frame_seq: 0,
|
||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||
audio_processing_config,
|
||
pending_10ms: [0.0; crate::frame::FRAME_10MS_SAMPLES],
|
||
pending_10ms_len: 0,
|
||
mono_scratch: Vec::with_capacity(4096),
|
||
frame_scratch: Vec::with_capacity(FRAME_SAMPLES),
|
||
audio_processing_stats,
|
||
// Start in the past so the first ingest emits immediately.
|
||
last_level_emit: std::time::Instant::now()
|
||
.checked_sub(LEVEL_METER_INTERVAL)
|
||
.unwrap_or_else(std::time::Instant::now),
|
||
}
|
||
}
|
||
|
||
/// Consume an arbitrary-rate, multichannel cpal buffer; produce
|
||
/// 48 kHz mono frames; encode and send when `transmit_active`
|
||
/// is true (PTT engaged).
|
||
// TODO(realtime-audio): This method runs on the cpal audio callback
|
||
// thread with a ~10 ms deadline. Known pre-existing violations of the
|
||
// realtime safety constraint that should be addressed in a future
|
||
// iteration:
|
||
// 1. Blocking Mutex::lock().unwrap() on self (via cpal callback)
|
||
// 2. Potential allocation in mono_scratch.reserve() and
|
||
// pcm_accum.extend_from_slice() when buffer capacity is exceeded
|
||
// 3. Encode-path warn!/error! logging via send_voip_frame closures
|
||
// These were present before the VAD integration and are not
|
||
// introduced by this changeset.
|
||
fn ingest<T: ToF32 + Copy>(&mut self, buf: &[T]) {
|
||
// 1. Down-mix to mono (pre-gain). Always performed so the level
|
||
// meter reflects real mic input even when PTT is released.
|
||
let in_channels = self.in_channels;
|
||
let mic_gain = self.mic_gain;
|
||
self.mono_scratch.clear();
|
||
let frame_count = buf.len() / in_channels.max(1);
|
||
self.mono_scratch.reserve(frame_count);
|
||
for frame in buf.chunks(in_channels) {
|
||
let sum: f32 = frame.iter().map(|s| s.to_f32_sample()).sum();
|
||
self.mono_scratch.push(sum / frame.len() as f32);
|
||
}
|
||
|
||
// Level meter: pay sqrt() + log10() only when at least
|
||
// LEVEL_METER_INTERVAL has elapsed, regardless of the
|
||
// platform's cpal callback cadence.
|
||
let now = std::time::Instant::now();
|
||
if now.duration_since(self.last_level_emit) >= LEVEL_METER_INTERVAL {
|
||
self.last_level_emit = now;
|
||
self.audio_processing_stats
|
||
.set_input_dbfs(crate::frame::dbfs(&self.mono_scratch));
|
||
}
|
||
|
||
if mic_gain != 1.0 {
|
||
for s in &mut self.mono_scratch {
|
||
*s *= mic_gain;
|
||
}
|
||
}
|
||
|
||
// 2. Resample to 48 kHz if needed. We re-borrow
|
||
// `mono_scratch` as a shared slice per branch to satisfy
|
||
// the borrow checker against `&mut self` on the
|
||
// resample path.
|
||
let vad_start_offset = self.pcm_accum.len();
|
||
if self.in_sample_rate == SAMPLE_RATE {
|
||
// Disjoint-borrow: copy the slice into pcm_accum without
|
||
// aliasing &mut self.
|
||
let (src, dst) = (&self.mono_scratch, &mut self.pcm_accum);
|
||
dst.extend_from_slice(src);
|
||
} else {
|
||
// resample_into_accum reads `mono` and writes
|
||
// `self.pcm_accum` / `self.resample_*`. These fields are
|
||
// disjoint from `self.mono_scratch`, but the function
|
||
// signature takes `&mut self`, so we take ownership of
|
||
// the scratch buffer briefly via `std::mem::take`, run
|
||
// the resampler, then move the buffer back so its
|
||
// capacity is preserved across callbacks.
|
||
let mono = std::mem::take(&mut self.mono_scratch);
|
||
self.resample_into_accum(&mono);
|
||
self.mono_scratch = mono;
|
||
}
|
||
|
||
self.process_pending_vad_frames(vad_start_offset);
|
||
|
||
if !self.transmit_active.load(Ordering::Relaxed) {
|
||
self.pcm_accum.clear();
|
||
return;
|
||
}
|
||
|
||
// 3. Encode any complete frames. Clamp each sample to
|
||
// [-1.0, 1.0] before handing to libopus's float encoder —
|
||
// out-of-range samples are hard-clipped inside libopus,
|
||
// which produces audible distortion on transient peaks.
|
||
// Soft-clamping at the engine boundary preserves headroom
|
||
// and matches what every other VoIP client does.
|
||
while self.pcm_accum.len() >= FRAME_SAMPLES {
|
||
// Reuse `self.frame_scratch` to avoid a per-frame Vec
|
||
// allocation on the realtime audio thread; same
|
||
// rationale as the engine.rs:1389-1397 precedent.
|
||
let frame = &mut self.frame_scratch;
|
||
frame.clear();
|
||
frame.extend(self.pcm_accum.drain(..FRAME_SAMPLES));
|
||
for s in frame.iter_mut() {
|
||
if *s > 1.0 {
|
||
*s = 1.0;
|
||
} else if *s < -1.0 {
|
||
*s = -1.0;
|
||
}
|
||
}
|
||
match self
|
||
.encoder
|
||
.encode_float(&frame[..], &mut self.opus_out[..])
|
||
{
|
||
Ok(len) => {
|
||
crate::opus_voice::send_voip_frame(
|
||
&self.voice_out_tx,
|
||
&self.opus_out,
|
||
len,
|
||
|| {
|
||
warn!(
|
||
target: "chanora_audio",
|
||
"voice_out queue full; dropping frame"
|
||
);
|
||
},
|
||
|| {
|
||
warn!(target: "chanora_audio", "voice_out closed; stopping send");
|
||
},
|
||
);
|
||
}
|
||
Err(e) => {
|
||
error!(target: "chanora_audio", error = %e, "opus encode failed");
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
fn process_pending_vad_frames(&mut self, start_offset: usize) {
|
||
let mut offset = start_offset.min(self.pcm_accum.len());
|
||
while offset < self.pcm_accum.len() {
|
||
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
|
||
let take = remaining.min(self.pcm_accum.len() - offset);
|
||
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
|
||
.copy_from_slice(&self.pcm_accum[offset..offset + take]);
|
||
self.pending_10ms_len += take;
|
||
offset += take;
|
||
|
||
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
|
||
let frame = self.pending_10ms;
|
||
self.process_10ms_capture_frame(&frame);
|
||
self.pending_10ms_len = 0;
|
||
}
|
||
}
|
||
}
|
||
|
||
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
|
||
// Realtime callback: do not log here. Publish state via atomics
|
||
// and let a non-realtime consumer translate transitions into
|
||
// info/warn events. The transmit/diagnostic stats stream
|
||
// already exposes vad_fallback_active for this purpose.
|
||
self.fallback_warned_backend = Some(failed_backend);
|
||
}
|
||
|
||
fn sync_vad_backend(&mut self, voice_activity_mode: bool, vad_backend: crate::VadBackend) {
|
||
if !voice_activity_mode {
|
||
self.current_vad_backend = crate::VadBackend::Disabled;
|
||
self.fallback_warned_backend = None;
|
||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||
return;
|
||
}
|
||
|
||
let silero_epoch = crate::vad::silero_model_epoch();
|
||
let silero_changed =
|
||
vad_backend == crate::VadBackend::SileroOnnx && silero_epoch != self.silero_model_epoch;
|
||
if vad_backend == self.current_vad_backend && !silero_changed {
|
||
return;
|
||
}
|
||
|
||
self.current_vad_backend = vad_backend;
|
||
self.silero_model_epoch = silero_epoch;
|
||
self.fallback_warned_backend = None;
|
||
self.vad_state.reset();
|
||
|
||
match vad_backend {
|
||
crate::VadBackend::SileroOnnx => {
|
||
let worker_available = self
|
||
.silero_vad_worker
|
||
.try_lock()
|
||
.map(|worker| worker.is_some())
|
||
.unwrap_or(false);
|
||
if worker_available {
|
||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||
} else {
|
||
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
|
||
self.audio_processing_stats.set_vad_fallback_active(true);
|
||
}
|
||
}
|
||
crate::VadBackend::WebrtcVad => {
|
||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||
}
|
||
crate::VadBackend::EnergyDebug => {
|
||
self.audio_processing_stats.set_vad_fallback_active(true);
|
||
}
|
||
crate::VadBackend::Disabled => {
|
||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||
}
|
||
}
|
||
}
|
||
|
||
fn process_10ms_capture_frame(&mut self, frame: &[f32; crate::frame::FRAME_10MS_SAMPLES]) {
|
||
let input_dbfs = crate::frame::dbfs(frame);
|
||
let (vad_backend, vad_hangover) = self
|
||
.audio_processing_config
|
||
.try_lock()
|
||
.map(|cfg| (cfg.vad_backend, cfg.vad_hangover_ms))
|
||
.unwrap_or((
|
||
crate::VadBackend::WebrtcVad,
|
||
crate::voice_activity::VAD_HANGOVER_MS,
|
||
));
|
||
let voice_activity_mode = self
|
||
.voice_activity_selector
|
||
.as_ref()
|
||
.map(|selector| selector.mode() == crate::TransmitMode::VoiceActivity)
|
||
.unwrap_or(false);
|
||
|
||
if voice_activity_mode {
|
||
self.sync_vad_backend(true, vad_backend);
|
||
self.vad_state.configure(
|
||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||
vad_hangover,
|
||
crate::voice_activity::VAD_MIN_TX_MS,
|
||
);
|
||
} else {
|
||
self.sync_vad_backend(false, vad_backend);
|
||
}
|
||
|
||
let (vad_probability, gate_open, used_fallback_vad) = if voice_activity_mode {
|
||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||
let capture_seq = self.capture_frame_seq;
|
||
let mut used_fallback_vad = false;
|
||
let vad = match vad_backend {
|
||
crate::VadBackend::Disabled => crate::vad::VadOutput {
|
||
probability: 1.0,
|
||
speech: true,
|
||
},
|
||
crate::VadBackend::SileroOnnx => {
|
||
// Single `try_lock` that both probes availability and
|
||
// sends the frame. The guard is scoped to the block
|
||
// so it drops before the fallback path (which needs
|
||
// `&mut self` for `mark_vad_fallback_active`).
|
||
// Returns Some(VadOutput) on a successful, non-stale
|
||
// send; None means "fall back to WebRTC VAD".
|
||
let worker_output = {
|
||
let guard = self.silero_vad_worker.try_lock().ok();
|
||
guard.and_then(|guard| {
|
||
let worker = guard.as_ref()?;
|
||
if worker.try_send(capture_seq, frame) && !worker.is_stale(capture_seq)
|
||
{
|
||
let p = worker.latest_probability();
|
||
Some(crate::vad::VadOutput {
|
||
probability: p,
|
||
speech: p >= 0.5,
|
||
})
|
||
} else {
|
||
None
|
||
}
|
||
})
|
||
};
|
||
if let Some(output) = worker_output {
|
||
output
|
||
} else {
|
||
used_fallback_vad = true;
|
||
self.mark_vad_fallback_active(vad_backend);
|
||
crate::vad::VoiceActivityDetector::process_10ms(
|
||
&mut self.vad_detector,
|
||
frame,
|
||
)
|
||
}
|
||
}
|
||
crate::VadBackend::WebrtcVad | crate::VadBackend::EnergyDebug => {
|
||
used_fallback_vad = vad_backend == crate::VadBackend::EnergyDebug;
|
||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, frame)
|
||
}
|
||
};
|
||
(
|
||
vad.probability,
|
||
self.vad_state.update(vad.speech),
|
||
used_fallback_vad,
|
||
)
|
||
} else {
|
||
(0.0, false, false)
|
||
};
|
||
self.audio_processing_stats
|
||
.set_vad_fallback_active(used_fallback_vad);
|
||
|
||
let vad_active = voice_activity_mode && gate_open;
|
||
if let Some(selector) = &self.voice_activity_selector {
|
||
selector.set_voice_activity_open(vad_active);
|
||
}
|
||
self.audio_processing_stats.update_capture(
|
||
input_dbfs,
|
||
input_dbfs,
|
||
vad_probability,
|
||
vad_active,
|
||
self.transmit_active.load(Ordering::Relaxed),
|
||
);
|
||
self.audio_processing_stats
|
||
.record_capture_frame(frame.iter().all(|sample| sample.abs() <= 0.000_001));
|
||
}
|
||
|
||
/// Simple linear resampler for `in_sample_rate → 48000`.
|
||
///
|
||
/// The resampler maintains continuity across cpal buffer
|
||
/// boundaries by treating `self.resample_last` as a virtual
|
||
/// sample at fractional index `0.0`, followed by the incoming
|
||
/// `mono` slice at indices `1.0..=mono.len()`. Without the
|
||
/// virtual anchor sample, the first interpolation point of
|
||
/// every new buffer collapses to `mono[0]` for both `a` and
|
||
/// `b`, producing a sample-and-hold step at every cpal period
|
||
/// boundary. On Linux ALSA defaults that's a ~100 Hz buzz /
|
||
/// popping. Production-quality work would use a windowed sinc
|
||
/// kernel; this carries the previous sample only and keeps the
|
||
/// CPU cost trivial.
|
||
fn resample_into_accum(&mut self, mono: &[f32]) {
|
||
if mono.is_empty() {
|
||
return;
|
||
}
|
||
let ratio = self.in_sample_rate as f64 / SAMPLE_RATE as f64;
|
||
let mut pos = self.resample_pos;
|
||
// The virtual buffer has length mono.len() + 1: index 0 is
|
||
// the carried-over last sample, indices 1..=mono.len() are
|
||
// the new buffer. We emit output samples while `pos` is
|
||
// strictly less than mono.len() so we always have a valid
|
||
// right-hand neighbour. The leftover sub-sample offset is
|
||
// carried over via `resample_pos` (rebased to the next
|
||
// buffer's virtual index 0 below).
|
||
while pos < mono.len() as f64 {
|
||
let i = pos.floor() as isize;
|
||
let frac = pos - i as f64;
|
||
let a = if i <= 0 {
|
||
self.resample_last
|
||
} else {
|
||
mono[(i - 1) as usize]
|
||
};
|
||
let b = if i < mono.len() as isize {
|
||
mono[i as usize]
|
||
} else {
|
||
// Should not happen given the while-condition, but
|
||
// guard for the boundary where ratio < 1.0 and `pos`
|
||
// can step past mono.len() in the last iteration.
|
||
a
|
||
};
|
||
self.pcm_accum
|
||
.push((a as f64 + frac * (b - a) as f64) as f32);
|
||
pos += ratio;
|
||
}
|
||
// Carry the leftover sub-sample offset, rebased so the next
|
||
// buffer's virtual index 0 is the new `resample_last`.
|
||
self.resample_pos = pos - mono.len() as f64;
|
||
// Anchor for the next buffer's interpolation.
|
||
self.resample_last = *mono.last().unwrap();
|
||
}
|
||
}
|
||
|
||
/// Per-sample format conversion to f32 in the range [-1.0, 1.0].
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
trait ToF32 {
|
||
fn to_f32_sample(self) -> f32;
|
||
}
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
impl ToF32 for f32 {
|
||
fn to_f32_sample(self) -> f32 {
|
||
self
|
||
}
|
||
}
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
impl ToF32 for i16 {
|
||
fn to_f32_sample(self) -> f32 {
|
||
f32::from(self) / f32::from(i16::MAX)
|
||
}
|
||
}
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
impl ToF32 for u16 {
|
||
fn to_f32_sample(self) -> f32 {
|
||
(f32::from(self) - f32::from(i16::MAX) - 1.0) / f32::from(i16::MAX)
|
||
}
|
||
}
|
||
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
fn build_input_stream<T>(
|
||
device: &cpal::Device,
|
||
config: &cpal::StreamConfig,
|
||
state: Arc<Mutex<CaptureState>>,
|
||
) -> Result<cpal::Stream, AudioError>
|
||
where
|
||
T: SizedSample + ToF32 + Send + 'static,
|
||
{
|
||
let stream = device
|
||
.build_input_stream(
|
||
*config,
|
||
move |data: &[T], _: &cpal::InputCallbackInfo| {
|
||
let mut s = state.lock().unwrap_or_else(|e| e.into_inner());
|
||
s.ingest(data);
|
||
},
|
||
move |e| {
|
||
error!(target: "chanora_audio", error = %e, "input stream error");
|
||
},
|
||
None,
|
||
)
|
||
.map_err(|e| AudioError::Backend(format!("build_input_stream: {e}")))?;
|
||
Ok(stream)
|
||
}
|
||
|
||
// ---------- Playback pipeline ----------
|
||
|
||
#[cfg(not(target_os = "linux"))]
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
fn build_output_stream<T>(
|
||
device: &cpal::Device,
|
||
config: &cpal::StreamConfig,
|
||
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||
output_gain: Arc<AtomicU32>,
|
||
output_muted: Arc<AtomicBool>,
|
||
dev_sample_rate: u32,
|
||
dev_channels: usize,
|
||
) -> Result<cpal::Stream, AudioError>
|
||
where
|
||
T: SizedSample + FromF32 + Send + 'static,
|
||
{
|
||
// Per-channel resampler state shared across cpal callbacks for
|
||
// continuity at buffer boundaries. AudioHandler produces 48 kHz
|
||
// stereo f32; we map the first two device channels to L/R and
|
||
// fill any extra channels with silence. `pos` carries the
|
||
// fractional source-sample offset; `last_l` / `last_r` are the
|
||
// anchor samples from the previous callback (avoid the sample-
|
||
// and-hold step that would otherwise pop at every period
|
||
// boundary — same trick as the capture-side resampler).
|
||
let resample_ratio = SAMPLE_RATE as f64 / dev_sample_rate as f64;
|
||
// `same_rate` is the common case where the device is already at
|
||
// 48 kHz — bypass the resampler entirely.
|
||
let same_rate = dev_sample_rate == SAMPLE_RATE;
|
||
let resample_state: Arc<Mutex<PlaybackResampleState>> =
|
||
Arc::new(Mutex::new(PlaybackResampleState {
|
||
pos: 0.0,
|
||
last_l: 0.0,
|
||
last_r: 0.0,
|
||
}));
|
||
// Reusable scratch buffer for 48 kHz stereo samples coming out
|
||
// of AudioHandler. Allocating a fresh `Vec` per cpal callback on
|
||
// glibc malloc was costing measurable time on the realtime audio
|
||
// thread and contributing to the popping users heard. We resize
|
||
// the buffer to the per-callback need and only grow the
|
||
// backing allocation when it must — typical buffer-size jitter
|
||
// stays under the high-water mark and skips the allocator
|
||
// entirely after the first few callbacks.
|
||
let mut scratch: Vec<f32> = Vec::with_capacity(8192);
|
||
// Diagnostic: log a warning if the cpal callback wall-clock
|
||
// exceeds the period budget so we can correlate user-perceived
|
||
// popping with measurable underruns. The threshold is half a
|
||
// period at 48 kHz / 2 ch / 1024-frame typical period — about
|
||
// 10 ms. We rate-limit the warning to once per second.
|
||
let mut last_slow_warn = std::time::Instant::now()
|
||
.checked_sub(std::time::Duration::from_secs(2))
|
||
.unwrap_or_else(std::time::Instant::now);
|
||
let stream = device
|
||
.build_output_stream(
|
||
*config,
|
||
move |out: &mut [T], _: &cpal::OutputCallbackInfo| {
|
||
let cb_start = std::time::Instant::now();
|
||
let muted = output_muted.load(Ordering::Relaxed);
|
||
let dev_frames = out.len() / dev_channels.max(1);
|
||
let src_frames = if same_rate {
|
||
dev_frames
|
||
} else {
|
||
// Ask for a few extra source frames so we never
|
||
// starve on the resample fractional boundary.
|
||
((dev_frames as f64 * resample_ratio).ceil() as usize) + 2
|
||
};
|
||
let needed = src_frames * 2;
|
||
if scratch.len() < needed {
|
||
scratch.resize(needed, 0.0);
|
||
}
|
||
// Zero the live slice; AudioHandler::fill_buffer
|
||
// writes silence into untouched samples, but
|
||
// resizing up from a smaller call leaves residual
|
||
// values from earlier callbacks. Use `fill` which
|
||
// optimises to memset on f32.
|
||
scratch[..needed].fill(0.0);
|
||
// Lock-and-decode. We deliberately hold the lock
|
||
// only for the duration of fill_buffer; the inbound
|
||
// forwarder uses handle_packet which is queue-fast.
|
||
{
|
||
let mut h = handler.lock().unwrap_or_else(|e| e.into_inner());
|
||
h.fill_buffer(&mut scratch[..needed]);
|
||
}
|
||
|
||
if muted {
|
||
for dst in out.iter_mut() {
|
||
*dst = T::from_f32_sample(0.0);
|
||
}
|
||
} else {
|
||
let gain = f32::from_bits(output_gain.load(Ordering::Relaxed));
|
||
if same_rate && dev_channels == 2 {
|
||
// Fast path: device is already 48 kHz stereo.
|
||
for (dst, s) in out.iter_mut().zip(scratch[..needed].iter().copied()) {
|
||
*dst = T::from_f32_sample(s * gain);
|
||
}
|
||
} else {
|
||
// Resample 48 kHz stereo → device-rate ×
|
||
// device-channels with continuity across
|
||
// callback boundaries.
|
||
let mut state = resample_state.lock().unwrap_or_else(|e| e.into_inner());
|
||
let mut pos = state.pos;
|
||
let mut last_l = state.last_l;
|
||
let mut last_r = state.last_r;
|
||
for frame_idx in 0..dev_frames {
|
||
let i = pos.floor() as isize;
|
||
let frac = pos - i as f64;
|
||
let (a_l, a_r) = if i <= 0 {
|
||
(last_l, last_r)
|
||
} else {
|
||
let idx = ((i - 1) as usize) * 2;
|
||
(scratch[idx], scratch[idx + 1])
|
||
};
|
||
let i_usize = i.max(0) as usize;
|
||
let (b_l, b_r) = if i_usize < src_frames {
|
||
let idx = i_usize * 2;
|
||
(scratch[idx], scratch[idx + 1])
|
||
} else {
|
||
(a_l, a_r)
|
||
};
|
||
let l = (a_l as f64 + frac * (b_l - a_l) as f64) as f32 * gain;
|
||
let r = (a_r as f64 + frac * (b_r - a_r) as f64) as f32 * gain;
|
||
let base = frame_idx * dev_channels;
|
||
if dev_channels == 1 {
|
||
// Mono output device (typical on iOS
|
||
// .voiceChat / phone-call audio path):
|
||
// downmix L+R to a single channel
|
||
// rather than dropping the R side.
|
||
// Without the downmix, anything panned
|
||
// right in the AudioHandler stereo mix
|
||
// is silently lost \u2014 on speakerphone
|
||
// this manifested as quiet remote
|
||
// speakers being inaudible.
|
||
out[base] = T::from_f32_sample((l + r) * 0.5);
|
||
} else {
|
||
out[base] = T::from_f32_sample(l);
|
||
if dev_channels >= 2 {
|
||
out[base + 1] = T::from_f32_sample(r);
|
||
}
|
||
for c in 2..dev_channels {
|
||
out[base + c] = T::from_f32_sample(0.0);
|
||
}
|
||
}
|
||
pos += resample_ratio;
|
||
}
|
||
let consumed = pos.floor() as usize;
|
||
state.pos = pos - consumed as f64;
|
||
if consumed > 0 && consumed <= src_frames {
|
||
let idx = (consumed - 1) * 2;
|
||
last_l = scratch[idx];
|
||
last_r = scratch[idx + 1];
|
||
state.last_l = last_l;
|
||
state.last_r = last_r;
|
||
}
|
||
}
|
||
}
|
||
|
||
// Period-budget diagnostic.
|
||
let elapsed = cb_start.elapsed();
|
||
let period_us = (dev_frames as u64 * 1_000_000) / dev_sample_rate as u64;
|
||
if elapsed.as_micros() as u64 > period_us / 2
|
||
&& last_slow_warn.elapsed() > std::time::Duration::from_secs(1)
|
||
{
|
||
last_slow_warn = std::time::Instant::now();
|
||
warn!(
|
||
target: "chanora_audio",
|
||
callback_us = elapsed.as_micros() as u64,
|
||
period_us,
|
||
dev_frames,
|
||
"output callback exceeded half the period budget — possible underrun cause"
|
||
);
|
||
}
|
||
},
|
||
move |e| {
|
||
error!(target: "chanora_audio", error = %e, "output stream error");
|
||
},
|
||
None,
|
||
)
|
||
.map_err(|e| {
|
||
error!(
|
||
target: "chanora_audio",
|
||
error = %e,
|
||
requested_channels = config.channels,
|
||
requested_sample_rate = config.sample_rate,
|
||
"build_output_stream FAILED"
|
||
);
|
||
AudioError::Backend(format!("build_output_stream: {e}"))
|
||
})?;
|
||
Ok(stream)
|
||
}
|
||
|
||
/// Resampler state carried across output cpal callbacks. See
|
||
/// `build_output_stream` for the rationale.
|
||
#[cfg(not(target_os = "linux"))]
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
struct PlaybackResampleState {
|
||
pos: f64,
|
||
last_l: f32,
|
||
last_r: f32,
|
||
}
|
||
|
||
#[cfg(not(target_os = "linux"))]
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
trait FromF32 {
|
||
fn from_f32_sample(v: f32) -> Self;
|
||
}
|
||
#[cfg(not(target_os = "linux"))]
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
impl FromF32 for f32 {
|
||
fn from_f32_sample(v: f32) -> Self {
|
||
v
|
||
}
|
||
}
|
||
#[cfg(not(target_os = "linux"))]
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
impl FromF32 for i16 {
|
||
fn from_f32_sample(v: f32) -> Self {
|
||
(v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i16
|
||
}
|
||
}
|
||
#[cfg(not(target_os = "linux"))]
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
impl FromF32 for u16 {
|
||
fn from_f32_sample(v: f32) -> Self {
|
||
let s = (v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i32;
|
||
(s + i32::from(i16::MAX) + 1) as u16
|
||
}
|
||
}
|
||
|
||
// ---------- Android voice-communication routing ----------
|
||
//
|
||
// SDD-108 §1: `AudioManager.setMode(MODE_IN_COMMUNICATION)` engagement
|
||
// is the routing-level lever that tells Android "this is a voice
|
||
// call, please use the earpiece / engage hardware AEC / NS / AGC
|
||
// where the device supports it". The helpers here are the
|
||
// platform-write surface invoked by the engine's `ModeStack`
|
||
// (`crate::mode_stack`) — the stack owns refcount + snapshot
|
||
// semantics (SDD-108 §2), these helpers only perform the platform
|
||
// write / read.
|
||
//
|
||
// Placement rationale (SDD-108 §4): the JNI helpers for
|
||
// `AudioManager.getMode` / `AudioManager.setMode` live in
|
||
// `chanora_audio::engine` (not `chanora_bridge::android_init`) per
|
||
// SDD-108 §4, which assigns the AudioManager JNI surface to the Rust
|
||
// audio engine. This keeps the AudioManager interaction co-located
|
||
// with the engine state (`ModeStack`) that owns it, so the
|
||
// snapshot/restore lifecycle and the JNI calls evolve together.
|
||
//
|
||
// `AudioManager.MODE_IN_COMMUNICATION == 3` per the Android SDK.
|
||
#[cfg(target_os = "android")]
|
||
pub const ANDROID_MODE_IN_COMMUNICATION: i32 = 3;
|
||
|
||
/// SDD-108 §5: typed error for the Android `AudioManager` JNI surface.
|
||
///
|
||
/// Replaces the previous ad-hoc `Result<_, String>` so callers can
|
||
/// pattern-match on the failure category (attach vs. method call vs.
|
||
/// other) and log/route accordingly. `Display` renders a stable
|
||
/// human-readable form that is safe to feed into the existing
|
||
/// `tracing::warn!(error = %e, ...)` sites.
|
||
#[cfg(target_os = "android")]
|
||
#[derive(Debug, Clone)]
|
||
pub enum AudioModeError {
|
||
/// `JavaVM::from_raw` or `attach_current_thread` failed: the
|
||
/// engine could not reach the JVM at all.
|
||
JniAttachFailed(String),
|
||
/// A specific JNI method call failed (e.g. `getMode`, `setMode`,
|
||
/// `getSystemService`). `method` is a static string for grep-ability.
|
||
MethodCallFailed {
|
||
method: &'static str,
|
||
detail: String,
|
||
},
|
||
/// Catch-all for non-JNI-method failures (null context, panic in
|
||
/// the JNI body, etc.).
|
||
Other(String),
|
||
}
|
||
|
||
#[cfg(target_os = "android")]
|
||
impl std::fmt::Display for AudioModeError {
|
||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
match self {
|
||
Self::JniAttachFailed(d) => write!(f, "JNI attach failed: {d}"),
|
||
Self::MethodCallFailed { method, detail } => {
|
||
write!(f, "JNI method `{method}` failed: {detail}")
|
||
}
|
||
Self::Other(d) => write!(f, "{d}"),
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(target_os = "android")]
|
||
impl std::error::Error for AudioModeError {}
|
||
|
||
#[cfg(target_os = "android")]
|
||
impl From<jni::errors::Error> for AudioModeError {
|
||
fn from(value: jni::errors::Error) -> Self {
|
||
Self::JniAttachFailed(value.to_string())
|
||
}
|
||
}
|
||
|
||
/// JNI helper shared by `android_get_audio_mode` and
|
||
/// `android_set_audio_mode`: attach to the current thread and return
|
||
/// the `AudioManager` jobject. Centralised so SDD-108's two platform
|
||
/// entry points share one bootstrapping path.
|
||
#[cfg(target_os = "android")]
|
||
fn android_audio_manager_call<F, R>(op: F) -> Result<R, AudioModeError>
|
||
where
|
||
F: for<'local> FnOnce(
|
||
&mut jni::Env<'local>,
|
||
&jni::objects::JObject<'local>,
|
||
) -> Result<R, AudioModeError>
|
||
+ std::panic::UnwindSafe,
|
||
{
|
||
use jni::objects::{JObject, JString, JValue};
|
||
|
||
let result = std::panic::catch_unwind(|| -> Result<R, AudioModeError> {
|
||
let ctx = ndk_context::android_context();
|
||
let vm_ptr = ctx.vm();
|
||
if vm_ptr.is_null() {
|
||
return Err(AudioModeError::JniAttachFailed(
|
||
"ndk_context vm is null".to_string(),
|
||
));
|
||
}
|
||
// SAFETY: ndk_context::android_context guarantees `vm` points
|
||
// at a live JavaVM* set by our bridge_init JNI hook. The
|
||
// unsafe block contains only the cast required by
|
||
// `JavaVM::from_raw`.
|
||
let jvm = unsafe { jni::JavaVM::from_raw(vm_ptr as *mut _) };
|
||
jvm.attach_current_thread(|env| -> Result<R, AudioModeError> {
|
||
let context_obj = unsafe { JObject::from_raw(env, ctx.context() as jni::sys::jobject) };
|
||
|
||
let service_name: JString =
|
||
env.new_string("audio")
|
||
.map_err(|e| AudioModeError::MethodCallFailed {
|
||
method: "new_string",
|
||
detail: e.to_string(),
|
||
})?;
|
||
let service_name_obj = JObject::from(service_name);
|
||
let audio_manager = env
|
||
.call_method(
|
||
&context_obj,
|
||
jni::jni_str!("getSystemService"),
|
||
jni::jni_sig!("(Ljava/lang/String;)Ljava/lang/Object;"),
|
||
&[JValue::Object(&service_name_obj)],
|
||
)
|
||
.map_err(|e| AudioModeError::MethodCallFailed {
|
||
method: "getSystemService",
|
||
detail: e.to_string(),
|
||
})?
|
||
.l()
|
||
.map_err(|e| AudioModeError::MethodCallFailed {
|
||
method: "getSystemService",
|
||
detail: format!("obj cast: {e}"),
|
||
})?;
|
||
if audio_manager.is_null() {
|
||
return Err(AudioModeError::Other(
|
||
"AudioManager service is null".to_string(),
|
||
));
|
||
}
|
||
op(env, &audio_manager)
|
||
})
|
||
});
|
||
match result {
|
||
Ok(inner) => inner,
|
||
Err(_) => Err(AudioModeError::Other(
|
||
"panic in JNI audio_manager call".to_string(),
|
||
)),
|
||
}
|
||
}
|
||
|
||
/// SDD-108 §1 platform-read: `AudioManager.getMode()`.
|
||
///
|
||
/// Returns the integer mode constant currently active on the system.
|
||
/// Called by the engine on first acquire (0 → 1 transition) so that
|
||
/// `ModeStack` can snapshot the prior mode for restoration on last
|
||
/// release.
|
||
#[cfg(target_os = "android")]
|
||
pub fn android_get_audio_mode() -> Result<i32, AudioModeError> {
|
||
android_audio_manager_call(|env, audio_manager| {
|
||
env.call_method(
|
||
audio_manager,
|
||
jni::jni_str!("getMode"),
|
||
jni::jni_sig!("()I"),
|
||
&[],
|
||
)
|
||
.map_err(|e| AudioModeError::MethodCallFailed {
|
||
method: "getMode",
|
||
detail: e.to_string(),
|
||
})?
|
||
.i()
|
||
.map_err(|e| AudioModeError::MethodCallFailed {
|
||
method: "getMode",
|
||
detail: format!("int cast: {e}"),
|
||
})
|
||
})
|
||
}
|
||
|
||
/// SDD-108 §1 platform-write: `AudioManager.setMode(mode)`.
|
||
///
|
||
/// `mode` is the Android `AudioManager.MODE_*` integer constant.
|
||
/// Use [`ANDROID_MODE_IN_COMMUNICATION`] for engagement; pass back
|
||
/// the snapshotted prior mode (from
|
||
/// [`crate::mode_stack::ModeRelease::LastRelease`]) for restoration.
|
||
#[cfg(target_os = "android")]
|
||
pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> {
|
||
use jni::objects::JValue;
|
||
android_audio_manager_call(move |env, audio_manager| {
|
||
env.call_method(
|
||
audio_manager,
|
||
jni::jni_str!("setMode"),
|
||
jni::jni_sig!("(I)V"),
|
||
&[JValue::Int(mode)],
|
||
)
|
||
.map_err(|e| AudioModeError::MethodCallFailed {
|
||
method: "setMode",
|
||
detail: e.to_string(),
|
||
})?;
|
||
Ok(())
|
||
})
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// SDD-120 §3 bench seam.
|
||
//
|
||
// Exposes a minimal factory for `CaptureState` plus a thin `ingest_f32`
|
||
// shim so the criterion bench harness in `crates/chanora_audio/benches/`
|
||
// can drive the same realtime capture code path the production cpal
|
||
// callback uses, without re-implementing CaptureState in the bench file.
|
||
// Marked `#[doc(hidden)]` so the public API surface is unaffected; this
|
||
// is not a supported external API. Only compiled on cpal-capture targets
|
||
// because Apple and Android use native voice backends instead.
|
||
// ---------------------------------------------------------------------------
|
||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||
#[doc(hidden)]
|
||
pub mod bench_seam {
|
||
use super::{Arc, AtomicBool, AtomicU32, CaptureState, OutPacket};
|
||
use tokio::sync::mpsc;
|
||
|
||
/// Opaque handle wrapping a CaptureState plus the dummy mpsc
|
||
/// receiver that prevents the channel sender from erroring out
|
||
/// when the bench drives `ingest`. The receiver is held inside
|
||
/// the handle so it lives for the bench's lifetime.
|
||
pub struct CaptureBenchHandle {
|
||
state: CaptureState,
|
||
// Keep the receiver alive so sends from CaptureState::encode_and_send
|
||
// do not fail; the bench discards what would be transmitted.
|
||
_rx: mpsc::Receiver<OutPacket>,
|
||
transmit_active: Arc<AtomicBool>,
|
||
}
|
||
|
||
impl CaptureBenchHandle {
|
||
/// Construct a CaptureState wired to a private mpsc + a
|
||
/// pre-asserted transmit-active flag so `ingest` exercises
|
||
/// the full down-mix → resample → encode → send pipeline.
|
||
///
|
||
/// `in_sample_rate` selects the input rate (48000 for the
|
||
/// passthrough path, 44100 / 16000 to exercise the linear
|
||
/// resampler). `in_channels` selects the channel layout
|
||
/// (typically 1 or 2).
|
||
pub fn new(in_sample_rate: u32, in_channels: usize) -> Self {
|
||
let encoder =
|
||
crate::opus_voice::new_voip_encoder("cpal bench").expect("opus encoder init");
|
||
let (tx, rx) = mpsc::channel::<OutPacket>(64);
|
||
let transmit_active = Arc::new(AtomicBool::new(true));
|
||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||
// Bridge the bench's private mpsc<OutPacket> to the
|
||
// realtime-thread-safe EncodedVoiceFrameSender that
|
||
// CaptureState expects. The worker task forwards
|
||
// encoded Opus frames to `tx` via `frames_sent`.
|
||
let voice_out_tx =
|
||
crate::opus_voice::start_out_packet_worker(tx, frames_sent, "cpal-bench")
|
||
.expect("start_out_packet_worker");
|
||
let state = CaptureState::new(
|
||
encoder,
|
||
in_sample_rate,
|
||
in_channels,
|
||
1.0,
|
||
voice_out_tx,
|
||
transmit_active.clone(),
|
||
None,
|
||
Arc::new(std::sync::Mutex::new(
|
||
crate::AudioProcessingConfig::default(),
|
||
)),
|
||
Arc::new(std::sync::Mutex::new(None)),
|
||
Arc::new(crate::SharedAudioProcessingStats::default()),
|
||
);
|
||
Self {
|
||
state,
|
||
_rx: rx,
|
||
transmit_active,
|
||
}
|
||
}
|
||
|
||
/// Drive one cpal-callback-equivalent buffer through the
|
||
/// realtime capture pipeline.
|
||
#[inline]
|
||
pub fn ingest_f32(&mut self, buf: &[f32]) {
|
||
self.state.ingest(buf);
|
||
}
|
||
|
||
/// Set the PTT gate. Defaults to true (bench measures the
|
||
/// transmitting path).
|
||
pub fn set_transmit_active(&self, active: bool) {
|
||
self.transmit_active
|
||
.store(active, std::sync::atomic::Ordering::Relaxed);
|
||
}
|
||
}
|
||
}
|