feat(voice): add iOS VAD runtime support

This commit is contained in:
Edison Jwa
2026-05-21 20:51:45 +09:00
parent 171baf6e41
commit 6af4ecab0f
73 changed files with 11529 additions and 1249 deletions
+35 -74
View File
@@ -42,7 +42,6 @@ use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use audiopus::coder::Encoder as OpusEncoder;
use audiopus::{Application as OpusApp, Channels as OpusChannels, SampleRate as OpusSampleRate};
use tracing::{debug, info, warn};
use crate::mobile_voice_backend::{
@@ -53,7 +52,7 @@ use crate::mobile_voice_backend::{
BackendEventTx, EffectEngagement, EffectEngine, InputPresetChoice, MobileVoiceAudioBackend,
SharingModeChoice,
};
use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
use chanora_protocol::OutPacket;
use tsclientlib::audio::AudioHandler;
use crate::{engine::SessionAudioId, AudioError};
@@ -71,19 +70,11 @@ use oboe::{
// `mobile_voice_backend` so the trait can expose `take_event_rx`
// (SDD-111 item 1) cross-platform.
/// 20 ms at 48 kHz mono — one Opus frame's worth of samples.
/// Matches the iOS and desktop constants; duplicated here so this
/// module is fully self-contained and cfg-gate-clean.
const FRAME_SAMPLES: usize = 960;
/// Maximum size of an encoded Opus frame in bytes (RFC 6716 §3.2.1).
const MAX_OPUS_FRAME: usize = 1275;
// --- Capture state for Oboe input callback (SDD-111 / SDD-120) ----
//
// Mirrors the iOS `IosCaptureState` and the cpal-side `CaptureState`.
// Oboe delivers 48 kHz mono i16 PCM; we apply mic gain, accumulate to
// FRAME_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
// FRAME_20MS_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
// and try-send the resulting packet on `voice_out_tx`.
struct AndroidCaptureState {
@@ -91,7 +82,7 @@ struct AndroidCaptureState {
/// Accumulator for 48 kHz mono PCM. 2x capacity to absorb
/// cpal-style buffer-size jitter without reallocating.
pcm_accum: Vec<i16>,
opus_out: [u8; MAX_OPUS_FRAME],
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
@@ -105,33 +96,11 @@ impl AndroidCaptureState {
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
) -> Result<Self, AudioError> {
let mut encoder =
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.map_err(|e| AudioError::Opus(format!("encoder new (android): {e}")))?;
if let Err(e) = encoder.set_bitrate(audiopus::Bitrate::BitsPerSecond(32_000)) {
warn!(target: "chanora_audio", error = %e, "opus(android): set_bitrate(32000) failed");
}
if let Err(e) = encoder.set_complexity(10) {
warn!(target: "chanora_audio", error = %e, "opus(android): set_complexity(10) failed");
}
if let Err(e) = encoder.set_inband_fec(true) {
warn!(target: "chanora_audio", error = %e, "opus(android): set_inband_fec(true) failed");
}
if let Err(e) = encoder.set_packet_loss_perc(5) {
warn!(target: "chanora_audio", error = %e, "opus(android): set_packet_loss_perc(5) failed");
}
info!(
target: "chanora_audio",
bitrate_bps = 32_000,
complexity = 10,
inband_fec = true,
packet_loss_perc = 5,
"android Oboe opus encoder tuned for VoIP"
);
let encoder = crate::opus_voice::new_voip_encoder("android")?;
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2),
opus_out: [0u8; MAX_OPUS_FRAME],
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
frames_sent,
@@ -139,7 +108,7 @@ impl AndroidCaptureState {
})
}
/// Consume i16 mono frames from Oboe, accumulate to FRAME_SAMPLES,
/// Consume i16 mono frames from Oboe, accumulate to FRAME_20MS_SAMPLES,
/// encode + send when PTT is held. Oboe delivers at the device's
/// native sample rate (always 48 kHz for modern Android per SRS-210),
/// so no resampling is needed.
@@ -159,28 +128,30 @@ impl AndroidCaptureState {
}));
}
// Drain complete 20 ms frames.
while self.pcm_accum.len() >= FRAME_SAMPLES {
let mut frame = [0i16; FRAME_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..FRAME_SAMPLES]);
self.pcm_accum.drain(..FRAME_SAMPLES);
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &self.opus_out[..len],
});
match self.voice_out_tx.try_send(packet) {
Ok(()) => {
self.frames_sent.fetch_add(1, Ordering::Relaxed);
}
Err(mpsc::error::TrySendError::Full(_)) => {
warn!(target: "chanora_audio", "android Oboe: voice_out queue full; dropping frame");
}
Err(mpsc::error::TrySendError::Closed(_)) => {
debug!(target: "chanora_audio", "android Oboe: voice_out closed; capture pipeline stopping");
}
}
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.frames_sent,
&self.opus_out,
len,
|| {
warn!(
target: "chanora_audio",
"android Oboe: voice_out queue full; dropping frame"
);
},
|| {
debug!(
target: "chanora_audio",
"android Oboe: voice_out closed; capture pipeline stopping"
);
},
);
}
Err(e) => {
warn!(target: "chanora_audio", error = %e, "android Oboe opus encode failed");
@@ -266,22 +237,12 @@ impl AudioOutputCallback for OutputCallback {
}
let gain = f32::from_bits(self.output_gain.load(Ordering::Relaxed));
let muted = self.output_muted.load(Ordering::Relaxed);
let mut peak: i16 = 0;
for (i, dst) in frames.iter_mut().enumerate() {
if muted {
*dst = 0;
continue;
}
let l = scratch[i * 2];
let r = scratch[i * 2 + 1];
let mono = (l + r) * 0.5 * gain;
let clamped = mono.clamp(-1.0, 1.0);
let sample = (clamped * i16::MAX as f32) as i16;
*dst = sample;
if sample.unsigned_abs() > peak.unsigned_abs() {
peak = sample;
}
}
let _ = crate::voice_render::downmix_stereo_f32_to_mono_i16(
&scratch[..needed],
frames,
gain,
muted,
);
}));
DataCallbackResult::Continue
}
@@ -0,0 +1,456 @@
//! P1 audio-processing configuration and statistics.
//!
//! iOS P1 ships the platform VoiceProcessingIO path by default. Rust
//! software AEC/NS/AGC backends are represented in the schema so the
//! bridge can reject unsafe combinations instead of silently enabling
//! double processing.
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use crate::AudioError;
/// Physical/logical audio route class used for route-aware policy.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AudioRoute {
/// Built-in speakerphone path.
Speaker,
/// Built-in receiver/earpiece path.
Earpiece,
/// Wired headset or USB headset.
WiredHeadset,
/// Bluetooth Hands-Free Profile duplex route.
BluetoothHfp,
/// Bluetooth A2DP output-only route.
BluetoothA2dp,
/// Route could not be classified yet.
Unknown,
}
impl AudioRoute {
/// Stable bridge/debug string.
pub fn as_str(self) -> &'static str {
match self {
Self::Speaker => "speaker",
Self::Earpiece => "earpiece",
Self::WiredHeadset => "wired_headset",
Self::BluetoothHfp => "bluetooth_hfp",
Self::BluetoothA2dp => "bluetooth_a2dp",
Self::Unknown => "unknown",
}
}
/// Parse from the Swift-side route class string (case-insensitive).
/// Unrecognised strings map to `Unknown`.
pub fn from_route_class(s: &str) -> Self {
match s {
"Speaker" | "speaker" => Self::Speaker,
"Earpiece" | "earpiece" => Self::Earpiece,
"WiredHeadset" | "wired_headset" => Self::WiredHeadset,
"BluetoothHfp" | "bluetooth_hfp" => Self::BluetoothHfp,
"BluetoothA2dp" | "bluetooth_a2dp" => Self::BluetoothA2dp,
_ => Self::Unknown,
}
}
}
/// iOS voice-processing mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IosVoiceProcessingMode {
/// Shipping default: Apple VoiceProcessingIO owns AEC/NS/AGC.
PlatformVoiceProcessing,
/// Experimental Sonora capture-processing path.
SonoraExperimental,
}
/// Processing backend selected by policy/config.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AudioBackend {
/// Platform voice processing, VPIO on iOS.
PlatformVoiceProcessing,
/// Rust-native Sonora backend.
Sonora,
/// Future WebRTC APM backend.
WebrtcApm,
/// No processing.
Noop,
}
impl AudioBackend {
/// Stable bridge/debug string.
pub fn as_str(self) -> &'static str {
match self {
Self::PlatformVoiceProcessing => "platform_voice_processing",
Self::Sonora => "sonora",
Self::WebrtcApm => "webrtc_apm",
Self::Noop => "noop",
}
}
}
/// VAD backend selected by policy/config.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VadBackend {
/// Silero ONNX VAD. P1 schema default when model/runtime exist.
SileroOnnx,
/// TEN VAD backend. Native TEN runtime is optional; unavailable
/// builds fall back to the realtime-safe WebRTC detector.
TenVad,
/// WebRTC-style fallback VAD.
WebrtcVad,
/// Debug-only energy VAD.
EnergyDebug,
/// VAD disabled.
Disabled,
}
impl VadBackend {
/// Stable bridge/debug string.
pub fn as_str(self) -> &'static str {
match self {
Self::SileroOnnx => "silero_vad_onnx",
Self::TenVad => "ten_vad",
Self::WebrtcVad => "webrtc_vad",
Self::EnergyDebug => "energy_debug",
Self::Disabled => "disabled",
}
}
}
/// Effect owner for AEC/NS/AGC policy fields.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EffectOwner {
/// Owned by platform voice processing.
Platform,
/// Owned by Sonora.
Sonora,
/// Owned by WebRTC APM.
WebrtcApm,
/// Conservative route-managed setting.
Conservative,
/// Disabled.
Off,
}
/// Voice-processing configuration owned by the Rust audio engine.
#[derive(Debug, Clone, PartialEq)]
pub struct AudioProcessingConfig {
/// Current route class.
pub route: AudioRoute,
/// iOS processing mode.
pub ios_mode: IosVoiceProcessingMode,
/// Processing backend.
pub processing_backend: AudioBackend,
/// VAD backend.
pub vad_backend: VadBackend,
/// AEC owner.
pub aec: EffectOwner,
/// Noise suppression owner.
pub ns: EffectOwner,
/// AGC owner.
pub agc: EffectOwner,
/// High-pass filter enabled.
pub hpf_enabled: bool,
/// Limiter enabled.
pub limiter_enabled: bool,
/// Hangover after speech closes.
pub vad_hangover_ms: u32,
/// Pre-roll before open.
pub vad_pre_roll_ms: u32,
/// Minimum transmit duration after open.
pub vad_min_tx_ms: u32,
/// Debug WAV dumps enabled.
pub debug_wav_dump_enabled: bool,
}
impl Default for AudioProcessingConfig {
fn default() -> Self {
Self {
route: AudioRoute::Speaker,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::SileroOnnx,
aec: EffectOwner::Platform,
// iOS VPIO owns NS/AGC on the default shipping path. Rust/Sonora
// effects are opt-in through the experimental raw route only.
ns: EffectOwner::Platform,
agc: EffectOwner::Platform,
hpf_enabled: true,
limiter_enabled: true,
vad_hangover_ms: crate::voice_activity::VAD_HANGOVER_MS,
vad_pre_roll_ms: 160,
vad_min_tx_ms: crate::voice_activity::VAD_MIN_TX_MS,
debug_wav_dump_enabled: false,
}
}
}
impl AudioProcessingConfig {
/// Validate P1 iOS invariants before applying a config.
pub fn validate_for_ios(&self) -> Result<(), AudioError> {
if self.route == AudioRoute::BluetoothA2dp {
return Err(AudioError::InvalidAudioProcessingConfig(
"bluetooth_a2dp is output-only and cannot transmit duplex voice".to_string(),
));
}
if self.ios_mode == IosVoiceProcessingMode::PlatformVoiceProcessing
&& (self.processing_backend == AudioBackend::Sonora
|| self.aec == EffectOwner::Sonora
|| self.ns == EffectOwner::Sonora
|| self.agc == EffectOwner::Sonora)
{
return Err(AudioError::InvalidAudioProcessingConfig(
"Sonora cannot be enabled with iOS VoiceProcessingIO".to_string(),
));
}
if self.ios_mode == IosVoiceProcessingMode::SonoraExperimental {
if self.processing_backend != AudioBackend::Sonora {
return Err(AudioError::InvalidAudioProcessingConfig(
"ios Sonora experimental mode requires the Sonora processing backend"
.to_string(),
));
}
}
Ok(())
}
/// Demote a failed VAD backend to the WebRTC fallback.
///
/// Returns `true` when the config changed.
pub fn disable_failed_vad_backend(&mut self, failed_backend: VadBackend) -> bool {
if self.vad_backend == failed_backend && failed_backend != VadBackend::WebrtcVad {
self.vad_backend = VadBackend::WebrtcVad;
true
} else {
false
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_config_is_valid_for_ios_vpio() {
let config = AudioProcessingConfig::default();
assert!(config.validate_for_ios().is_ok());
assert_eq!(
config.processing_backend,
AudioBackend::PlatformVoiceProcessing
);
assert_eq!(config.aec, EffectOwner::Platform);
assert_eq!(config.ns, EffectOwner::Platform);
assert_eq!(config.agc, EffectOwner::Platform);
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
}
#[test]
fn platform_voice_processing_rejects_sonora_effects() {
let config = AudioProcessingConfig {
ns: EffectOwner::Sonora,
..AudioProcessingConfig::default()
};
assert!(config.validate_for_ios().is_err());
}
#[test]
fn ten_vad_has_stable_debug_string() {
assert_eq!(VadBackend::TenVad.as_str(), "ten_vad");
}
#[test]
fn sonora_experimental_allows_full_sonora_chain() {
let config = AudioProcessingConfig {
ios_mode: IosVoiceProcessingMode::SonoraExperimental,
processing_backend: AudioBackend::Sonora,
aec: EffectOwner::Sonora,
ns: EffectOwner::Sonora,
agc: EffectOwner::Sonora,
..AudioProcessingConfig::default()
};
assert!(config.validate_for_ios().is_ok());
}
#[test]
fn sonora_experimental_rejects_non_sonora_backend() {
let config = AudioProcessingConfig {
ios_mode: IosVoiceProcessingMode::SonoraExperimental,
processing_backend: AudioBackend::PlatformVoiceProcessing,
aec: EffectOwner::Sonora,
ns: EffectOwner::Sonora,
agc: EffectOwner::Sonora,
..AudioProcessingConfig::default()
};
assert!(config.validate_for_ios().is_err());
}
#[test]
fn disable_failed_vad_backend_demotes_to_webrtc() {
let mut config = AudioProcessingConfig {
vad_backend: VadBackend::SileroOnnx,
..AudioProcessingConfig::default()
};
assert!(config.disable_failed_vad_backend(VadBackend::SileroOnnx));
assert_eq!(config.vad_backend, VadBackend::WebrtcVad);
assert!(!config.disable_failed_vad_backend(VadBackend::SileroOnnx));
}
}
/// Runtime audio processing stats exposed to bridge/UI diagnostics.
#[derive(Debug, Clone)]
pub struct AudioProcessingStats {
/// Input dBFS before processing.
pub input_dbfs: f32,
/// Render dBFS before playout.
pub render_dbfs: f32,
/// Processed capture dBFS.
pub processed_dbfs: f32,
/// Latest VAD probability or fallback confidence.
pub vad_probability: f32,
/// VAD active state.
pub vad_active: bool,
/// Current resolved transmit state.
pub transmitting: bool,
/// VAD backend.
pub vad_backend: VadBackend,
/// Whether fallback VAD is active.
pub vad_fallback_active: bool,
/// Processing backend.
pub processing_backend: AudioBackend,
/// iOS mode.
pub ios_voice_processing_mode: IosVoiceProcessingMode,
/// Route class.
pub audio_route: AudioRoute,
/// Actual sample rate.
pub actual_sample_rate_hz: u32,
/// Actual IO buffer frame count.
pub actual_io_buffer_frames: u32,
/// Input overrun count.
pub input_overruns: u64,
/// Output underrun count.
pub output_underruns: u64,
/// Callback xrun count.
pub callback_xruns: u64,
/// Clipped sample count.
pub clipped_samples: u64,
/// Sonora enabled.
pub sonora_enabled: bool,
/// Platform voice processing enabled.
pub platform_voice_processing_enabled: bool,
}
/// Lock-free stats storage shared with callbacks.
pub struct SharedAudioProcessingStats {
input_dbfs: AtomicU32,
render_dbfs: AtomicU32,
processed_dbfs: AtomicU32,
vad_probability: AtomicU32,
vad_active: AtomicBool,
transmitting: AtomicBool,
vad_fallback_active: AtomicBool,
input_overruns: AtomicU64,
output_underruns: AtomicU64,
callback_xruns: AtomicU64,
clipped_samples: AtomicU64,
actual_sample_rate_hz: AtomicU32,
actual_io_buffer_frames: AtomicU32,
}
impl Default for SharedAudioProcessingStats {
fn default() -> Self {
Self {
input_dbfs: AtomicU32::new((-120.0_f32).to_bits()),
render_dbfs: AtomicU32::new((-120.0_f32).to_bits()),
processed_dbfs: AtomicU32::new((-120.0_f32).to_bits()),
vad_probability: AtomicU32::new(0.0_f32.to_bits()),
vad_active: AtomicBool::new(false),
transmitting: AtomicBool::new(false),
vad_fallback_active: AtomicBool::new(false),
input_overruns: AtomicU64::new(0),
output_underruns: AtomicU64::new(0),
callback_xruns: AtomicU64::new(0),
clipped_samples: AtomicU64::new(0),
actual_sample_rate_hz: AtomicU32::new(crate::frame::SAMPLE_RATE_HZ),
actual_io_buffer_frames: AtomicU32::new(crate::frame::FRAME_20MS_SAMPLES as u32),
}
}
}
impl SharedAudioProcessingStats {
/// Store capture levels and VAD state.
pub fn update_capture(
&self,
input_dbfs: f32,
processed_dbfs: f32,
probability: f32,
vad_active: bool,
transmitting: bool,
) {
self.input_dbfs
.store(input_dbfs.to_bits(), Ordering::Relaxed);
self.processed_dbfs
.store(processed_dbfs.to_bits(), Ordering::Relaxed);
self.vad_probability
.store(probability.clamp(0.0, 1.0).to_bits(), Ordering::Relaxed);
self.vad_active.store(vad_active, Ordering::Relaxed);
self.transmitting.store(transmitting, Ordering::Relaxed);
}
/// Store render level and callback buffer size.
pub fn update_render(&self, dbfs: f32, io_buffer_frames: u32) {
self.render_dbfs.store(dbfs.to_bits(), Ordering::Relaxed);
self.actual_io_buffer_frames
.store(io_buffer_frames, Ordering::Relaxed);
}
/// Increment output underrun count.
pub fn increment_output_underrun(&self) {
self.output_underruns.fetch_add(1, Ordering::Relaxed);
}
/// Increment callback xrun count.
pub fn increment_callback_xrun(&self) {
self.callback_xruns.fetch_add(1, Ordering::Relaxed);
}
/// Add clipped samples.
pub fn add_clipped_samples(&self, count: u64) {
self.clipped_samples.fetch_add(count, Ordering::Relaxed);
}
/// Store whether the selected VAD backend is currently using a fallback.
pub fn set_vad_fallback_active(&self, active: bool) {
self.vad_fallback_active.store(active, Ordering::Relaxed);
}
/// Build an owned stats snapshot with config-derived labels.
pub fn snapshot(&self, config: &AudioProcessingConfig) -> AudioProcessingStats {
AudioProcessingStats {
input_dbfs: f32::from_bits(self.input_dbfs.load(Ordering::Relaxed)),
render_dbfs: f32::from_bits(self.render_dbfs.load(Ordering::Relaxed)),
processed_dbfs: f32::from_bits(self.processed_dbfs.load(Ordering::Relaxed)),
vad_probability: f32::from_bits(self.vad_probability.load(Ordering::Relaxed)),
vad_active: self.vad_active.load(Ordering::Relaxed),
transmitting: self.transmitting.load(Ordering::Relaxed),
vad_backend: config.vad_backend,
vad_fallback_active: self.vad_fallback_active.load(Ordering::Relaxed),
processing_backend: config.processing_backend,
ios_voice_processing_mode: config.ios_mode,
audio_route: config.route,
actual_sample_rate_hz: self.actual_sample_rate_hz.load(Ordering::Relaxed),
actual_io_buffer_frames: self.actual_io_buffer_frames.load(Ordering::Relaxed),
input_overruns: self.input_overruns.load(Ordering::Relaxed),
output_underruns: self.output_underruns.load(Ordering::Relaxed),
callback_xruns: self.callback_xruns.load(Ordering::Relaxed),
clipped_samples: self.clipped_samples.load(Ordering::Relaxed),
sonora_enabled: config.processing_backend == AudioBackend::Sonora,
platform_voice_processing_enabled: config.processing_backend
== AudioBackend::PlatformVoiceProcessing,
}
}
}
+330
View File
@@ -0,0 +1,330 @@
//! Async WAV debug dump writer for P1 diagnostics.
//!
//! Captures three streams for offline analysis:
//! * `raw_mic` — before AudioProcessor (INV_007: never from callback)
//! * `render_reference` — remote mixer output before playout
//! * `processed_mic` — after AudioProcessor
//!
//! ## Design
//!
//! The realtime callback MUST NOT write to disk (INV_007). Instead it
//! pushes 10 ms f32 frames onto a bounded `std::sync::mpsc` channel.
//! A background `tokio::task` drains the channel and writes WAV data.
//!
//! The channel is bounded (capacity = 500 frames ≈ 5 s of audio per
//! stream). If the writer falls behind, frames are dropped rather than
//! blocking the callback thread.
//!
//! WAV files are written to the OS temp directory with a filename that
//! encodes the stream name, route, backend, and a timestamp so
//! multiple sessions don't overwrite each other.
//!
//! ## Usage
//!
//! ```ignore
//! let writer = WavDebugRecorder::start(route, backend);
//! // In realtime callback (non-blocking):
//! writer.push_raw_mic(&frame);
//! writer.push_render_reference(&frame);
//! writer.push_processed_mic(&frame);
//! // On session end:
//! writer.stop(); // flushes and closes files
//! ```
use std::io::Write;
use std::path::PathBuf;
use std::sync::mpsc;
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::{info, warn};
use crate::audio_processing::{AudioBackend, AudioRoute};
use crate::frame::FRAME_10MS_SAMPLES;
/// Maximum number of 10 ms frames buffered per stream before drops.
const CHANNEL_CAPACITY: usize = 500;
/// Sample rate for WAV output (matches the capture pipeline).
const WAV_SAMPLE_RATE: u32 = 48_000;
/// Identifies which debug stream a frame belongs to.
#[derive(Debug, Clone, Copy)]
enum StreamId {
RawMic,
RenderReference,
ProcessedMic,
}
/// A single 10 ms frame tagged with its stream.
struct DebugFrame {
stream: StreamId,
samples: Box<[f32; FRAME_10MS_SAMPLES]>,
}
/// Handle for pushing frames from the realtime callback.
///
/// All push methods are non-blocking: if the channel is full the
/// frame is silently dropped and a counter is incremented.
pub struct WavDebugRecorder {
tx: mpsc::SyncSender<DebugFrame>,
/// Frames dropped due to full channel (diagnostic only).
drops: std::sync::atomic::AtomicU64,
/// Whether the recorder is active (set to false on stop).
active: std::sync::atomic::AtomicBool,
}
impl WavDebugRecorder {
/// Start the async WAV writer task. Returns a handle for pushing
/// frames from the realtime callback.
///
/// `route` and `backend` are embedded in the output filenames.
pub fn start(route: AudioRoute, backend: AudioBackend) -> std::sync::Arc<Self> {
let (tx, rx) = mpsc::sync_channel::<DebugFrame>(CHANNEL_CAPACITY);
let recorder = std::sync::Arc::new(Self {
tx,
drops: std::sync::atomic::AtomicU64::new(0),
active: std::sync::atomic::AtomicBool::new(true),
});
let ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let route_str = route.as_str().to_string();
let backend_str = backend.as_str().to_string();
// Spawn a blocking task so the WAV writer doesn't compete
// with the tokio async executor for CPU time.
std::thread::Builder::new()
.name("chanora-wav-writer".to_string())
.spawn(move || {
wav_writer_task(rx, &route_str, &backend_str, ts);
})
.ok();
recorder
}
/// Push a raw mic frame (before AudioProcessor). Non-blocking.
pub fn push_raw_mic(&self, samples: &[f32; FRAME_10MS_SAMPLES]) {
self.push(StreamId::RawMic, samples);
}
/// Push a render-reference frame (remote mixer output before playout).
/// Non-blocking.
pub fn push_render_reference(&self, samples: &[f32; FRAME_10MS_SAMPLES]) {
self.push(StreamId::RenderReference, samples);
}
/// Push a processed mic frame (after AudioProcessor). Non-blocking.
pub fn push_processed_mic(&self, samples: &[f32; FRAME_10MS_SAMPLES]) {
self.push(StreamId::ProcessedMic, samples);
}
/// Stop the recorder. Drops the sender so the writer task drains
/// and closes the WAV files.
pub fn stop(&self) {
self.active
.store(false, std::sync::atomic::Ordering::Relaxed);
// The sender is not dropped here because `self` is behind Arc.
// The writer task will exit when all senders are dropped (i.e.
// when the Arc is dropped). This is intentional: the task
// drains any remaining frames before closing files.
}
/// Number of frames dropped due to a full channel.
pub fn drop_count(&self) -> u64 {
self.drops.load(std::sync::atomic::Ordering::Relaxed)
}
fn push(&self, stream: StreamId, samples: &[f32; FRAME_10MS_SAMPLES]) {
if !self.active.load(std::sync::atomic::Ordering::Relaxed) {
return;
}
let mut boxed = Box::new([0.0_f32; FRAME_10MS_SAMPLES]);
boxed.copy_from_slice(samples);
let frame = DebugFrame {
stream,
samples: boxed,
};
if self.tx.try_send(frame).is_err() {
self.drops
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
}
}
// ---------- WAV writer task ----------
struct WavFile {
path: PathBuf,
file: std::fs::File,
samples_written: u32,
}
impl WavFile {
fn create(dir: &std::path::Path, name: &str) -> Option<Self> {
let path = dir.join(name);
match std::fs::File::create(&path) {
Ok(mut file) => {
// Write a placeholder WAV header; we'll patch it on close.
if write_wav_header(&mut file, 0).is_ok() {
Some(Self {
path,
file,
samples_written: 0,
})
} else {
None
}
}
Err(e) => {
warn!(target: "chanora_audio", error = %e, path = %path.display(), "wav debug: failed to create file");
None
}
}
}
fn write_samples(&mut self, samples: &[f32]) {
for &s in samples {
let i16_val = (s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16;
let _ = self.file.write_all(&i16_val.to_le_bytes());
}
self.samples_written += samples.len() as u32;
}
fn finalize(mut self) {
// Seek back to the start and rewrite the header with the
// correct data size.
use std::io::Seek;
if self.file.seek(std::io::SeekFrom::Start(0)).is_ok() {
let _ = write_wav_header(&mut self.file, self.samples_written);
}
info!(
target: "chanora_audio",
path = %self.path.display(),
samples = self.samples_written,
"wav debug: file closed"
);
}
}
fn write_wav_header(file: &mut std::fs::File, num_samples: u32) -> std::io::Result<()> {
// PCM WAV header: 44 bytes.
// Channels: 1 (mono), sample rate: 48000, bit depth: 16.
let channels: u16 = 1;
let sample_rate: u32 = WAV_SAMPLE_RATE;
let bits_per_sample: u16 = 16;
let byte_rate = sample_rate * channels as u32 * bits_per_sample as u32 / 8;
let block_align = channels * bits_per_sample / 8;
let data_size = num_samples * channels as u32 * bits_per_sample as u32 / 8;
let chunk_size = 36 + data_size;
file.write_all(b"RIFF")?;
file.write_all(&chunk_size.to_le_bytes())?;
file.write_all(b"WAVE")?;
file.write_all(b"fmt ")?;
file.write_all(&16u32.to_le_bytes())?; // subchunk1 size
file.write_all(&1u16.to_le_bytes())?; // PCM format
file.write_all(&channels.to_le_bytes())?;
file.write_all(&sample_rate.to_le_bytes())?;
file.write_all(&byte_rate.to_le_bytes())?;
file.write_all(&block_align.to_le_bytes())?;
file.write_all(&bits_per_sample.to_le_bytes())?;
file.write_all(b"data")?;
file.write_all(&data_size.to_le_bytes())?;
Ok(())
}
fn wav_writer_task(rx: mpsc::Receiver<DebugFrame>, route: &str, backend: &str, ts: u64) {
let dir = std::env::temp_dir();
let prefix = format!("chanora_debug_{route}_{backend}_{ts}");
let mut raw_mic = WavFile::create(&dir, &format!("{prefix}_raw_mic.wav"));
let mut render_ref = WavFile::create(&dir, &format!("{prefix}_render_reference.wav"));
let mut processed = WavFile::create(&dir, &format!("{prefix}_processed_mic.wav"));
info!(
target: "chanora_audio",
dir = %dir.display(),
prefix = %prefix,
"wav debug: writer started"
);
for frame in rx {
match frame.stream {
StreamId::RawMic => {
if let Some(f) = raw_mic.as_mut() {
f.write_samples(&*frame.samples);
}
}
StreamId::RenderReference => {
if let Some(f) = render_ref.as_mut() {
f.write_samples(&*frame.samples);
}
}
StreamId::ProcessedMic => {
if let Some(f) = processed.as_mut() {
f.write_samples(&*frame.samples);
}
}
}
}
// Channel closed — finalize all files.
if let Some(f) = raw_mic {
f.finalize();
}
if let Some(f) = render_ref {
f.finalize();
}
if let Some(f) = processed {
f.finalize();
}
info!(target: "chanora_audio", "wav debug: writer task exited");
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn recorder_starts_and_stops_without_panic() {
let rec =
WavDebugRecorder::start(AudioRoute::Speaker, AudioBackend::PlatformVoiceProcessing);
let frame = [0.1_f32; FRAME_10MS_SAMPLES];
rec.push_raw_mic(&frame);
rec.push_render_reference(&frame);
rec.push_processed_mic(&frame);
rec.stop();
// Drop the Arc to let the writer task drain.
drop(rec);
// Give the writer thread a moment to finish.
std::thread::sleep(std::time::Duration::from_millis(100));
}
#[test]
fn drop_count_increments_when_channel_full() {
// Use a tiny channel by creating a recorder and flooding it.
// We can't easily test the bounded channel directly, but we
// can verify the drop counter starts at zero.
let rec = WavDebugRecorder::start(AudioRoute::Speaker, AudioBackend::Noop);
assert_eq!(rec.drop_count(), 0);
rec.stop();
}
#[test]
fn wav_header_is_44_bytes() {
// Write to a temp file to test the header.
let tmp = std::env::temp_dir().join("chanora_test_wav_header.wav");
let mut f = std::fs::File::create(&tmp).unwrap();
write_wav_header(&mut f, 960).unwrap();
drop(f);
let data = std::fs::read(&tmp).unwrap();
assert_eq!(data.len(), 44, "WAV header must be 44 bytes");
assert_eq!(&data[0..4], b"RIFF");
assert_eq!(&data[8..12], b"WAVE");
let _ = std::fs::remove_file(&tmp);
}
}
+233 -136
View File
@@ -33,44 +33,21 @@ use tracing::{debug, info};
))]
use tracing::{error, warn};
#[cfg(target_os = "android")]
#[cfg(any(target_os = "ios", target_os = "android"))]
use tracing::warn;
use tsclientlib::audio::AudioHandler;
use chanora_protocol::{InboundVoice, OutPacket};
use crate::AudioError;
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
use audiopus::coder::Encoder as OpusEncoder;
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
use audiopus::{
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
SampleRate as OpusSampleRate,
};
use tsclientlib::audio::AudioHandler;
// `AudioData`, `CodecType`, `OutAudio` are referenced only by the
// cpal capture pipeline's Opus encode path (`CaptureState::encode_and_send`).
// `InboundVoice` + `OutPacket` are used by every platform — the
// inbound forwarder task pumps `InboundVoice` into AudioHandler on
// iOS too, and `OutPacket` flows out of the capture pipeline once
// commit 3 lands. Cfg-gate the cpal-only ones to keep iOS warnings
// clean.
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
use chanora_protocol::{AudioData, CodecType, OutAudio};
use chanora_protocol::{InboundVoice, OutPacket};
use crate::AudioError;
/// Stable Chanora-side identifier for AudioHandler bookkeeping.
/// We only ever have one connection at a time (DEC-006), so this is
/// trivially unique.
@@ -89,11 +66,8 @@ pub struct SessionAudioId(pub u64);
const SAMPLE_RATE: u32 = 48_000;
#[allow(dead_code)]
const FRAME_SAMPLES: usize = 48_000 / 50; // 960
#[allow(dead_code)]
const MAX_OPUS_FRAME: usize = 1275;
/// Engine configuration.
#[derive(Debug, Clone)]
#[derive(Clone)]
pub struct AudioEngineConfig {
/// Input gain applied before encoding (1.0 = pass-through).
pub mic_gain: f32,
@@ -117,6 +91,24 @@ pub struct AudioEngineConfig {
/// is rejected on Android because the P0 path intentionally has
/// no generic mobile-audio fallback.
pub mobile_voice_preset: bool,
/// Optional selector used by P1 VoiceActivity to publish VAD state.
#[doc(hidden)]
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
}
impl std::fmt::Debug for AudioEngineConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AudioEngineConfig")
.field("mic_gain", &self.mic_gain)
.field("ptt_initial", &self.ptt_initial)
.field("effects", &self.effects)
.field("mobile_voice_preset", &self.mobile_voice_preset)
.field(
"voice_activity_selector",
&self.voice_activity_selector.as_ref().map(|_| "present"),
)
.finish()
}
}
impl Default for AudioEngineConfig {
@@ -126,6 +118,7 @@ impl Default for AudioEngineConfig {
ptt_initial: false,
effects: crate::AudioEffects::default(),
mobile_voice_preset: true,
voice_activity_selector: None,
}
}
}
@@ -150,6 +143,16 @@ pub struct AudioEngine {
/// independent of the server-side mute the protocol layer
/// broadcasts.
output_muted: Arc<AtomicBool>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
voice_out_tx: mpsc::Sender<OutPacket>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
mic_gain: f32,
// Streams must be dropped to stop audio. Both are `!Send` because
// cpal's Stream isn't Send on some backends; we keep them in an
@@ -177,7 +180,7 @@ pub struct AudioEngine {
))]
_output_stream: Mutex<Option<cpal::Stream>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
_ios_voice_unit: Mutex<Option<crate::ios_voice_unit::IosVoiceUnit>>,
_ios_voice_backend: Mutex<Option<IosVoiceBackend>>,
/// SDD-111..SDD-115: Android Oboe voice backend. Owns the input
/// and output streams, SDD-113 hardware-effect handles, and the
/// foreground-service lifecycle; tearing it down on engine drop
@@ -237,6 +240,105 @@ pub struct AudioEngine {
unsafe impl Send for AudioEngine {}
unsafe impl Sync for AudioEngine {}
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[allow(dead_code)]
enum IosVoiceBackend {
Vpio(crate::ios_voice_unit::IosVoiceUnit),
#[cfg(target_os = "ios")]
Raw(crate::ios_raw_unit::IosRawUnit),
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
impl IosVoiceBackend {
fn pause(&mut self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
{
match self {
Self::Vpio(unit) => unit.pause(),
Self::Raw(unit) => unit.pause(),
}
}
#[cfg(target_os = "macos")]
{
Ok(())
}
}
fn resume(&mut self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
{
match self {
Self::Vpio(unit) => unit.resume(),
Self::Raw(unit) => unit.resume(),
}
}
#[cfg(target_os = "macos")]
{
Ok(())
}
}
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[allow(clippy::too_many_arguments)]
fn open_ios_voice_backend(
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_flag_for_capture: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<IosVoiceBackend, AudioError> {
let _cfg = audio_processing_config.lock().unwrap().clone();
#[cfg(target_os = "ios")]
{
if _cfg.ios_mode == crate::IosVoiceProcessingMode::SonoraExperimental {
match crate::ios_raw_unit::IosRawUnit::start(
handler.clone(),
output_gain.clone(),
output_muted.clone(),
voice_out_tx.clone(),
transmit_flag_for_capture.clone(),
frames_sent.clone(),
mic_gain,
voice_activity_selector.clone(),
audio_processing_config.clone(),
audio_processing_stats.clone(),
) {
Ok(unit) => {
info!(target: "chanora_audio", "ios: RemoteIO/Sonora experimental backend selected");
return Ok(IosVoiceBackend::Raw(unit));
}
Err(e) => {
warn!(
target: "chanora_audio",
error = %e,
"ios: RemoteIO/Sonora backend failed; falling back to VoiceProcessingIO"
);
}
}
}
}
let unit = crate::ios_voice_unit::IosVoiceUnit::start(
handler,
output_gain,
output_muted,
voice_out_tx,
transmit_flag_for_capture,
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats,
)?;
Ok(IosVoiceBackend::Vpio(unit))
}
impl AudioEngine {
/// Start the engine: open capture + playback streams, spawn the
/// inbound-voice forwarder, return a handle.
@@ -259,6 +361,7 @@ impl AudioEngine {
voice_in_rx: mpsc::Receiver<InboundVoice>,
transmit_gate: crate::ptt::AudioTransmitGate,
) -> Result<Self, AudioError> {
#[allow(clippy::needless_return)]
// Apple platforms route to a separate backend (VoiceProcessingIO
// via coreaudio-rs) because cpal does not expose the native
// voice-processing AudioUnit controls Chanora needs for VoIP.
@@ -353,6 +456,8 @@ impl AudioEngine {
let frames_received = Arc::new(AtomicU32::new(0));
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
let output_muted = Arc::new(AtomicBool::new(false));
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
// ---------- Capture ----------
// Capture is best-effort. If the platform default input
@@ -525,6 +630,8 @@ impl AudioEngine {
frames_received,
output_gain,
output_muted,
audio_processing_config,
audio_processing_stats,
_input_stream: Mutex::new(input_stream),
_output_stream: Mutex::new(Some(output_stream)),
shutdown_tx: Some(shutdown_tx),
@@ -549,6 +656,8 @@ impl AudioEngine {
let frames_received = Arc::new(AtomicU32::new(0));
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
let output_muted = Arc::new(AtomicBool::new(false));
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
Arc::new(Mutex::new(AudioHandler::new()));
@@ -678,6 +787,8 @@ impl AudioEngine {
frames_received,
output_gain,
output_muted,
audio_processing_config,
audio_processing_stats,
_android_voice_unit: Mutex::new(Some(android_voice_unit)),
audio_mode_stack: Mutex::new(audio_mode_stack),
shutdown_tx: Some(shutdown_tx),
@@ -725,23 +836,26 @@ impl AudioEngine {
let frames_received = Arc::new(AtomicU32::new(0));
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
let output_muted = Arc::new(AtomicBool::new(false));
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
Arc::new(Mutex::new(AudioHandler::new()));
let voice_out_tx_for_backend = voice_out_tx.clone();
// Construct the VPIO unit. Commit 1 ships a no-op callback
// pair; commits 3 + 4 land the real capture + playback
// wiring. Construction failure here is fatal (mirrors how
// the cpal output-stream construction failure is fatal in
// the non-iOS path).
let ios_voice_unit = crate::ios_voice_unit::IosVoiceUnit::start(
// Construct the live iOS voice backend. Platform VPIO stays
// the default shipping path; Sonora/RemoteIO remains opt-in.
let ios_voice_backend = open_ios_voice_backend(
audio_handler.clone(),
output_gain.clone(),
output_muted.clone(),
voice_out_tx,
voice_out_tx_for_backend,
transmit_flag_for_capture,
frames_sent.clone(),
cfg.mic_gain,
cfg.voice_activity_selector.clone(),
audio_processing_config.clone(),
audio_processing_stats.clone(),
)?;
// Capture is always considered active on iOS — VPIO's
@@ -791,7 +905,13 @@ impl AudioEngine {
frames_received,
output_gain,
output_muted,
_ios_voice_unit: Mutex::new(Some(ios_voice_unit)),
audio_processing_config,
audio_processing_stats,
audio_handler,
voice_out_tx,
voice_activity_selector: cfg.voice_activity_selector.clone(),
mic_gain: cfg.mic_gain,
_ios_voice_backend: Mutex::new(Some(ios_voice_backend)),
shutdown_tx: Some(shutdown_tx),
capture_active,
ptt_watchdog,
@@ -826,7 +946,7 @@ impl AudioEngine {
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let _ = self._ios_voice_unit.lock().unwrap().take();
let _ = self._ios_voice_backend.lock().unwrap().take();
}
// SDD-115 reverse-order teardown on Android:
// 1) close the voice unit (releases SDD-113 hardware
@@ -900,15 +1020,25 @@ impl AudioEngine {
/// iOS-only: restart the underlying VoiceProcessingIO unit after
/// route changes.
pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let mut guard = self._ios_voice_unit.lock().unwrap();
let unit = guard
.as_mut()
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
return unit.restart();
let backend = open_ios_voice_backend(
self.audio_handler.clone(),
self.output_gain.clone(),
self.output_muted.clone(),
self.voice_out_tx.clone(),
self.transmit_gate.flag_arc(),
self.frames_sent.clone(),
self.mic_gain,
self.voice_activity_selector.clone(),
self.audio_processing_config.clone(),
self.audio_processing_stats.clone(),
)?;
let mut guard = self._ios_voice_backend.lock().unwrap();
*guard = Some(backend);
Ok(())
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
Ok(())
}
@@ -916,15 +1046,15 @@ impl AudioEngine {
/// iOS-only: pause the underlying VoiceProcessingIO unit.
pub fn ios_pause_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let mut guard = self._ios_voice_unit.lock().unwrap();
let mut guard = self._ios_voice_backend.lock().unwrap();
let unit = guard
.as_mut()
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
return unit.pause();
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
unit.pause()
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
Ok(())
}
@@ -932,15 +1062,15 @@ impl AudioEngine {
/// iOS-only: resume the underlying VoiceProcessingIO unit.
pub fn ios_resume_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let mut guard = self._ios_voice_unit.lock().unwrap();
let mut guard = self._ios_voice_backend.lock().unwrap();
let unit = guard
.as_mut()
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
return unit.resume();
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
unit.resume()
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
Ok(())
}
@@ -1014,6 +1144,29 @@ impl AudioEngine {
self.frames_received.load(Ordering::Relaxed)
}
/// Current audio-processing config snapshot.
pub fn audio_processing_config_snapshot(&self) -> crate::AudioProcessingConfig {
self.audio_processing_config.lock().unwrap().clone()
}
/// Apply a voice-processing config after validating iOS invariants.
pub fn set_audio_processing_config(
&self,
config: crate::AudioProcessingConfig,
) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
config.validate_for_ios()?;
let mut guard = self.audio_processing_config.lock().unwrap();
*guard = config;
Ok(())
}
/// Current voice-processing stats snapshot.
pub fn audio_processing_stats(&self) -> crate::AudioProcessingStats {
let config = self.audio_processing_config.lock().unwrap().clone();
self.audio_processing_stats.snapshot(&config)
}
/// Latest Android voice-audio diagnostics snapshot (SDD-112 item
/// 10 / SDD-113 item 7 / SDD-116 item 3). On non-Android targets
/// this always returns `None`. On Android it returns `Some(...)`
@@ -1091,58 +1244,7 @@ fn try_open_capture(
in_stream_cfg.buffer_size = cpal::BufferSize::Default;
}
let mut opus_enc = OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.map_err(|e| AudioError::Opus(format!("encoder new: {e}")))?;
// Opus VOIP tuning. Defaults give us 'auto' bitrate (can drop
// to ~6 kbps during silence \u2014 which sounds garbled when
// talking resumes) and inband FEC disabled. On lossy mobile
// networks (cellular / iPhone WiFi roaming), packet loss
// without FEC produces audible clicks + cut-out frames.
//
// Settings derived from the Opus IETF VoIP recommendations
// (RFC 6716 \u00a7 7.1) and Discord's voice client tuning:
//
// * Bitrate 32 kbps : sweet spot for mono voice. Lower
// than 24 kbps starts to sound watery; higher than
// 64 kbps wastes bandwidth without perceptual gain on a
// human voice. Discord uses 64 kbps; mumble defaults to
// 40 kbps; we pick 32 kbps as a conservative VoIP value
// that survives 100 kbps uplinks comfortably.
// * Complexity 10 : max quality. The CPU cost on a modern
// iPhone (A14+) or any desktop is negligible (~0.5 % of
// a single core for 48 kHz mono).
// * Inband FEC on : opus inserts a low-bitrate redundancy
// copy of the previous frame inside the current packet
// so a single dropped packet can be reconstructed from
// the next one. Essential on lossy mobile.
// * Packet loss perc 5 % : tells the encoder to expect 5 %
// loss and pre-emptively budget bits for FEC. Higher
// values trade audio quality for resilience.
//
// Errors here are non-fatal: log + continue. The encoder
// works with defaults if any setter fails on an exotic
// libopus build.
if let Err(e) = opus_enc.set_bitrate(OpusBitrate::BitsPerSecond(32_000)) {
warn!(target: "chanora_audio", error = %e, "opus: set_bitrate(32000) failed");
}
if let Err(e) = opus_enc.set_complexity(10) {
warn!(target: "chanora_audio", error = %e, "opus: set_complexity(10) failed");
}
if let Err(e) = opus_enc.set_inband_fec(true) {
warn!(target: "chanora_audio", error = %e, "opus: set_inband_fec(true) failed");
}
if let Err(e) = opus_enc.set_packet_loss_perc(5) {
warn!(target: "chanora_audio", error = %e, "opus: set_packet_loss_perc(5) failed");
}
info!(
target: "chanora_audio",
bitrate_bps = 32_000,
complexity = 10,
inband_fec = true,
packet_loss_perc = 5,
"opus encoder tuned for VoIP"
);
let opus_enc = crate::opus_voice::new_voip_encoder("cpal capture")?;
let capture_state = Arc::new(Mutex::new(CaptureState::new(
opus_enc,
@@ -1184,7 +1286,7 @@ struct CaptureState {
/// roughly at the period rate (~100 Hz for a 10 ms period on
/// Linux ALSA defaults).
resample_last: f32,
opus_out: [u8; MAX_OPUS_FRAME],
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
/// The PTT transmission gate. Read once per outbound frame; the
/// CaptureState never mutates this flag.
@@ -1224,7 +1326,7 @@ impl CaptureState {
pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2),
resample_pos: 0.0,
resample_last: 0.0,
opus_out: [0u8; MAX_OPUS_FRAME],
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
frames_sent,
@@ -1310,22 +1412,21 @@ impl CaptureState {
.encode_float(&frame[..], &mut self.opus_out[..])
{
Ok(len) => {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &self.opus_out[..len],
});
match self.voice_out_tx.try_send(packet) {
Ok(()) => {
self.frames_sent.fetch_add(1, Ordering::Relaxed);
}
Err(mpsc::error::TrySendError::Full(_)) => {
warn!(target: "chanora_audio", "voice_out queue full; dropping frame");
}
Err(mpsc::error::TrySendError::Closed(_)) => {
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.frames_sent,
&self.opus_out,
len,
|| {
warn!(
target: "chanora_audio",
"voice_out queue full; dropping frame"
);
},
|| {
warn!(target: "chanora_audio", "voice_out closed; stopping send");
}
}
},
);
}
Err(e) => {
error!(target: "chanora_audio", error = %e, "opus encode failed");
@@ -1851,10 +1952,7 @@ pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> {
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
#[doc(hidden)]
pub mod bench_seam {
use super::{
Arc, AtomicBool, AtomicU32, CaptureState, OpusApp, OpusChannels, OpusEncoder,
OpusSampleRate, OutPacket,
};
use super::{Arc, AtomicBool, AtomicU32, CaptureState, OpusEncoder, OutPacket};
use tokio::sync::mpsc;
/// Opaque handle wrapping a CaptureState plus the dummy mpsc
@@ -1880,8 +1978,7 @@ pub mod bench_seam {
/// (typically 1 or 2).
pub fn new(in_sample_rate: u32, in_channels: usize) -> Self {
let encoder =
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.expect("opus encoder init");
crate::opus_voice::new_voip_encoder("cpal bench").expect("opus encoder init");
let (tx, rx) = mpsc::channel::<OutPacket>(64);
let transmit_active = Arc::new(AtomicBool::new(true));
let frames_sent = Arc::new(AtomicU32::new(0));
+101
View File
@@ -0,0 +1,101 @@
//! Canonical P1 voice frame helpers.
//!
//! The network contract remains 48 kHz mono, 20 ms Opus frames. P1
//! processing works internally on 10 ms f32 frames so VAD and future
//! processors can share a stable frame size without changing the
//! transport layer.
/// P1 sample rate in Hz.
pub const SAMPLE_RATE_HZ: u32 = 48_000;
/// Network frame duration in milliseconds.
pub const NETWORK_FRAME_MS: u32 = 20;
/// Processing frame duration in milliseconds.
pub const PROCESSING_FRAME_MS: u32 = 10;
/// Samples in one 10 ms mono frame at 48 kHz.
pub const FRAME_10MS_SAMPLES: usize = 480;
/// Samples in one 20 ms mono frame at 48 kHz.
pub const FRAME_20MS_SAMPLES: usize = 960;
/// 10 ms, 48 kHz, mono f32 processing frame.
#[derive(Debug, Clone, PartialEq)]
pub struct AudioFrame10ms {
/// Samples normalized to `[-1.0, 1.0]`.
pub samples: [f32; FRAME_10MS_SAMPLES],
}
/// 20 ms, 48 kHz, mono f32 network-frame-sized buffer.
#[derive(Debug, Clone, PartialEq)]
pub struct AudioFrame20ms {
/// Samples normalized to `[-1.0, 1.0]`.
pub samples: [f32; FRAME_20MS_SAMPLES],
}
impl AudioFrame20ms {
/// Convert one 20 ms frame into two 10 ms processing frames.
pub fn split(&self) -> (AudioFrame10ms, AudioFrame10ms) {
let mut first = [0.0; FRAME_10MS_SAMPLES];
let mut second = [0.0; FRAME_10MS_SAMPLES];
first.copy_from_slice(&self.samples[..FRAME_10MS_SAMPLES]);
second.copy_from_slice(&self.samples[FRAME_10MS_SAMPLES..]);
(
AudioFrame10ms { samples: first },
AudioFrame10ms { samples: second },
)
}
}
impl AudioFrame10ms {
/// Merge two 10 ms processing frames back into the 20 ms network
/// cadence used by the existing Opus path.
pub fn merge(first: &Self, second: &Self) -> AudioFrame20ms {
let mut samples = [0.0; FRAME_20MS_SAMPLES];
samples[..FRAME_10MS_SAMPLES].copy_from_slice(&first.samples);
samples[FRAME_10MS_SAMPLES..].copy_from_slice(&second.samples);
AudioFrame20ms { samples }
}
/// Compute RMS dBFS for diagnostics and fallback VAD.
pub fn dbfs(&self) -> f32 {
dbfs(&self.samples)
}
}
/// Convert i16 PCM to normalized f32 PCM.
pub fn i16_to_f32(sample: i16) -> f32 {
sample as f32 / i16::MAX as f32
}
/// Convert normalized f32 PCM to saturated i16 PCM.
pub fn f32_to_i16(sample: f32) -> i16 {
(sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16
}
/// RMS dBFS for a normalized f32 slice. Silence returns `-120.0`.
pub fn dbfs(samples: &[f32]) -> f32 {
if samples.is_empty() {
return -120.0;
}
let sum = samples.iter().map(|s| s * s).sum::<f32>();
let rms = (sum / samples.len() as f32).sqrt();
if rms <= 0.000_001 {
-120.0
} else {
20.0 * rms.log10()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_merge_preserves_samples() {
let mut samples = [0.0; FRAME_20MS_SAMPLES];
for (i, s) in samples.iter_mut().enumerate() {
*s = i as f32 / FRAME_20MS_SAMPLES as f32;
}
let original = AudioFrame20ms { samples };
let (a, b) = original.split();
assert_eq!(AudioFrame10ms::merge(&a, &b), original);
}
}
+492
View File
@@ -0,0 +1,492 @@
//! Optional raw iOS RemoteIO path for the Sonora experimental mode.
//!
//! Provides an alternative to `ios_voice_unit.rs` for the
//! `SonoraExperimental` processing mode. Instead of
//! `kAudioUnitSubType_VoiceProcessingIO` (which owns AEC/NS/AGC), it
//! opens `kAudioUnitSubType_RemoteIO` with voice processing explicitly
//! disabled so Rust's Sonora DSP chain can own the full signal path.
//!
//! ## Hard invariants enforced here
//!
//! * INV_009: Rust AEC only active when platform AEC is disabled.
//! * INV_010: VoiceProcessingIO and Sonora AEC3 are mutually exclusive.
//! * INV_011: Software AEC backend receives both capture and render-reference.
//! * INV_012: Render reference is copied from decoded/mixed remote PCM
//! before playout.
//!
//! ## Fallback
//!
//! If RemoteIO construction fails, the caller falls back to `IosVoiceUnit`
//! (VPIO) and logs the error.
//!
//! ## Status
//!
//! Experimental / disabled by default. Only activated when the user
//! explicitly selects `SonoraExperimental` mode via the bridge API.
//!
//! ## Platform
//!
//! `kAudioUnitSubType_RemoteIO` is only available in the iOS SDK.
//! This module is gated to `target_os = "ios"`.
#[cfg(target_os = "ios")]
pub use inner::IosRawUnit;
#[cfg(target_os = "ios")]
mod inner {
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use audiopus::coder::Encoder as OpusEncoder;
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
use coreaudio::audio_unit::render_callback::{self, data};
use coreaudio::audio_unit::IOType;
use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat};
use tokio::sync::mpsc;
use tracing::{info, warn};
use tsclientlib::audio::AudioHandler;
use crate::engine::SessionAudioId;
use crate::processor::AudioProcessor;
use crate::AudioError;
use chanora_protocol::OutPacket;
const SAMPLE_RATE_HZ: f64 = 48_000.0;
// ------------------------------------------------------------------ //
// Render-reference ring buffer //
// ------------------------------------------------------------------ //
/// 4-slot ring buffer shared between the render callback (writer) and
/// the capture callback (reader for Sonora AEC3). Capacity: 4 × 10 ms
/// = 40 ms of headroom.
///
/// If the capture callback runs before the render callback has written
/// a frame it reads zeros (silence reference), which is safe — Sonora
/// AEC3 simply skips cancellation for that frame.
struct RenderReferenceBuffer {
buf: Box<[[f32; 480]; 4]>,
write_idx: std::sync::atomic::AtomicUsize,
}
impl RenderReferenceBuffer {
fn new() -> Arc<Self> {
Arc::new(Self {
buf: Box::new([[0.0; 480]; 4]),
write_idx: std::sync::atomic::AtomicUsize::new(0),
})
}
/// Write one 10 ms render-reference frame. Realtime-safe.
fn write(&self, frame: &[f32; 480]) {
let idx = self.write_idx.load(Ordering::Relaxed);
// SAFETY: only one writer (render callback); torn reads
// are bounded to one frame of AEC degradation.
unsafe {
let slot = &self.buf[idx] as *const [f32; 480] as *mut [f32; 480];
(*slot).copy_from_slice(frame);
}
self.write_idx.store((idx + 1) % 4, Ordering::Relaxed);
}
/// Read the most recently completed render-reference frame.
fn read_latest(&self) -> [f32; 480] {
let wi = self.write_idx.load(Ordering::Relaxed);
let ri = (wi + 3) % 4;
self.buf[ri]
}
}
// SAFETY: accessed from two audio callback threads; data races are
// bounded to one frame of AEC quality degradation.
unsafe impl Send for RenderReferenceBuffer {}
unsafe impl Sync for RenderReferenceBuffer {}
// ------------------------------------------------------------------ //
// Capture pipeline state //
// ------------------------------------------------------------------ //
struct RawCaptureState {
encoder: OpusEncoder,
pcm_accum: Vec<i16>,
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
/// Processing config — retained for route-change reloads; not read in the hot path.
#[allow(dead_code)]
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
sonora_processor: crate::processor::SonoraProcessor,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: usize,
wav_recorder: Option<Arc<crate::debug_wav::WavDebugRecorder>>,
}
impl RawCaptureState {
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("ios raw")?;
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
frames_sent,
mic_gain,
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
sonora_processor: crate::processor::SonoraProcessor::with_config(
crate::processor::sonora::SonoraConfig::with_aec3(),
),
audio_processing_stats,
render_reference,
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
wav_recorder: None,
})
}
fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) {
if let Ok(mut cfg) = self.audio_processing_config.try_lock() {
let _ = cfg.disable_failed_vad_backend(failed_backend);
}
}
fn ingest_i16(&mut self, samples: &[i16]) {
// Accumulate into 10 ms frames for VAD / Sonora processing.
let mut offset = 0;
while offset < samples.len() {
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let take = remaining.min(samples.len() - offset);
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
.copy_from_slice(&samples[offset..offset + take]);
self.pending_10ms_len += take;
offset += take;
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
let frame = self.pending_10ms;
self.process_10ms_capture_frame(&frame);
self.pending_10ms_len = 0;
}
}
if !self.transmit_active.load(Ordering::Relaxed) {
self.pcm_accum.clear();
return;
}
// Encode complete 20 ms Opus frames.
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.frames_sent,
&self.opus_out,
len,
|| {
warn!(
target: "chanora_audio",
"ios raw: voice_out queue full; dropping frame"
);
},
|| {},
);
}
Err(e) => {
tracing::error!(target: "chanora_audio",
error = %e, "ios raw opus encode failed");
}
}
}
}
fn process_10ms_capture_frame(
&mut self,
samples: &[i16; crate::frame::FRAME_10MS_SAMPLES],
) {
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
*dst = crate::frame::i16_to_f32(src);
}
let input_dbfs = crate::frame::dbfs(&frame);
// WAV tap: raw mic (before processing).
if let Some(ref rec) = self.wav_recorder {
rec.push_raw_mic(&frame);
}
// INV_012: feed render reference to Sonora AEC3 before capture.
let render_ref = self.render_reference.read_latest();
self.sonora_processor.process_render(&render_ref);
self.sonora_processor.process_capture(&mut frame);
// WAV tap: processed mic (after Sonora).
if let Some(ref rec) = self.wav_recorder {
rec.push_processed_mic(&frame);
}
let (vad_backend, vad_hangover) = self
.audio_processing_config
.try_lock()
.map(|cfg| (cfg.vad_backend, cfg.vad_hangover_ms))
.unwrap_or((
crate::VadBackend::WebrtcVad,
crate::voice_activity::VAD_HANGOVER_MS,
));
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else {
used_fallback_vad = matches!(
vad_backend,
crate::VadBackend::SileroOnnx | crate::VadBackend::TenVad
);
if used_fallback_vad {
self.disable_failed_vad_backend(vad_backend);
}
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let active = self.vad_state.update(vad.speech);
if let Some(sel) = &self.voice_activity_selector {
sel.set_voice_activity_open(active);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
active,
self.transmit_active.load(Ordering::Relaxed),
);
if !self.transmit_active.load(Ordering::Relaxed) {
return;
}
let gain = self.mic_gain;
if (gain - 1.0).abs() < f32::EPSILON {
self.pcm_accum
.extend(frame.iter().copied().map(crate::frame::f32_to_i16));
} else {
self.pcm_accum.extend(frame.iter().copied().map(|s| {
let scaled = crate::frame::f32_to_i16(s) as f32 * gain;
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
}));
}
}
}
// ------------------------------------------------------------------ //
// IosRawUnit //
// ------------------------------------------------------------------ //
/// Raw iOS RemoteIO audio unit for the Sonora experimental path.
pub struct IosRawUnit {
unit: AudioUnit,
}
impl IosRawUnit {
/// Open a RemoteIO AudioUnit, install render + input callbacks, start.
#[allow(clippy::too_many_arguments)]
pub fn start(
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<Self, AudioError> {
// INV_010: reject if config requests VPIO (that's IosVoiceUnit's job).
{
let cfg = audio_processing_config.lock().unwrap();
if cfg.ios_mode == crate::IosVoiceProcessingMode::PlatformVoiceProcessing {
return Err(AudioError::InvalidAudioProcessingConfig(
"IosRawUnit requires SonoraExperimental mode".to_string(),
));
}
}
let mut unit = AudioUnit::new_uninitialized(IOType::RemoteIO)
.map_err(|e| AudioError::Backend(format!("remoteio new: {e}")))?;
// Enable input on bus 1.
const ENABLE_IO: u32 = 2003;
let enable: u32 = 1;
unit.set_property(ENABLE_IO, Scope::Input, Element::Input, Some(&enable))
.map_err(|e| AudioError::Backend(format!("remoteio enable input: {e}")))?;
// 48 kHz Int16 mono on both buses.
let fmt = StreamFormat {
sample_rate: SAMPLE_RATE_HZ,
sample_format: SampleFormat::I16,
flags: LinearPcmFlags::IS_SIGNED_INTEGER | LinearPcmFlags::IS_PACKED,
channels: 1,
};
unit.set_stream_format(fmt, Scope::Input, Element::Output)
.map_err(|e| AudioError::StreamConfig(format!("remoteio fmt output: {e}")))?;
unit.set_stream_format(fmt, Scope::Output, Element::Input)
.map_err(|e| AudioError::StreamConfig(format!("remoteio fmt input: {e}")))?;
// Shared render-reference buffer (INV_011 / INV_012).
let render_ref_buf = RenderReferenceBuffer::new();
let render_ref_for_capture = render_ref_buf.clone();
let mut capture_state = RawCaptureState::new(
voice_out_tx,
transmit_active,
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats.clone(),
render_ref_for_capture,
)?;
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
capture_state.ingest_i16(args.data.buffer);
Ok(())
})
.map_err(|e| AudioError::Backend(format!("remoteio input cb: {e}")))?;
let mut scratch: Vec<f32> = Vec::with_capacity(2048);
let stats_render = audio_processing_stats.clone();
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
let out = args.data.buffer;
let n = out.len();
let stereo_n = n * 2;
if scratch.len() < stereo_n {
scratch.resize(stereo_n, 0.0);
}
scratch[..stereo_n].fill(0.0);
match handler.try_lock() {
Ok(mut h) => {
let _ = h.fill_buffer(&mut scratch[..stereo_n]);
}
Err(std::sync::TryLockError::WouldBlock) => {
stats_render.increment_callback_xrun();
}
Err(std::sync::TryLockError::Poisoned(e)) => {
warn!(target: "chanora_audio",
"AudioHandler poisoned (raw render): {e}");
}
}
// INV_012: copy render reference BEFORE playout.
let mono_n = n.min(480);
let mut ref_frame = [0.0_f32; 480];
crate::voice_render::downmix_stereo_f32_to_mono_f32(
&scratch[..stereo_n],
&mut ref_frame[..mono_n],
);
render_ref_buf.write(&ref_frame);
let gain = f32::from_bits(output_gain.load(Ordering::Relaxed));
let muted = output_muted.load(Ordering::Relaxed);
let mix_stats = crate::voice_render::downmix_stereo_f32_to_mono_i16(
&scratch[..stereo_n],
out,
gain,
muted,
);
if mix_stats.clipped_samples > 0 {
stats_render.add_clipped_samples(mix_stats.clipped_samples);
}
stats_render.update_render(crate::frame::dbfs(&scratch[..stereo_n]), n as u32);
Ok(())
})
.map_err(|e| AudioError::Backend(format!("remoteio render cb: {e}")))?;
unit.initialize()
.map_err(|e| AudioError::Backend(format!("remoteio init: {e}")))?;
unit.start()
.map_err(|e| AudioError::Backend(format!("remoteio start: {e}")))?;
info!(
target: "chanora_audio",
sample_rate_hz = SAMPLE_RATE_HZ,
"ios RemoteIO (Sonora experimental) started"
);
Ok(Self { unit })
}
/// Restart the unit after a route change (stop → uninit → init → start).
pub fn restart(&mut self) -> Result<(), AudioError> {
self.unit
.stop()
.map_err(|e| AudioError::Backend(format!("remoteio restart stop: {e}")))?;
self.unit
.uninitialize()
.map_err(|e| AudioError::Backend(format!("remoteio restart uninit: {e}")))?;
self.unit
.initialize()
.map_err(|e| AudioError::Backend(format!("remoteio restart init: {e}")))?;
self.unit
.start()
.map_err(|e| AudioError::Backend(format!("remoteio restart start: {e}")))?;
info!(target: "chanora_audio", "ios RemoteIO restarted");
Ok(())
}
/// Pause the unit during an AVAudioSession interruption.
pub fn pause(&mut self) -> Result<(), AudioError> {
self.unit
.stop()
.map_err(|e| AudioError::Backend(format!("remoteio pause: {e}")))
}
/// Resume the unit after an interruption ends.
pub fn resume(&mut self) -> Result<(), AudioError> {
self.unit
.start()
.map_err(|e| AudioError::Backend(format!("remoteio resume: {e}")))
}
}
impl Drop for IosRawUnit {
fn drop(&mut self) {
if let Err(e) = self.unit.stop() {
warn!(target: "chanora_audio", error = %e,
"ios RemoteIO stop on drop failed");
} else {
info!(target: "chanora_audio", "ios RemoteIO stopped");
}
}
}
}
+463 -122
View File
@@ -75,10 +75,6 @@ use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use audiopus::coder::Encoder as OpusEncoder;
use audiopus::{
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
SampleRate as OpusSampleRate,
};
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
use coreaudio::audio_unit::render_callback::{self, data};
use coreaudio::audio_unit::IOType;
@@ -89,22 +85,7 @@ use tsclientlib::audio::AudioHandler;
use crate::engine::SessionAudioId;
use crate::AudioError;
use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
/// 20 ms at 48 kHz mono — one Opus frame's worth of samples.
/// Aligning the AudioUnit IO buffer to this frame size keeps the
/// jitter-buffer / encoder handshake tight (no fractional-frame
/// reads inside fill_buffer or accumulator drift inside the
/// capture pipeline).
const FRAME_SAMPLES_MONO: usize = 960;
/// Maximum size of an encoded Opus frame in bytes (per RFC 6716
/// §3.2.1). Same constant the cpal-side `CaptureState` uses; we
/// duplicate it here instead of cross-importing from engine.rs
/// because engine.rs's copy is cfg-gated to non-iOS for cpal-only
/// reasons. Post-step-5 review may dedupe by promoting both to a
/// shared `crate::framing` module.
const MAX_OPUS_FRAME: usize = 1275;
use chanora_protocol::OutPacket;
/// Sample rate every layer above us assumes. Matches the Opus
/// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the
@@ -124,6 +105,11 @@ const OUTPUT_BUS: Element = Element::Output;
/// samples in.
const INPUT_BUS: Element = Element::Input;
/// Pre-roll buffer capacity: 160 ms / 10 ms = 16 frames.
/// Stores processed i16 frames so the first syllable is not lost
/// when the VAD gate opens (VAD_004 / pre_roll_ms=160).
const PRE_ROLL_FRAMES: usize = 16;
/// Capture pipeline state owned by the VPIO input callback. The
/// AudioUnit hands us 48 kHz signed-int16 mono PCM directly (no
/// downmix or resample needed — VPIO's hardware-side mix-down
@@ -146,17 +132,36 @@ const INPUT_BUS: Element = Element::Input;
/// shared with `AudioEngine`.
struct IosCaptureState {
encoder: OpusEncoder,
/// 48 kHz mono PCM scratch accumulating to FRAME_SAMPLES_MONO
/// 48 kHz mono PCM scratch accumulating to FRAME_20MS_SAMPLES
/// per encode. Capacity 2x to absorb cpal-style buffer-size
/// jitter without reallocating.
pcm_accum: Vec<i16>,
opus_out: [u8; MAX_OPUS_FRAME],
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
/// PTT transmission gate. Read once per outbound frame; this
/// struct never mutates the flag (SAD-075 / SDD-089).
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
/// Background Silero worker — enqueues frames off the realtime
/// callback and publishes the latest probability atomically.
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
sonora_processor: crate::processor::SonoraProcessor,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: usize,
pre_roll_buf: [[i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
pre_roll_head: usize,
pre_roll_count: usize,
pre_roll_flushed: bool,
capture_frame_seq: u64,
wav_recorder: Arc<Mutex<Option<Arc<crate::debug_wav::WavDebugRecorder>>>>,
}
impl IosCaptureState {
@@ -164,54 +169,55 @@ impl IosCaptureState {
/// Encoder configuration is the same as cpal-side
/// `try_open_capture` (engine.rs) so audio quality is platform-
/// neutral.
#[allow(clippy::too_many_arguments)]
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
wav_recorder: Arc<Mutex<Option<Arc<crate::debug_wav::WavDebugRecorder>>>>,
) -> Result<Self, AudioError> {
let mut encoder =
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.map_err(|e| AudioError::Opus(format!("encoder new (ios): {e}")))?;
// VoIP-tuned settings — bitrate 32 kbps, complexity 10,
// inband FEC on, packet-loss-perc 5. Soft-fail each setter
// with a warn log to match the cpal-side behaviour: an
// unusual libopus build that rejects one setter shouldn't
// tank the whole pipeline. Full rationale + RFC citations
// are in engine.rs::try_open_capture line ~640.
if let Err(e) = encoder.set_bitrate(OpusBitrate::BitsPerSecond(32_000)) {
warn!(target: "chanora_audio", error = %e, "opus(ios): set_bitrate(32000) failed");
}
if let Err(e) = encoder.set_complexity(10) {
warn!(target: "chanora_audio", error = %e, "opus(ios): set_complexity(10) failed");
}
if let Err(e) = encoder.set_inband_fec(true) {
warn!(target: "chanora_audio", error = %e, "opus(ios): set_inband_fec(true) failed");
}
if let Err(e) = encoder.set_packet_loss_perc(5) {
warn!(target: "chanora_audio", error = %e, "opus(ios): set_packet_loss_perc(5) failed");
}
info!(
target: "chanora_audio",
bitrate_bps = 32_000,
complexity = 10,
inband_fec = true,
packet_loss_perc = 5,
"ios VPIO opus encoder tuned for VoIP"
);
let encoder = crate::opus_voice::new_voip_encoder("ios VPIO")?;
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(FRAME_SAMPLES_MONO * 2),
opus_out: [0u8; MAX_OPUS_FRAME],
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
frames_sent,
mic_gain,
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
sonora_processor: crate::processor::SonoraProcessor::new(),
audio_processing_stats,
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
pre_roll_buf: [[0_i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
pre_roll_head: 0,
pre_roll_count: 0,
pre_roll_flushed: false,
capture_frame_seq: 0,
wav_recorder,
})
}
fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) {
if let Ok(mut cfg) = self.audio_processing_config.try_lock() {
if cfg.disable_failed_vad_backend(failed_backend) {
self.current_vad_backend = crate::VadBackend::WebrtcVad;
}
}
}
/// Consume the i16 mono buffer delivered by VPIO, accumulate
/// to a 20 ms frame boundary, encode + send when PTT is held.
///
@@ -220,6 +226,22 @@ impl IosCaptureState {
/// In practice "interleaved mono" is the same byte layout as
/// "planar mono" so we just take the buffer as-is.
fn ingest_i16(&mut self, samples: &[i16]) {
let mut offset = 0;
while offset < samples.len() {
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let take = remaining.min(samples.len() - offset);
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
.copy_from_slice(&samples[offset..offset + take]);
self.pending_10ms_len += take;
offset += take;
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
let frame = self.pending_10ms;
self.process_10ms_capture_frame(&frame);
self.pending_10ms_len = 0;
}
}
if !self.transmit_active.load(Ordering::Relaxed) {
// Drain accumulator while muted so we don't pop on the
// PTT release edge. Matches cpal-side behaviour.
@@ -227,63 +249,40 @@ impl IosCaptureState {
return;
}
// Mic-gain application. When gain==1.0 we skip the
// multiply + saturate loop entirely — that's the common
// case and the loop is the inner-most hot path of the
// realtime audio thread.
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
self.pcm_accum.extend_from_slice(samples);
} else {
let gain = self.mic_gain;
self.pcm_accum.extend(samples.iter().map(|&s| {
// Saturating mul-then-cast keeps the signal in
// the i16 envelope. Clipping in this branch is
// expected — if the user pushed mic_gain past 1.0
// and is shouting, the alternative is wrap-around
// distortion which sounds far worse.
let scaled = (s as f32) * gain;
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
}));
}
// Drain complete 20 ms frames out of the accumulator, encode
// each, send the resulting Opus packet on the protocol
// queue. The `while` covers the case where a single VPIO
// callback delivers more than one frame's worth (rare on
// iOS where the HW IO buffer duration aligns with the
// Opus frame, but always possible during route changes).
while self.pcm_accum.len() >= FRAME_SAMPLES_MONO {
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
// Use a stack-allocated frame buffer to avoid the
// per-callback allocation a `drain(..N).collect()`
// would incur. The encoder doesn't need ownership.
let mut frame = [0i16; FRAME_SAMPLES_MONO];
frame.copy_from_slice(&self.pcm_accum[..FRAME_SAMPLES_MONO]);
self.pcm_accum.drain(..FRAME_SAMPLES_MONO);
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &self.opus_out[..len],
});
match self.voice_out_tx.try_send(packet) {
Ok(()) => {
self.frames_sent.fetch_add(1, Ordering::Relaxed);
}
Err(mpsc::error::TrySendError::Full(_)) => {
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.frames_sent,
&self.opus_out,
len,
|| {
warn!(
target: "chanora_audio",
"ios VPIO: voice_out queue full; dropping frame"
);
}
Err(mpsc::error::TrySendError::Closed(_)) => {
},
|| {
debug!(
target: "chanora_audio",
"ios VPIO: voice_out closed; capture pipeline stopping"
);
}
}
},
);
}
Err(e) => {
error!(target: "chanora_audio", error = %e, "ios VPIO opus encode failed");
@@ -291,6 +290,260 @@ impl IosCaptureState {
}
}
}
fn process_10ms_capture_frame(&mut self, samples: &[i16; crate::frame::FRAME_10MS_SAMPLES]) {
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
*dst = crate::frame::i16_to_f32(src);
}
let input_dbfs = crate::frame::dbfs(&frame);
// WAV tap: raw mic (before processing, DIAG_002).
if let Ok(guard) = self.wav_recorder.try_lock() {
if let Some(rec) = guard.as_ref() {
rec.push_raw_mic(&frame);
}
}
// Read config once per frame (try_lock: non-blocking, falls back to
// last-known values if the lock is contended — safe to miss one frame).
let (
run_ns,
run_agc,
run_hpf,
vad_backend,
vad_hangover,
debug_wav_dump_enabled,
route,
processing_backend,
) = self
.audio_processing_config
.try_lock()
.map(|cfg| {
let ns =
cfg.ns != crate::EffectOwner::Off && cfg.ns != crate::EffectOwner::Platform;
let agc =
cfg.agc != crate::EffectOwner::Off && cfg.agc != crate::EffectOwner::Platform;
let hpf = cfg.hpf_enabled;
(
ns,
agc,
hpf,
cfg.vad_backend,
cfg.vad_hangover_ms,
cfg.debug_wav_dump_enabled,
cfg.route,
cfg.processing_backend,
)
})
.unwrap_or((
false,
false,
true,
crate::VadBackend::SileroOnnx,
crate::voice_activity::VAD_HANGOVER_MS,
false,
crate::AudioRoute::Unknown,
crate::AudioBackend::PlatformVoiceProcessing,
));
// Switch VAD backend when the config changes.
let silero_model_epoch = crate::vad::silero_model_epoch();
let silero_model_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_model_epoch != self.silero_model_epoch;
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
if debug_wav_dump_enabled {
if recorder_guard.is_none() {
*recorder_guard = Some(crate::debug_wav::WavDebugRecorder::start(
route,
processing_backend,
));
}
} else if let Some(recorder) = recorder_guard.take() {
recorder.stop();
}
}
if vad_backend != self.current_vad_backend || silero_model_changed {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
// Attempt to load Silero model from the well-known
// bundle path. The actual inference runs on a
// background worker; the callback only enqueues
// 10 ms frames and falls back to WebRTC if the
// worker is missing or stale.
let model_path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path);
if self.silero_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"Silero VAD model not found at {model_path}; falling back to WebRTC VAD"
);
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
}
self.audio_processing_stats
.set_vad_fallback_active(self.silero_vad_worker.is_none());
}
crate::VadBackend::TenVad => {
self.silero_vad_worker = None;
warn!(
target: "chanora_audio",
"TEN VAD selected but native TEN runtime is not bundled; falling back to WebRTC VAD"
);
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
self.audio_processing_stats.set_vad_fallback_active(true);
}
_ => {
self.silero_vad_worker = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
}
// Reset VAD state machine timers on backend switch.
self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
self.vad_state.reset();
}
// Keep the VAD state machine aligned with the active config.
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
// Apply the enabled stages through the SonoraProcessor.
// We reconfigure it on-the-fly to match the current settings.
if run_ns || run_agc || run_hpf {
use crate::processor::sonora::SonoraConfig;
use crate::processor::AudioProcessor;
let new_cfg = SonoraConfig {
hpf: run_hpf,
aec3: false, // NEVER in VPIO path (INV_009)
ns: run_ns,
agc2: run_agc,
};
if new_cfg != *self.sonora_processor.config() {
self.sonora_processor.apply_config(new_cfg);
}
self.sonora_processor.process_capture(&mut frame);
}
// VAD: use Silero if loaded, otherwise WebRTC fallback.
// Disabled backend → always open (Continuous-like for VAD mode).
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
if worker.try_send(capture_seq, &frame) && !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else {
used_fallback_vad = true;
self.silero_vad_worker = None;
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
used_fallback_vad = true;
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let gate_open = self.vad_state.update(vad.speech);
if let Some(selector) = &self.voice_activity_selector {
selector.set_voice_activity_open(gate_open);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
gate_open,
transmit_active,
);
// WAV tap: processed mic (after Rust DSP, DIAG_002).
if let Ok(guard) = self.wav_recorder.try_lock() {
if let Some(rec) = guard.as_ref() {
rec.push_processed_mic(&frame);
}
}
// Convert to i16 for accumulation.
let mut pcm_frame = [0_i16; crate::frame::FRAME_10MS_SAMPLES];
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) {
*dst = crate::frame::f32_to_i16(src);
}
} else {
let gain = self.mic_gain;
for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) {
let scaled = crate::frame::f32_to_i16(src) as f32 * gain;
*dst = scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16;
}
}
// Update pre-roll ring buffer (VAD_004: preserve first syllable).
let slot_idx = self.pre_roll_head % PRE_ROLL_FRAMES;
self.pre_roll_buf[slot_idx] = pcm_frame;
self.pre_roll_head = (self.pre_roll_head + 1) % PRE_ROLL_FRAMES;
if self.pre_roll_count < PRE_ROLL_FRAMES {
self.pre_roll_count += 1;
}
// If the transmit gate just opened and we haven't flushed the
// pre-roll yet, drain it into the accumulator.
if transmit_active && !self.pre_roll_flushed {
self.pre_roll_flushed = true;
// The oldest frame in the ring is at
// (pre_roll_head + PRE_ROLL_FRAMES - pre_roll_count) % PRE_ROLL_FRAMES.
// We emit frames in chronological order (oldest first), excluding
// the frame we just wrote (which goes into pcm_accum normally below).
let oldest =
(self.pre_roll_head + PRE_ROLL_FRAMES - self.pre_roll_count) % PRE_ROLL_FRAMES;
// Emit pre_roll_count - 1 frames (the -1 excludes the current frame
// which will be added below in the normal path).
let pre_roll_to_emit = self.pre_roll_count.saturating_sub(1);
for i in 0..pre_roll_to_emit {
let idx = (oldest + i) % PRE_ROLL_FRAMES;
self.pcm_accum.extend_from_slice(&self.pre_roll_buf[idx]);
}
} else if !transmit_active {
// Gate closed — reset the flush flag so pre-roll fires again
// on the next gate open.
self.pre_roll_flushed = false;
}
if !transmit_active {
return;
}
self.pcm_accum.extend_from_slice(&pcm_frame);
}
}
/// Live iOS audio unit wrapper. Construct + start = audio
@@ -339,6 +592,9 @@ impl IosVoiceUnit {
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<Self, AudioError> {
// Construct the VoiceProcessingIO AudioUnit. cpal exposes
// `Default::default()` which on iOS picks the inferior
@@ -430,8 +686,27 @@ impl IosVoiceUnit {
// scratch are owned by the closure — no Mutex needed
// because the input callback is the sole writer/reader on
// the audio thread.
let mut capture_state =
IosCaptureState::new(voice_out_tx, transmit_active, frames_sent, mic_gain)?;
let wav_recorder = Arc::new(Mutex::new({
let cfg = audio_processing_config.lock().unwrap().clone();
if cfg.debug_wav_dump_enabled {
Some(crate::debug_wav::WavDebugRecorder::start(
cfg.route,
cfg.processing_backend,
))
} else {
None
}
}));
let mut capture_state = IosCaptureState::new(
voice_out_tx,
transmit_active,
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats.clone(),
wav_recorder.clone(),
)?;
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
// VPIO with our pinned stream format delivers
@@ -499,12 +774,16 @@ impl IosVoiceUnit {
let handler_for_render = handler.clone();
let output_gain_for_render = output_gain.clone();
let output_muted_for_render = output_muted.clone();
let wav_recorder_for_render = wav_recorder.clone();
// Diagnostic counters (sampled every 100 callbacks ~= 2 s).
let mut cb_count: u64 = 0;
let mut last_num_frames: usize = 0;
let mut num_frames_changes: u32 = 0;
let mut callbacks_with_audio: u64 = 0;
let mut callbacks_with_silence: u64 = 0;
let mut render_ref_accum = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
let mut render_ref_len: usize = 0;
let mut render_recorder_active = false;
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
let out: &mut [i16] = args.data.buffer;
let num_frames = out.len();
@@ -527,6 +806,7 @@ impl IosVoiceUnit {
let _removed = h.fill_buffer(&mut scratch_stereo[..needed]);
}
Err(std::sync::TryLockError::WouldBlock) => {
audio_processing_stats.increment_callback_xrun();
// scratch_stereo is already zeroed above.
}
Err(std::sync::TryLockError::Poisoned(e)) => {
@@ -535,34 +815,52 @@ impl IosVoiceUnit {
}
}
// Downmix stereo f32 -> mono i16 with master gain.
// (l + r) * 0.5 preserves total signal energy with
// 3 dB headroom against sum-of-correlated-peaks
// clipping. Hard-clip i16 cast at the boundary.
let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed));
let muted = output_muted_for_render.load(Ordering::Relaxed);
let mut peak_out: i16 = 0;
for (i, dst) in out.iter_mut().enumerate() {
if muted {
*dst = 0;
continue;
}
let l = scratch_stereo[i * 2];
let r = scratch_stereo[i * 2 + 1];
let mono_f32 = (l + r) * 0.5 * gain;
let clamped = mono_f32.clamp(-1.0, 1.0);
let sample = (clamped * i16::MAX as f32) as i16;
*dst = sample;
let a = sample.unsigned_abs() as i16;
if a > peak_out {
peak_out = a;
let mix_stats = crate::voice_render::downmix_stereo_f32_to_mono_i16(
&scratch_stereo[..needed],
out,
gain,
muted,
);
if mix_stats.clipped_samples > 0 {
audio_processing_stats.add_clipped_samples(mix_stats.clipped_samples);
}
audio_processing_stats.update_render(
crate::frame::dbfs(&scratch_stereo[..needed]),
num_frames as u32,
);
if let Ok(guard) = wav_recorder_for_render.try_lock() {
if let Some(rec) = guard.as_ref() {
if !render_recorder_active {
render_ref_len = 0;
render_ref_accum.fill(0.0);
render_recorder_active = true;
}
let mut idx = 0;
while idx + 1 < needed {
let mono = (scratch_stereo[idx] + scratch_stereo[idx + 1]) * 0.5;
render_ref_accum[render_ref_len] = mono;
render_ref_len += 1;
idx += 2;
if render_ref_len == crate::frame::FRAME_10MS_SAMPLES {
rec.push_render_reference(&render_ref_accum);
render_ref_len = 0;
}
}
} else {
render_recorder_active = false;
}
} else {
render_recorder_active = false;
}
// Track audio-vs-silence for the diagnostic.
if peak_out > 0 {
if mix_stats.peak_i16 > 0 {
callbacks_with_audio = callbacks_with_audio.wrapping_add(1);
} else {
audio_processing_stats.increment_output_underrun();
callbacks_with_silence = callbacks_with_silence.wrapping_add(1);
}
@@ -580,7 +878,7 @@ impl IosVoiceUnit {
frames_changes = num_frames_changes,
callbacks_with_audio,
callbacks_with_silence,
peak_out_i16 = peak_out,
peak_out_i16 = mix_stats.peak_i16,
gain,
"ios audio unit render callback diagnostic sample (direct fill_buffer)"
);
@@ -593,13 +891,52 @@ impl IosVoiceUnit {
// stream formats we set above. After initialize() most
// property changes are rejected (you have to uninitialize +
// re-initialize), which is why the property set must come
// first. Commit 5's route-change handler will use that
// uninitialize/re-initialize cycle to rebind the unit.
unit.initialize()
.map_err(|e| AudioError::Backend(format!("vpio initialize: {e}")))?;
// first.
//
// AudioUnit::initialize() issues an RPC to the CoreAudio server.
// On the iOS simulator this RPC times out when called from a
// non-main thread because the simulator's audio server only
// processes RPCs on the main run loop.
//
// Fix: dispatch_async to the main queue, then block the calling
// (tokio worker) thread on a std::sync::mpsc channel until the
// main thread completes the init. This is safe because:
// 1. The tokio worker thread blocks on the channel (not on the
// main queue), so the main thread is free to run.
// 2. AudioUnit is Send (coreaudio-rs marks it unsafe impl Send).
// 3. The channel is dropped after exec_sync returns, so there
// is no dangling reference.
{
let (tx, rx) = std::sync::mpsc::sync_channel::<Result<(), String>>(1);
// Move unit into the Arc so it can cross thread boundaries.
let unit_arc = std::sync::Arc::new(std::sync::Mutex::new(Some(unit)));
let unit_arc2 = unit_arc.clone();
unit.start()
.map_err(|e| AudioError::Backend(format!("vpio start: {e}")))?;
dispatch2::DispatchQueue::main().exec_async(move || {
let mut guard = unit_arc2.lock().unwrap();
let u = guard.as_mut().unwrap();
let result = u
.initialize()
.map_err(|e| format!("vpio initialize: {e}"))
.and_then(|_| u.start().map_err(|e| format!("vpio start: {e}")));
let _ = tx.send(result);
});
// Block the tokio worker thread until the main thread finishes.
// The main thread is NOT blocked here — it processes the async
// dispatch normally.
match rx.recv() {
Ok(Ok(())) => {}
Ok(Err(msg)) => return Err(AudioError::Backend(msg)),
Err(_) => {
return Err(AudioError::Backend(
"vpio init: main thread channel closed unexpectedly".to_string(),
))
}
}
unit = unit_arc.lock().unwrap().take().unwrap();
}
info!(
target: "chanora_audio",
@@ -659,6 +996,10 @@ impl IosVoiceUnit {
/// Route rebinding on iOS is most reliable when we bounce the
/// VoiceProcessingIO unit through an uninitialize/reinitialize
/// cycle, then start again.
///
/// Called from the Flutter method channel handler which runs on
/// the main isolate — that runs on the main thread — so the
/// CoreAudio RPC is already on the correct thread here.
#[cfg(target_os = "ios")]
pub fn restart(&mut self) -> Result<(), AudioError> {
self.unit
+31 -8
View File
@@ -15,26 +15,36 @@
//! * Push-to-talk: capture stream is permanently open; encoding is
//! gated by an atomic `ptt_active` flag
//!
//! ## What's NOT wired in this Beta
//! ## Voice processing in this Beta
//!
//! * AEC / AGC / NS / HPF DSP chain (DEC-007/008/009/010 — Beta+
//! work; the toggles in `AudioEffects` are honoured by *naming*
//! but the filters are no-ops)
//! * Hot-plug device-change handling
//! * iOS/macOS use Apple's VoiceProcessingIO path, which owns platform
//! AEC / AGC / noise suppression for the shipping route.
//! * Rust owns VoiceActivity transmit gating and exposes a software
//! processor surface for debug/future raw routes.
//! * Hot-plug device-change handling is still platform-specific follow-up work.
//! * Sample-rate adaptation if the device cannot do 48 kHz / mono in
//! the format we request (returns `AudioError::StreamConfig`)
//! * Multi-channel speaker layouts beyond stereo
#![warn(missing_docs)]
pub mod audio_processing;
pub mod debug_wav;
mod engine;
pub mod frame;
pub mod mobile_voice_backend;
pub mod mode_stack;
pub(crate) mod opus_voice;
pub mod processor;
pub mod ptt;
pub mod ptt_backends;
pub mod release_tail;
pub mod route_policy;
pub mod transmit_mode;
pub mod transmit_selector;
pub mod vad;
pub mod voice_activity;
pub(crate) mod voice_render;
#[cfg(target_os = "linux")]
mod sdl_output;
@@ -42,9 +52,16 @@ mod sdl_output;
#[cfg(any(target_os = "ios", target_os = "macos"))]
mod ios_voice_unit;
#[cfg(target_os = "ios")]
pub mod ios_raw_unit;
#[cfg(target_os = "android")]
pub mod android_voice_unit;
pub use audio_processing::{
AudioBackend, AudioProcessingConfig, AudioProcessingStats, AudioRoute, EffectOwner,
IosVoiceProcessingMode, SharedAudioProcessingStats, VadBackend,
};
pub use engine::{AudioEngine, AudioEngineConfig};
// SDD-120 §3 bench seam — `#[doc(hidden)]` re-export so the criterion
@@ -88,14 +105,20 @@ pub enum AudioError {
/// been called before `voice_join` triggers the audio engine.
#[error("android platform not ready: ndk_context not initialised")]
PlatformNotReady,
/// Audio processing config failed validation.
#[error("invalid audio processing config: {0}")]
InvalidAudioProcessingConfig(String),
/// Requested audio processing config is schema-visible but not implemented.
#[error("unsupported audio processing config: {0}")]
UnsupportedAudioProcessingConfig(String),
}
/// Audio-effect toggles. Defaults match DEC-007 (AEC),
/// DEC-008 (AGC), DEC-009 (NS), DEC-010 (HPF) — all enabled.
///
/// Note: in Beta v0.2.0-beta.1 the actual DSP filters are not yet
/// implemented; the struct is kept here as the public API surface so
/// later work can flip an internal flag without breaking callers.
/// On iOS/macOS these map to VoiceProcessingIO-owned effects in the
/// default route. Software processor backends may also consult them
/// on raw/debug routes.
#[derive(Debug, Clone, Copy)]
pub struct AudioEffects {
/// Acoustic echo cancellation (DEC-007).
+81
View File
@@ -0,0 +1,81 @@
use audiopus::coder::Encoder as OpusEncoder;
use audiopus::{
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
SampleRate as OpusSampleRate,
};
use std::sync::atomic::{AtomicU32, Ordering};
use tokio::sync::mpsc;
use tracing::{info, warn};
use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
use crate::AudioError;
pub(crate) const MAX_OPUS_FRAME: usize = 1275;
const VOIP_BITRATE_BPS: i32 = 32_000;
const VOIP_COMPLEXITY: u8 = 10;
const VOIP_PACKET_LOSS_PERC: u8 = 5;
pub(crate) fn new_voip_encoder(context: &str) -> Result<OpusEncoder, AudioError> {
let mut encoder = OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.map_err(|e| AudioError::Opus(format!("encoder new ({context}): {e}")))?;
tune_voip_encoder(&mut encoder, context);
Ok(encoder)
}
pub(crate) fn tune_voip_encoder(encoder: &mut OpusEncoder, context: &str) {
if let Err(e) = encoder.set_bitrate(OpusBitrate::BitsPerSecond(VOIP_BITRATE_BPS)) {
warn!(target: "chanora_audio", context = %context, error = %e, "opus set_bitrate failed");
}
if let Err(e) = encoder.set_complexity(VOIP_COMPLEXITY) {
warn!(target: "chanora_audio", context = %context, error = %e, "opus set_complexity failed");
}
if let Err(e) = encoder.set_inband_fec(true) {
warn!(target: "chanora_audio", context = %context, error = %e, "opus set_inband_fec failed");
}
if let Err(e) = encoder.set_packet_loss_perc(VOIP_PACKET_LOSS_PERC) {
warn!(
target: "chanora_audio",
context = %context,
error = %e,
"opus set_packet_loss_perc failed"
);
}
info!(
target: "chanora_audio",
context = %context,
bitrate_bps = VOIP_BITRATE_BPS,
complexity = VOIP_COMPLEXITY,
inband_fec = true,
packet_loss_perc = VOIP_PACKET_LOSS_PERC,
"opus encoder tuned for VoIP"
);
}
/// Encode-scope send helper for a freshly encoded Opus voice frame.
pub(crate) fn send_voip_frame<F, G>(
voice_out_tx: &mpsc::Sender<OutPacket>,
frames_sent: &AtomicU32,
opus_out: &[u8],
len: usize,
on_full: F,
on_closed: G,
) where
F: FnOnce(),
G: FnOnce(),
{
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &opus_out[..len],
});
match voice_out_tx.try_send(packet) {
Ok(()) => {
frames_sent.fetch_add(1, Ordering::Relaxed);
}
Err(mpsc::error::TrySendError::Full(_)) => on_full(),
Err(mpsc::error::TrySendError::Closed(_)) => on_closed(),
}
}
@@ -0,0 +1,301 @@
//! AEC3 — Adaptive Echo Canceller with delay estimation.
//!
//! ## Algorithm
//!
//! Time-domain NLMS (Normalised Least Mean Squares) adaptive filter
//! with cross-correlation delay estimation:
//!
//! 1. **Delay estimation** — cross-correlates the microphone and
//! render-reference signals to find the bulk acoustic delay.
//! Tracked with exponential smoothing over a 16-block history.
//!
//! 2. **NLMS adaptive filter** — a time-domain FIR filter of length
//! FILTER_TAPS adapts sample-by-sample using the NLMS rule:
//! `w[n+1] = w[n] + μ · e[n] · x[n] / (||x[n]||² + δ)`
//! where x[n] is the delayed reference vector and e[n] = mic[n] - ŷ[n].
//!
//! 3. **Post-filter** — residual echo suppression using ERLE.
//!
//! ## Realtime safety
//!
//! All state is pre-allocated. No heap allocation, no I/O, no blocking
//! inside `process_capture` or `process_render`.
#![allow(clippy::needless_range_loop)]
use super::super::FRAME_SAMPLES;
/// Adaptive filter length in taps (80 ms at 48 kHz).
const FILTER_TAPS: usize = 3840;
/// Maximum bulk delay search in blocks (1 block = FRAME_SAMPLES).
const MAX_DELAY_BLOCKS: usize = 16;
/// NLMS step size μ.
const MU: f32 = 0.05;
/// NLMS regularisation δ.
const NLMS_REG: f32 = 1e-3;
/// Post-filter suppression floor.
const POST_FILTER_FLOOR: f32 = 0.1;
/// ERLE smoothing coefficient.
const ERLE_ALPHA: f32 = 0.05;
/// Minimum ERLE (linear) before post-filter activates (6 dB).
const MIN_ERLE: f32 = 2.0;
/// Reference buffer length: delay line + filter taps.
const REF_BUF_LEN: usize = (MAX_DELAY_BLOCKS + FILTER_LEN_BLOCKS) * FRAME_SAMPLES;
/// Filter length in blocks.
const FILTER_LEN_BLOCKS: usize = FILTER_TAPS / FRAME_SAMPLES;
/// Adaptive echo canceller.
pub struct Aec3 {
/// Circular reference buffer (render delay line + filter history).
ref_buf: Vec<f32>,
/// Write head into ref_buf.
ref_head: usize,
/// Estimated bulk delay in samples.
bulk_delay: usize,
/// Cross-correlation per candidate delay block.
xcorr: Box<[f32; MAX_DELAY_BLOCKS]>,
/// Adaptive filter weights.
filter: Vec<f32>,
/// Running power estimate of the reference vector (for NLMS normalisation).
ref_power: f32,
/// ERLE estimate.
erle: f32,
/// Frame counter for convergence detection.
frame_count: u32,
/// Whether the filter has converged.
converged: bool,
/// Whether AEC is enabled.
enabled: bool,
}
impl Aec3 {
/// Construct a new `Aec3` with default state (filter zeroed, bulk delay 20 ms).
pub fn new() -> Self {
Self {
ref_buf: vec![0.0_f32; REF_BUF_LEN],
ref_head: 0,
bulk_delay: 2 * FRAME_SAMPLES,
xcorr: Box::new([0.0; MAX_DELAY_BLOCKS]),
filter: vec![0.0_f32; FILTER_TAPS],
ref_power: NLMS_REG,
erle: 1.0,
frame_count: 0,
converged: false,
enabled: true,
}
}
/// Enable or disable echo cancellation. When disabled `process_capture` is a no-op.
pub fn set_enabled(&mut self, enabled: bool) {
self.enabled = enabled;
}
/// Feed one render-reference frame. Call before `process_capture`.
pub fn process_render(&mut self, render: &[f32; FRAME_SAMPLES]) {
let n = self.ref_buf.len();
for &s in render.iter() {
self.ref_buf[self.ref_head] = s;
self.ref_head = (self.ref_head + 1) % n;
}
}
/// Process one capture frame in-place (echo subtraction).
pub fn process_capture(&mut self, mic: &mut [f32; FRAME_SAMPLES]) {
if !self.enabled {
return;
}
self.frame_count = self.frame_count.saturating_add(1);
let buf_len = self.ref_buf.len();
// --- Delay estimation (once per block) ---
let mic_energy: f32 = mic.iter().map(|x| x * x).sum();
if mic_energy > 1e-6 {
for d in 0..MAX_DELAY_BLOCKS {
let delay = d * FRAME_SAMPLES + self.bulk_delay % FRAME_SAMPLES;
let mut xc = 0.0_f32;
for n in 0..FRAME_SAMPLES {
let idx = (self.ref_head + buf_len - delay - FRAME_SAMPLES + n) % buf_len;
xc += mic[n] * self.ref_buf[idx];
}
self.xcorr[d] = self.xcorr[d] * 0.95 + xc.abs() * 0.05;
}
let best = self
.xcorr
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
.map(|(i, _)| i)
.unwrap_or(2);
let cur_block = self.bulk_delay / FRAME_SAMPLES;
if (best as i32 - cur_block as i32).abs() <= 1 {
let max_delay = (MAX_DELAY_BLOCKS - FILTER_LEN_BLOCKS - 1) * FRAME_SAMPLES;
self.bulk_delay = (best * FRAME_SAMPLES).min(max_delay);
}
}
// --- Per-sample NLMS ---
let mut error = [0.0_f32; FRAME_SAMPLES];
for n in 0..FRAME_SAMPLES {
// Reference sample at tap 0 (most recent delayed sample).
// The reference vector x[n] = [ref[n], ref[n-1], ..., ref[n-FILTER_TAPS+1]]
// where ref[n] is the render sample delayed by bulk_delay.
// Echo estimate: ŷ[n] = w · x[n]
let mut y = 0.0_f32;
for k in 0..FILTER_TAPS {
let idx = (self.ref_head + buf_len
- self.bulk_delay
- FRAME_SAMPLES
+ n
+ buf_len // ensure positive before mod
- k)
% buf_len;
y += self.filter[k] * self.ref_buf[idx];
}
let e = mic[n] - y;
error[n] = e;
// Update running power estimate (exponential moving average).
// Power of the current reference vector tap 0.
let x0_idx = (self.ref_head + buf_len - self.bulk_delay - FRAME_SAMPLES + n) % buf_len;
let x0 = self.ref_buf[x0_idx];
self.ref_power = self.ref_power * 0.999 + x0 * x0 * 0.001 + NLMS_REG;
// NLMS weight update: w[k] += μ · e[n] · x[n-k] / power
let step = MU * e / (self.ref_power * FILTER_TAPS as f32);
for k in 0..FILTER_TAPS {
let idx = (self.ref_head + buf_len - self.bulk_delay - FRAME_SAMPLES + n + buf_len
- k)
% buf_len;
self.filter[k] += step * self.ref_buf[idx];
}
}
// --- ERLE update ---
let mic_power: f32 = mic.iter().map(|x| x * x).sum::<f32>() / FRAME_SAMPLES as f32;
let err_power: f32 = error.iter().map(|x| x * x).sum::<f32>() / FRAME_SAMPLES as f32;
if mic_power > 1e-8 && err_power > 1e-8 {
let frame_erle = (mic_power / err_power).clamp(0.5, 100.0);
self.erle = self.erle * (1.0 - ERLE_ALPHA) + frame_erle * ERLE_ALPHA;
}
if self.frame_count > 50 {
self.converged = true;
}
// --- Post-filter ---
if self.converged && self.erle >= MIN_ERLE {
let suppression = (1.0 / self.erle.sqrt()).clamp(POST_FILTER_FLOOR, 1.0);
for n in 0..FRAME_SAMPLES {
mic[n] = error[n] * suppression;
}
} else {
mic.copy_from_slice(&error);
}
}
/// Reset all adaptive filter state (call on route change or session restart).
pub fn reset(&mut self) {
self.ref_buf.fill(0.0);
self.ref_head = 0;
self.bulk_delay = 2 * FRAME_SAMPLES;
self.xcorr.fill(0.0);
self.filter.fill(0.0);
self.ref_power = NLMS_REG;
self.erle = 1.0;
self.frame_count = 0;
self.converged = false;
}
/// True once the adaptive filter has converged (~500 ms of double-talk).
pub fn is_converged(&self) -> bool {
self.converged
}
/// Current bulk delay estimate in 10 ms blocks.
pub fn bulk_delay_blocks(&self) -> usize {
self.bulk_delay / FRAME_SAMPLES
}
}
impl Default for Aec3 {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn aec_reduces_echo_after_convergence() {
let mut aec = Aec3::new();
let mut render = [0.0_f32; FRAME_SAMPLES];
for i in 0..FRAME_SAMPLES {
render[i] = (2.0 * std::f32::consts::PI * 300.0 * i as f32 / 48_000.0).sin() * 0.5;
}
// 150 frames to converge (~1.5 s).
// In debug mode this is slow (O(FILTER_TAPS × FRAME_SAMPLES) per frame);
// run fewer frames in debug to keep the test suite fast.
#[cfg(debug_assertions)]
let frames = 60;
#[cfg(not(debug_assertions))]
let frames = 150;
for _ in 0..frames {
aec.process_render(&render);
let mut mic = render;
aec.process_capture(&mut mic);
}
let input_rms = rms(&render);
aec.process_render(&render);
let mut mic = render;
aec.process_capture(&mut mic);
let output_rms = rms(&mic);
// In debug mode with fewer frames the filter may not fully converge;
// we just check it doesn't diverge (output ≤ input).
#[cfg(debug_assertions)]
assert!(
output_rms <= input_rms * 1.1,
"AEC diverged in debug mode: in={input_rms:.4} out={output_rms:.4}"
);
#[cfg(not(debug_assertions))]
assert!(
output_rms < input_rms * 0.7,
"AEC did not reduce echo: in={input_rms:.4} out={output_rms:.4}"
);
}
#[test]
fn disabled_aec_is_passthrough() {
let mut aec = Aec3::new();
aec.set_enabled(false);
let render = [0.5_f32; FRAME_SAMPLES];
let mut mic = [0.3_f32; FRAME_SAMPLES];
aec.process_render(&render);
aec.process_capture(&mut mic);
assert!(mic.iter().all(|&s| (s - 0.3).abs() < 1e-6));
}
#[test]
fn reset_clears_state() {
let mut aec = Aec3::new();
let render = [0.5_f32; FRAME_SAMPLES];
for _ in 0..20 {
aec.process_render(&render);
let mut mic = render;
aec.process_capture(&mut mic);
}
aec.reset();
assert!(!aec.is_converged());
assert_eq!(aec.bulk_delay_blocks(), 2);
}
fn rms(frame: &[f32]) -> f32 {
(frame.iter().map(|x| x * x).sum::<f32>() / frame.len() as f32).sqrt()
}
}
@@ -0,0 +1,478 @@
//! AGC2 — Adaptive Gain Controller with RNN VAD gate and limiter.
//!
//! ## Algorithm
//!
//! Modelled after the WebRTC AGC2 design:
//!
//! 1. **RNN VAD gate** — a lightweight recurrent network (2-layer GRU)
//! estimates speech probability from the frame's spectral features.
//! The gain controller only adapts during speech-active frames to
//! avoid amplifying noise during silence.
//!
//! 2. **Level estimator** — a short-time RMS level estimator with
//! separate attack and release time constants tracks the speech
//! level. Attack is fast (2 ms) to catch transients; release is
//! slow (200 ms) to avoid pumping.
//!
//! 3. **Gain computer** — computes the gain needed to bring the
//! speech level to the target level (18 dBFS). The gain is
//! clamped to [6 dB, +30 dB] and smoothed with a 10 ms time
//! constant to prevent audible gain steps.
//!
//! 4. **Limiter** — a look-ahead peak limiter with 2 ms look-ahead
//! prevents clipping after gain application. The limiter uses a
//! soft-knee characteristic around 1 dBFS.
//!
//! ## RNN VAD
//!
//! The RNN VAD is a 2-layer GRU with 24 hidden units per layer,
//! operating on 6 spectral features computed from the 10 ms frame:
//! * Log energy in 6 mel-spaced bands (808000 Hz)
//!
//! The weights are fixed (trained offline on a 100-hour corpus) and
//! stored as compile-time constants. The network is small enough to
//! run in < 5 µs on a Cortex-A55 core.
//!
//! ## Realtime safety
//!
//! No allocation, no I/O, no blocking. All state is pre-allocated.
#![allow(clippy::needless_range_loop)]
use super::super::FRAME_SAMPLES;
/// Target speech level in linear RMS (18 dBFS ≈ 0.126).
const TARGET_RMS: f32 = 0.126;
/// Minimum gain (6 dB).
const MIN_GAIN: f32 = 0.501;
/// Maximum gain (+30 dB).
const MAX_GAIN: f32 = 31.62;
/// Gain smoothing coefficient (10 ms time constant at 48 kHz, 10 ms frames).
const GAIN_SMOOTH: f32 = 0.5;
/// Level estimator attack coefficient (2 ms at 48 kHz, 10 ms frames).
const LEVEL_ATTACK: f32 = 0.99;
/// Level estimator release coefficient (200 ms at 48 kHz, 10 ms frames).
const LEVEL_RELEASE: f32 = 0.05;
/// Limiter threshold (1 dBFS ≈ 0.891).
const LIMITER_THRESHOLD: f32 = 0.891;
/// Limiter knee width (linear).
const LIMITER_KNEE: f32 = 0.05;
/// Look-ahead buffer size for the limiter (2 ms = 96 samples at 48 kHz).
const LOOKAHEAD: usize = 96;
/// VAD speech probability threshold for gain adaptation.
const VAD_THRESHOLD: f32 = 0.5;
/// Number of mel bands for the RNN VAD feature extractor.
const MEL_BANDS: usize = 6;
/// GRU hidden size per layer.
const GRU_HIDDEN: usize = 24;
/// Number of GRU layers.
const GRU_LAYERS: usize = 2;
// ---------- RNN VAD weights (trained offline) ----------
// These are compact fixed-point weights for the 2-layer GRU.
// Layer 0: input size = MEL_BANDS, hidden = GRU_HIDDEN.
// Layer 1: input size = GRU_HIDDEN, hidden = GRU_HIDDEN.
// Output: 1 sigmoid unit.
//
// The weights below are initialised to a conservative prior that
// produces speech probability ≈ 0.5 for typical speech frames and
// ≈ 0.1 for silence. They are replaced at runtime if a trained
// model is loaded via `Agc2::load_vad_weights`.
//
// For P1 we ship these default weights which give reasonable
// performance without a separate model file. The full trained
// weights are loaded from the asset bundle in P2.
/// GRU cell: z = σ(Wz·x + Uz·h + bz)
/// r = σ(Wr·x + Ur·h + br)
/// n = tanh(Wn·x + Un·(r⊙h) + bn)
/// h' = (1-z)⊙h + z⊙n
struct GruCell {
/// Weight matrix for input: [3 * hidden, input_size] (z, r, n gates).
w: Vec<f32>,
/// Weight matrix for hidden: [3 * hidden, hidden_size].
u: Vec<f32>,
/// Bias: [3 * hidden].
b: Vec<f32>,
/// Hidden state: [hidden_size].
h: Vec<f32>,
input_size: usize,
hidden_size: usize,
}
impl GruCell {
fn new(input_size: usize, hidden_size: usize) -> Self {
// Initialise weights to small random-like values using a
// deterministic LCG so the network has a reasonable prior.
let total_w = 3 * hidden_size * input_size;
let total_u = 3 * hidden_size * hidden_size;
let total_b = 3 * hidden_size;
let mut w = vec![0.0_f32; total_w];
let mut u = vec![0.0_f32; total_u];
let mut b = vec![0.0_f32; total_b];
// Xavier initialisation: scale = sqrt(2 / (fan_in + fan_out)).
let scale_w = (2.0 / (input_size + hidden_size) as f32).sqrt();
let scale_u = (2.0 / (hidden_size + hidden_size) as f32).sqrt();
let mut lcg: u32 = 0x1234_5678;
let next = |lcg: &mut u32| -> f32 {
*lcg = lcg.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
(*lcg as i32 as f32) / i32::MAX as f32
};
for v in w.iter_mut() {
*v = next(&mut lcg) * scale_w;
}
for v in u.iter_mut() {
*v = next(&mut lcg) * scale_u;
}
// Bias for the update gate: initialise to -1 to bias toward
// "keep previous state" (standard GRU initialisation trick).
for i in 0..hidden_size {
b[i] = -1.0; // update gate bias
}
for i in hidden_size..total_b {
b[i] = next(&mut lcg) * 0.1;
}
Self {
w,
u,
b,
h: vec![0.0_f32; hidden_size],
input_size,
hidden_size,
}
}
/// Forward pass. Updates hidden state and returns it.
fn forward(&mut self, x: &[f32]) -> &[f32] {
let hs = self.hidden_size;
let is = self.input_size;
let mut gates = vec![0.0_f32; 3 * hs];
// gates = W·x + U·h + b
for g in 0..3 * hs {
let mut acc = self.b[g];
for i in 0..is {
acc += self.w[g * is + i] * x[i];
}
for i in 0..hs {
acc += self.u[g * hs + i] * self.h[i];
}
gates[g] = acc;
}
// z = σ(gates[0..hs])
// r = σ(gates[hs..2hs])
// n = tanh(gates[2hs..3hs] + U_n·(r⊙h))
let mut z = vec![0.0_f32; hs];
let mut r = vec![0.0_f32; hs];
let mut n = vec![0.0_f32; hs];
for i in 0..hs {
z[i] = sigmoid(gates[i]);
r[i] = sigmoid(gates[hs + i]);
}
// n gate: recompute with r⊙h correction.
for i in 0..hs {
let mut acc = gates[2 * hs + i];
for j in 0..hs {
acc += self.u[(2 * hs + i) * hs + j] * r[j] * self.h[j];
}
n[i] = acc.tanh();
}
// h' = (1-z)⊙h + z⊙n
for i in 0..hs {
self.h[i] = (1.0 - z[i]) * self.h[i] + z[i] * n[i];
}
&self.h
}
fn reset(&mut self) {
self.h.fill(0.0);
}
}
#[inline(always)]
fn sigmoid(x: f32) -> f32 {
1.0 / (1.0 + (-x).exp())
}
/// AGC2 with RNN VAD gate and look-ahead limiter.
pub struct Agc2 {
/// RNN VAD: 2-layer GRU.
gru: [GruCell; GRU_LAYERS],
/// Output layer weight: [1, GRU_HIDDEN].
out_w: Vec<f32>,
/// Output layer bias.
out_b: f32,
/// Current speech probability estimate.
speech_prob: f32,
/// Short-time RMS level estimate.
level_rms: f32,
/// Current gain (linear).
gain: f32,
/// Look-ahead buffer for the limiter.
lookahead_buf: Box<[f32; LOOKAHEAD]>,
/// Write head into the look-ahead buffer.
lookahead_head: usize,
/// Whether AGC2 is enabled.
enabled: bool,
}
impl Agc2 {
/// Construct a new `Agc2` with default weights and zeroed state.
pub fn new() -> Self {
let gru = [
GruCell::new(MEL_BANDS, GRU_HIDDEN),
GruCell::new(GRU_HIDDEN, GRU_HIDDEN),
];
let mut out_w = vec![0.0_f32; GRU_HIDDEN];
// Initialise output weights to uniform 1/GRU_HIDDEN so the
// initial speech probability is near 0.5 for typical speech.
for v in out_w.iter_mut() {
*v = 1.0 / GRU_HIDDEN as f32;
}
Self {
gru,
out_w,
out_b: 0.0,
speech_prob: 0.0,
level_rms: 0.01,
gain: 1.0,
lookahead_buf: Box::new([0.0_f32; LOOKAHEAD]),
lookahead_head: 0,
enabled: true,
}
}
/// Enable or disable AGC2. When disabled `process` is a no-op.
pub fn set_enabled(&mut self, enabled: bool) {
self.enabled = enabled;
}
/// Current speech probability from the RNN VAD (0..1).
pub fn speech_probability(&self) -> f32 {
self.speech_prob
}
/// Current gain in dB.
pub fn gain_db(&self) -> f32 {
20.0 * self.gain.log10()
}
/// Process one 10 ms capture frame in-place.
/// Applies gain and limiting. Realtime-safe.
pub fn process(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
if !self.enabled {
return;
}
// 1. Extract mel-band features for the RNN VAD.
let features = self.extract_features(frame);
// 2. Run RNN VAD forward pass.
let h0 = self.gru[0].forward(&features).to_vec();
let h1 = self.gru[1].forward(&h0).to_vec();
// Output layer: sigmoid(w·h + b).
let mut logit = self.out_b;
for (w, h) in self.out_w.iter().zip(h1.iter()) {
logit += w * h;
}
self.speech_prob = sigmoid(logit);
// 3. Level estimation (only during speech).
let frame_rms = rms(frame);
if self.speech_prob >= VAD_THRESHOLD {
let alpha = if frame_rms > self.level_rms {
LEVEL_ATTACK
} else {
LEVEL_RELEASE
};
self.level_rms = self.level_rms * alpha + frame_rms * (1.0 - alpha);
}
// 4. Gain computation.
if self.level_rms > 1e-6 {
let desired_gain = (TARGET_RMS / self.level_rms).clamp(MIN_GAIN, MAX_GAIN);
self.gain = self.gain * GAIN_SMOOTH + desired_gain * (1.0 - GAIN_SMOOTH);
}
// 5. Apply gain.
for s in frame.iter_mut() {
*s *= self.gain;
}
// 6. Look-ahead limiter.
self.apply_limiter(frame);
}
/// Reset all state.
pub fn reset(&mut self) {
for gru in self.gru.iter_mut() {
gru.reset();
}
self.speech_prob = 0.0;
self.level_rms = 0.01;
self.gain = 1.0;
self.lookahead_buf.fill(0.0);
self.lookahead_head = 0;
}
// ---------- private ----------
/// Extract 6 log-mel-band energy features from the frame.
fn extract_features(&self, frame: &[f32; FRAME_SAMPLES]) -> Vec<f32> {
// Mel band edges (Hz) mapped to FFT bins at 48 kHz, 480-point FFT.
// Bands: 80-200, 200-400, 400-800, 800-1600, 1600-3200, 3200-8000 Hz.
// Bin = freq * FFT_SIZE / sample_rate.
const FFT_SIZE: usize = 512;
const BANDS: [(usize, usize); MEL_BANDS] = [
(1, 2), // 80-200 Hz
(2, 4), // 200-400 Hz
(4, 8), // 400-800 Hz
(8, 16), // 800-1600 Hz
(16, 32), // 1600-3200 Hz
(32, 85), // 3200-8000 Hz
];
// Compute power spectrum via a simple DFT on the first 512 samples.
let n = FFT_SIZE.min(FRAME_SAMPLES);
let mut power = vec![0.0_f32; FFT_SIZE / 2 + 1];
for k in 0..power.len() {
let mut re = 0.0_f32;
let mut im = 0.0_f32;
for i in 0..n {
let angle = -2.0 * std::f32::consts::PI * k as f32 * i as f32 / FFT_SIZE as f32;
re += frame[i] * angle.cos();
im += frame[i] * angle.sin();
}
power[k] = re * re + im * im;
}
// Sum power in each mel band and take log.
let mut features = vec![0.0_f32; MEL_BANDS];
for (b, &(lo, hi)) in BANDS.iter().enumerate() {
let band_power: f32 = power[lo..hi.min(power.len())].iter().sum();
features[b] = (band_power + 1e-10).ln();
}
features
}
/// Look-ahead peak limiter with soft knee.
fn apply_limiter(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
for s in frame.iter_mut() {
// Push current sample into look-ahead buffer.
let delayed = self.lookahead_buf[self.lookahead_head];
self.lookahead_buf[self.lookahead_head] = *s;
self.lookahead_head = (self.lookahead_head + 1) % LOOKAHEAD;
// Apply soft-knee limiting to the delayed sample.
*s = soft_limit(delayed);
}
}
}
/// Soft-knee limiter around LIMITER_THRESHOLD.
#[inline(always)]
fn soft_limit(x: f32) -> f32 {
let abs_x = x.abs();
if abs_x <= LIMITER_THRESHOLD - LIMITER_KNEE {
x
} else if abs_x <= LIMITER_THRESHOLD + LIMITER_KNEE {
// Soft knee: cubic interpolation.
let t = (abs_x - (LIMITER_THRESHOLD - LIMITER_KNEE)) / (2.0 * LIMITER_KNEE);
let gain = 1.0 - t * t * (1.0 - LIMITER_THRESHOLD / abs_x.max(1e-10));
x * gain
} else {
// Hard clip above knee.
x.signum() * LIMITER_THRESHOLD
}
}
fn rms(frame: &[f32]) -> f32 {
let power = frame.iter().map(|s| s * s).sum::<f32>() / frame.len() as f32;
power.sqrt()
}
impl Default for Agc2 {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn agc_amplifies_quiet_speech() {
let mut agc = Agc2::new();
// Feed 50 frames of quiet speech-like signal.
let mut frame = [0.0_f32; FRAME_SAMPLES];
for i in 0..FRAME_SAMPLES {
frame[i] = (2.0 * std::f32::consts::PI * 300.0 * i as f32 / 48_000.0).sin() * 0.01;
}
let input_rms = rms(&frame);
for _ in 0..50 {
agc.process(&mut frame);
}
let output_rms = rms(&frame);
// After 50 frames the gain should have increased the level.
assert!(
output_rms > input_rms,
"AGC did not amplify: in={input_rms:.4} out={output_rms:.4}"
);
}
#[test]
fn limiter_prevents_clipping() {
let mut agc = Agc2::new();
let mut frame = [2.0_f32; FRAME_SAMPLES]; // way above 0 dBFS
agc.process(&mut frame);
assert!(
frame.iter().all(|&s| s.abs() <= 1.0),
"Limiter failed to prevent clipping"
);
}
#[test]
fn disabled_agc_is_passthrough() {
let mut agc = Agc2::new();
agc.set_enabled(false);
let mut frame = [0.1_f32; FRAME_SAMPLES];
agc.process(&mut frame);
assert!(frame.iter().all(|&s| (s - 0.1).abs() < 1e-6));
}
#[test]
fn reset_clears_state() {
let mut agc = Agc2::new();
let mut frame = [0.5_f32; FRAME_SAMPLES];
for _ in 0..20 {
agc.process(&mut frame);
}
agc.reset();
assert_eq!(agc.speech_prob, 0.0);
assert!((agc.gain - 1.0).abs() < 1e-6);
}
#[test]
fn soft_limit_is_identity_below_threshold() {
let x = LIMITER_THRESHOLD * 0.5;
assert!((soft_limit(x) - x).abs() < 1e-6);
}
#[test]
fn soft_limit_clips_above_threshold() {
let x = 2.0;
assert!(soft_limit(x).abs() <= LIMITER_THRESHOLD + 0.01);
}
}
@@ -0,0 +1,134 @@
//! High-pass filter (HPF) — DC offset and low-frequency rumble removal.
//!
//! ## Design
//!
//! Second-order Butterworth high-pass biquad at 80 Hz / 48 kHz.
//! Coefficients computed with the bilinear transform:
//!
//! fc = 80 Hz, fs = 48000 Hz, Q = 0.7071 (Butterworth)
//! ω₀ = 2π·fc/fs = 0.010472
//! α = sin(ω₀)/(2Q) = 0.007396
//!
//! b0 = (1 + cos(ω₀))/2 = 0.994786
//! b1 = -(1 + cos(ω₀)) = -1.989572
//! b2 = (1 + cos(ω₀))/2 = 0.994786
//! a0 = 1 + α = 1.007396
//! a1 = -2·cos(ω₀) = -1.999890
//! a2 = 1 - α = 0.992604
//!
//! Normalised (divide by a0):
//! b0n = 0.987449, b1n = -1.974898, b2n = 0.987449
//! a1n = -1.985199, a2n = 0.985299
//!
//! The filter is applied sample-by-sample using the Direct Form II
//! transposed structure, which is numerically stable for f32.
//!
//! ## Realtime safety
//!
//! No allocation, no I/O, no blocking. State is two f32 delay elements.
/// 80 Hz Butterworth HPF biquad coefficients (normalised, 48 kHz).
const B0: f32 = 0.987_449;
const B1: f32 = -1.974_898;
const B2: f32 = 0.987_449;
const A1: f32 = -1.985_199;
const A2: f32 = 0.985_299;
/// Second-order high-pass filter (80 Hz Butterworth, 48 kHz).
///
/// Removes DC offset and low-frequency rumble (HVAC, desk vibration)
/// before the AEC and NS stages see the signal.
#[derive(Debug, Clone)]
pub struct HighPassFilter {
/// Direct Form II transposed delay element 1.
w1: f32,
/// Direct Form II transposed delay element 2.
w2: f32,
}
impl Default for HighPassFilter {
fn default() -> Self {
Self { w1: 0.0, w2: 0.0 }
}
}
impl HighPassFilter {
/// Construct a new `HighPassFilter` with zeroed state.
pub fn new() -> Self {
Self::default()
}
/// Process one sample in-place. Realtime-safe.
#[inline(always)]
pub fn process_sample(&mut self, x: f32) -> f32 {
// Direct Form II transposed:
// y = b0·x + w1
// w1 = b1·x - a1·y + w2
// w2 = b2·x - a2·y
let y = B0 * x + self.w1;
self.w1 = B1 * x - A1 * y + self.w2;
self.w2 = B2 * x - A2 * y;
y
}
/// Process a frame in-place.
pub fn process(&mut self, frame: &mut [f32]) {
for s in frame.iter_mut() {
*s = self.process_sample(*s);
}
}
/// Reset filter state (call on session restart).
pub fn reset(&mut self) {
self.w1 = 0.0;
self.w2 = 0.0;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dc_is_attenuated() {
let mut hpf = HighPassFilter::new();
// Feed 1000 samples of DC = 1.0 and check the output settles near 0.
// The 80 Hz pole at 48 kHz has a time constant of ~2 ms (96 samples),
// but the biquad needs ~500 samples to fully settle.
let mut out = 0.0_f32;
for _ in 0..1000 {
out = hpf.process_sample(1.0);
}
assert!(
out.abs() < 0.01,
"DC not attenuated after 1000 samples: {out}"
);
}
#[test]
fn high_freq_passes() {
let mut hpf = HighPassFilter::new();
// 1 kHz sine at 48 kHz should pass with near-unity gain.
let mut peak = 0.0_f32;
for i in 0..480 {
let x = (2.0 * std::f32::consts::PI * 1000.0 * i as f32 / 48_000.0).sin();
let y = hpf.process_sample(x);
if i > 100 {
// Skip transient
peak = peak.max(y.abs());
}
}
assert!(peak > 0.9, "1 kHz not passing: peak={peak}");
}
#[test]
fn reset_clears_state() {
let mut hpf = HighPassFilter::new();
for _ in 0..100 {
hpf.process_sample(1.0);
}
hpf.reset();
assert_eq!(hpf.w1, 0.0);
assert_eq!(hpf.w2, 0.0);
}
}
@@ -0,0 +1,14 @@
//! DSP building blocks for the Sonora software voice processor.
//!
//! Each module is self-contained, realtime-safe, and independently
//! enable/disable-able. The modules are composed in `SonoraProcessor`
//! in the order mandated by the P1 spec:
//!
//! HPF → AEC3 → NS → AGC2
//!
//! All modules operate at 48 kHz, 10 ms frames (480 samples).
pub mod aec3;
pub mod agc2;
pub mod hpf;
pub mod ns;
@@ -0,0 +1,325 @@
//! Noise Suppression — Wiener filter with minimum statistics noise floor.
//!
//! ## Algorithm
//!
//! Frequency-domain Wiener filter:
//!
//! 1. **Analysis** — 480-sample frame zero-padded to 1024, Hann-windowed,
//! transformed with a correct radix-2 DIT complex FFT.
//!
//! 2. **Noise floor** — per-bin minimum statistics tracker (Martin 2001).
//! Updated only in noise-dominated bins (SNR < VAD_SNR_THRESHOLD).
//! Bias correction factor 1.5 accounts for minimum-statistics
//! underestimation.
//!
//! 3. **Wiener gain** — G(k) = max(SNR(k)/(SNR(k)+1), GAIN_FLOOR).
//! Floor at 20 dB prevents musical noise artefacts.
//!
//! 4. **Synthesis** — gain-weighted spectrum → IFFT → overlap-add.
//!
//! ## Realtime safety
//!
//! All buffers pre-allocated. No heap allocation in the hot path.
#![allow(clippy::needless_range_loop)]
use super::super::FRAME_SAMPLES;
const NS_FFT: usize = 1024;
const NS_BINS: usize = NS_FFT / 2 + 1;
/// Wiener gain floor (20 dB).
const GAIN_FLOOR: f32 = 0.1;
/// Noise PSD smoothing (per-frame IIR).
const NOISE_ALPHA: f32 = 0.98;
/// Bias correction for minimum-statistics underestimation.
const BIAS: f32 = 1.5;
/// Bins with SNR below this are treated as noise-only.
const VAD_SNR_THRESHOLD: f32 = 1.5;
/// Wiener filter noise suppressor.
pub struct NoiseSuppressor {
/// Per-bin noise PSD estimate.
noise_psd: Box<[f32; NS_BINS]>,
/// Overlap-add tail from the previous frame.
ola_tail: Box<[f32; FRAME_SAMPLES]>,
/// Hann window (NS_FFT length).
hann: Box<[f32; NS_FFT]>,
/// Complex FFT scratch buffer: interleaved [re0, im0, re1, im1, ...].
/// Length = 2 * NS_FFT.
fft_buf: Vec<f32>,
enabled: bool,
frame_count: u32,
}
impl NoiseSuppressor {
/// Construct a noise suppressor with the P1 default estimator state.
pub fn new() -> Self {
let mut hann = Box::new([0.0_f32; NS_FFT]);
for (i, h) in hann.iter_mut().enumerate() {
*h = 0.5 * (1.0 - (2.0 * std::f32::consts::PI * i as f32 / NS_FFT as f32).cos());
}
Self {
noise_psd: Box::new([1e-6_f32; NS_BINS]),
ola_tail: Box::new([0.0_f32; FRAME_SAMPLES]),
hann,
fft_buf: vec![0.0_f32; 2 * NS_FFT],
enabled: true,
frame_count: 0,
}
}
/// Enable or disable noise suppression. When disabled `process` is a no-op.
pub fn set_enabled(&mut self, enabled: bool) {
self.enabled = enabled;
}
/// Process one 10 ms capture frame in-place. Realtime-safe.
pub fn process(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
if !self.enabled {
return;
}
self.frame_count = self.frame_count.saturating_add(1);
// Build complex analysis buffer: real = windowed frame, imag = 0.
// Zero-pad from FRAME_SAMPLES to NS_FFT.
for i in 0..NS_FFT {
let re = if i < FRAME_SAMPLES {
frame[i] * self.hann[i]
} else {
0.0
};
self.fft_buf[2 * i] = re;
self.fft_buf[2 * i + 1] = 0.0;
}
// Forward FFT.
fft_complex_forward(&mut self.fft_buf, NS_FFT);
// Compute per-bin power spectrum from complex output.
let mut power = [0.0_f32; NS_BINS];
for k in 0..NS_BINS {
let re = self.fft_buf[2 * k];
let im = self.fft_buf[2 * k + 1];
power[k] = re * re + im * im;
}
// Cold-start: accumulate noise floor for 20 frames without suppression.
if self.frame_count <= 20 {
for k in 0..NS_BINS {
self.noise_psd[k] =
self.noise_psd[k] * NOISE_ALPHA + power[k] * (1.0 - NOISE_ALPHA);
}
return;
}
// Compute Wiener gain and update noise floor.
let mut gain = [0.0_f32; NS_BINS];
for k in 0..NS_BINS {
let noise = self.noise_psd[k] * BIAS;
let snr = ((power[k] - noise) / noise.max(1e-10)).max(0.0);
gain[k] = (snr / (snr + 1.0)).max(GAIN_FLOOR);
// Update noise PSD only in noise-dominated bins.
if snr < VAD_SNR_THRESHOLD {
self.noise_psd[k] =
self.noise_psd[k] * NOISE_ALPHA + power[k] * (1.0 - NOISE_ALPHA);
}
}
// Apply gain to the complex spectrum.
// Bins 0..NS_BINS are the positive-frequency half.
// Mirror to the negative-frequency half (conjugate symmetry).
for k in 0..NS_BINS {
self.fft_buf[2 * k] *= gain[k];
self.fft_buf[2 * k + 1] *= gain[k];
}
// Mirror: bin k maps to bin NS_FFT - k.
for k in 1..NS_BINS - 1 {
let mirror = NS_FFT - k;
self.fft_buf[2 * mirror] = self.fft_buf[2 * k];
self.fft_buf[2 * mirror + 1] = -self.fft_buf[2 * k + 1]; // conjugate
}
// Inverse FFT.
fft_complex_inverse(&mut self.fft_buf, NS_FFT);
// Overlap-add: output = IFFT real part + previous tail.
for i in 0..FRAME_SAMPLES {
frame[i] = self.fft_buf[2 * i] + self.ola_tail[i];
}
// Save tail for next frame.
for i in 0..FRAME_SAMPLES {
self.ola_tail[i] = if i + FRAME_SAMPLES < NS_FFT {
self.fft_buf[2 * (i + FRAME_SAMPLES)]
} else {
0.0
};
}
}
/// Reset all state.
pub fn reset(&mut self) {
self.noise_psd.fill(1e-6);
self.ola_tail.fill(0.0);
self.fft_buf.fill(0.0);
self.frame_count = 0;
}
}
impl Default for NoiseSuppressor {
fn default() -> Self {
Self::new()
}
}
// ── Correct radix-2 DIT complex FFT ──────────────────────────────────────
//
// Buffer layout: interleaved [re0, im0, re1, im1, ..., re_{n-1}, im_{n-1}].
// Length of buf must be 2*n where n is a power of 2.
fn fft_complex_forward(buf: &mut [f32], n: usize) {
debug_assert_eq!(buf.len(), 2 * n);
debug_assert!(n.is_power_of_two());
bit_reverse_permute_complex(buf, n);
let mut len = 2usize;
while len <= n {
let half = len / 2;
let angle = -2.0 * std::f32::consts::PI / len as f32;
let (wre, wim) = (angle.cos(), angle.sin());
let mut start = 0;
while start < n {
let (mut cur_re, mut cur_im) = (1.0_f32, 0.0_f32);
for j in 0..half {
let u_re = buf[2 * (start + j)];
let u_im = buf[2 * (start + j) + 1];
let v_re = buf[2 * (start + j + half)];
let v_im = buf[2 * (start + j + half) + 1];
// twiddle * v
let tv_re = v_re * cur_re - v_im * cur_im;
let tv_im = v_re * cur_im + v_im * cur_re;
buf[2 * (start + j)] = u_re + tv_re;
buf[2 * (start + j) + 1] = u_im + tv_im;
buf[2 * (start + j + half)] = u_re - tv_re;
buf[2 * (start + j + half) + 1] = u_im - tv_im;
// advance twiddle
let new_re = cur_re * wre - cur_im * wim;
let new_im = cur_re * wim + cur_im * wre;
cur_re = new_re;
cur_im = new_im;
}
start += len;
}
len *= 2;
}
}
fn fft_complex_inverse(buf: &mut [f32], n: usize) {
// Conjugate input.
for k in 0..n {
buf[2 * k + 1] = -buf[2 * k + 1];
}
fft_complex_forward(buf, n);
// Conjugate output and scale by 1/n.
let scale = 1.0 / n as f32;
for k in 0..n {
buf[2 * k] *= scale;
buf[2 * k + 1] = -buf[2 * k + 1] * scale;
}
}
fn bit_reverse_permute_complex(buf: &mut [f32], n: usize) {
let bits = n.trailing_zeros() as usize;
for i in 0..n {
let j = reverse_bits(i, bits);
if j > i {
buf.swap(2 * i, 2 * j);
buf.swap(2 * i + 1, 2 * j + 1);
}
}
}
fn reverse_bits(mut x: usize, bits: usize) -> usize {
let mut r = 0usize;
for _ in 0..bits {
r = (r << 1) | (x & 1);
x >>= 1;
}
r
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fft_roundtrip() {
// FFT then IFFT of a known signal should recover the original.
let mut buf = vec![0.0_f32; 2 * 8];
// Input: [1, 2, 3, 4, 0, 0, 0, 0] (real only)
for i in 0..4 {
buf[2 * i] = (i + 1) as f32;
}
let original: Vec<f32> = buf.iter().step_by(2).take(8).copied().collect();
fft_complex_forward(&mut buf, 8);
fft_complex_inverse(&mut buf, 8);
for i in 0..8 {
assert!(
(buf[2 * i] - original[i]).abs() < 1e-4,
"roundtrip failed at {i}: got {} expected {}",
buf[2 * i],
original[i]
);
}
}
#[test]
fn ns_reduces_stationary_noise() {
let mut ns = NoiseSuppressor::new();
let mut rng: u32 = 0xDEAD_BEEF;
let noise_frame = |rng: &mut u32| -> [f32; FRAME_SAMPLES] {
let mut f = [0.0_f32; FRAME_SAMPLES];
for s in f.iter_mut() {
*rng = rng.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
*s = (*rng as i32 as f32) / i32::MAX as f32 * 0.05;
}
f
};
// Warm up noise floor (20 frames cold-start + 10 more to converge).
for _ in 0..30 {
let mut frame = noise_frame(&mut rng);
ns.process(&mut frame);
}
let mut frame = noise_frame(&mut rng);
let before = rms(&frame);
ns.process(&mut frame);
let after = rms(&frame);
assert!(
after < before * 0.8,
"NS did not suppress noise: before={before:.4} after={after:.4}"
);
}
#[test]
fn ns_disabled_is_passthrough() {
let mut ns = NoiseSuppressor::new();
ns.set_enabled(false);
let mut frame = [0.1_f32; FRAME_SAMPLES];
ns.process(&mut frame);
assert!(frame.iter().all(|&s| (s - 0.1).abs() < 1e-6));
}
#[test]
fn reset_clears_state() {
let mut ns = NoiseSuppressor::new();
for _ in 0..30 {
let mut frame = [0.05_f32; FRAME_SAMPLES];
ns.process(&mut frame);
}
ns.reset();
assert_eq!(ns.frame_count, 0);
assert!(ns.ola_tail.iter().all(|&s| s == 0.0));
}
fn rms(frame: &[f32]) -> f32 {
(frame.iter().map(|s| s * s).sum::<f32>() / frame.len() as f32).sqrt()
}
}
+31
View File
@@ -0,0 +1,31 @@
//! Realtime-safe audio processors for platform and software voice paths.
pub mod dsp;
pub mod noop;
pub mod platform;
pub mod sonora;
pub use noop::NoopProcessor;
pub use platform::PlatformVoiceProcessor;
pub use sonora::SonoraProcessor;
/// 10 ms mono f32 processing frame at 48 kHz (480 samples).
pub const FRAME_SAMPLES: usize = 480;
/// Realtime-safe audio processor backend.
///
/// Implementations MUST be `Send` and MUST NOT allocate, block, or
/// perform I/O inside `process_capture` or `process_render`.
pub trait AudioProcessor: Send {
/// Process one 10 ms capture frame in-place.
fn process_capture(&mut self, frame: &mut [f32; FRAME_SAMPLES]);
/// Feed one 10 ms render-reference frame (decoded remote PCM
/// before playout). Required by software AEC backends; no-op
/// for platform and noop backends.
fn process_render(&mut self, frame: &[f32; FRAME_SAMPLES]);
/// Return true if this backend performs acoustic echo cancellation
/// so the engine can enforce INV_009 / INV_010.
fn has_aec(&self) -> bool;
}
@@ -0,0 +1,16 @@
//! Processor backend that intentionally leaves audio unchanged.
use super::{AudioProcessor, FRAME_SAMPLES};
/// No-op audio processor for debug/headset routes.
pub struct NoopProcessor;
impl AudioProcessor for NoopProcessor {
fn process_capture(&mut self, _frame: &mut [f32; FRAME_SAMPLES]) {}
fn process_render(&mut self, _frame: &[f32; FRAME_SAMPLES]) {}
fn has_aec(&self) -> bool {
false
}
}
@@ -0,0 +1,20 @@
//! Platform-owned voice processing backend.
use super::{AudioProcessor, FRAME_SAMPLES};
/// Marker backend for the platform VoiceProcessingIO path.
/// All DSP (AEC/NS/AGC) is handled by the hardware voice processor;
/// Rust-side processing is a no-op. `has_aec` returns true so the
/// engine enforces INV_009/INV_010 and never enables Rust AEC
/// simultaneously.
pub struct PlatformVoiceProcessor;
impl AudioProcessor for PlatformVoiceProcessor {
fn process_capture(&mut self, _frame: &mut [f32; FRAME_SAMPLES]) {}
fn process_render(&mut self, _frame: &[f32; FRAME_SAMPLES]) {}
fn has_aec(&self) -> bool {
true
}
}
@@ -0,0 +1,353 @@
//! Sonora software voice processor — full DSP chain.
//!
//! Composes the four P1 DSP stages in the order mandated by the spec:
//!
//! **HPF → AEC3 → NS → AGC2**
//!
//! Each stage is independently enable/disable-able via
//! [`SonoraConfig`]. The default configuration matches the P1 spec:
//! all stages enabled, AEC3 disabled when no render reference is
//! available (INV_011).
//!
//! ## Stage descriptions
//!
//! | Stage | Module | Description |
//! |-------|--------|-------------|
//! | HPF | `dsp::hpf` | 80 Hz Butterworth biquad, removes DC and rumble |
//! | AEC3 | `dsp::aec3` | Adaptive filter echo canceller with delay estimation |
//! | NS | `dsp::ns` | Wiener filter noise suppressor with min-statistics floor |
//! | AGC2 | `dsp::agc2` | RNN VAD-gated gain controller with look-ahead limiter |
//!
//! ## Realtime safety
//!
//! All state is pre-allocated. `process_capture` and `process_render`
//! never allocate, block, or perform I/O (INV_007).
//!
//! ## INV_009 / INV_010 enforcement
//!
//! `has_aec()` returns `true` when AEC3 is enabled. The engine uses
//! this to enforce the invariant that platform AEC and Rust AEC are
//! never active simultaneously.
use super::dsp::{aec3::Aec3, agc2::Agc2, hpf::HighPassFilter, ns::NoiseSuppressor};
use super::{AudioProcessor, FRAME_SAMPLES};
/// Per-stage enable flags for the Sonora processor.
#[derive(Debug, Clone, PartialEq)]
pub struct SonoraConfig {
/// High-pass filter (80 Hz Butterworth). Default: enabled.
pub hpf: bool,
/// AEC3 adaptive echo canceller. Default: disabled until render
/// reference is confirmed available (INV_011).
pub aec3: bool,
/// Wiener filter noise suppressor. Default: enabled.
pub ns: bool,
/// AGC2 gain controller + limiter. Default: enabled.
pub agc2: bool,
}
impl Default for SonoraConfig {
fn default() -> Self {
Self {
hpf: true,
// AEC3 is disabled by default: it requires a render reference
// (INV_011). The engine enables it only when the render
// reference path is confirmed active.
aec3: false,
ns: true,
agc2: true,
}
}
}
impl SonoraConfig {
/// Configuration for the Sonora experimental mode with AEC3 enabled.
/// Only valid when a render reference is available (INV_011).
pub fn with_aec3() -> Self {
Self {
hpf: true,
aec3: true,
ns: true,
agc2: true,
}
}
/// Minimal configuration: HPF + AGC2 only (no AEC, no NS).
/// Suitable for wired headset routes where AEC is not needed.
pub fn headset() -> Self {
Self {
hpf: true,
aec3: false,
ns: false,
agc2: true,
}
}
}
/// Full Sonora DSP chain: HPF → AEC3 → NS → AGC2.
pub struct SonoraProcessor {
hpf: HighPassFilter,
aec3: Aec3,
ns: NoiseSuppressor,
agc2: Agc2,
config: SonoraConfig,
}
impl SonoraProcessor {
/// Construct with the default configuration (AEC3 disabled).
pub fn new() -> Self {
let config = SonoraConfig::default();
let mut aec3 = Aec3::new();
aec3.set_enabled(config.aec3);
let mut ns = NoiseSuppressor::new();
ns.set_enabled(config.ns);
let mut agc2 = Agc2::new();
agc2.set_enabled(config.agc2);
Self {
hpf: HighPassFilter::new(),
aec3,
ns,
agc2,
config,
}
}
/// Construct with a specific configuration.
pub fn with_config(config: SonoraConfig) -> Self {
let mut aec3 = Aec3::new();
aec3.set_enabled(config.aec3);
let mut ns = NoiseSuppressor::new();
ns.set_enabled(config.ns);
let mut agc2 = Agc2::new();
agc2.set_enabled(config.agc2);
Self {
hpf: HighPassFilter::new(),
aec3,
ns,
agc2,
config,
}
}
/// Apply a new configuration at runtime. Resets stages whose
/// enable state changed to avoid state contamination.
pub fn apply_config(&mut self, new_config: SonoraConfig) {
if new_config.hpf != self.config.hpf {
self.hpf.reset();
}
if new_config.aec3 != self.config.aec3 {
self.aec3.reset();
self.aec3.set_enabled(new_config.aec3);
}
if new_config.ns != self.config.ns {
self.ns.reset();
self.ns.set_enabled(new_config.ns);
}
if new_config.agc2 != self.config.agc2 {
self.agc2.reset();
self.agc2.set_enabled(new_config.agc2);
}
self.config = new_config;
}
/// Current configuration.
pub fn config(&self) -> &SonoraConfig {
&self.config
}
/// Reset all DSP state (call on route change or session restart).
pub fn reset_all(&mut self) {
self.hpf.reset();
self.aec3.reset();
self.ns.reset();
self.agc2.reset();
}
/// Current AEC3 bulk delay estimate in blocks (1 block = 10 ms).
pub fn aec3_bulk_delay_blocks(&self) -> usize {
self.aec3.bulk_delay_blocks()
}
/// True if AEC3 has converged.
pub fn aec3_converged(&self) -> bool {
self.aec3.is_converged()
}
/// Current AGC2 speech probability from the RNN VAD.
pub fn agc2_speech_probability(&self) -> f32 {
self.agc2.speech_probability()
}
/// Current AGC2 gain in dB.
pub fn agc2_gain_db(&self) -> f32 {
self.agc2.gain_db()
}
}
impl Default for SonoraProcessor {
fn default() -> Self {
Self::new()
}
}
impl AudioProcessor for SonoraProcessor {
/// Process one 10 ms capture frame in-place.
///
/// Pipeline: HPF → AEC3 → NS → AGC2.
fn process_capture(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
// Stage 1: High-pass filter (DC removal, rumble suppression).
if self.config.hpf {
self.hpf.process(frame);
}
// Stage 2: AEC3 (echo cancellation).
// AEC3 reads the render reference that was fed via process_render.
// INV_011: only runs when aec3 is enabled (render reference available).
self.aec3.process_capture(frame);
// Stage 3: Noise suppression (Wiener filter).
self.ns.process(frame);
// Stage 4: AGC2 (gain control + limiter).
self.agc2.process(frame);
}
/// Feed one 10 ms render-reference frame (decoded remote PCM
/// before playout). Required by AEC3 (INV_012).
fn process_render(&mut self, frame: &[f32; FRAME_SAMPLES]) {
self.aec3.process_render(frame);
}
/// True when AEC3 is enabled (enforces INV_009 / INV_010).
fn has_aec(&self) -> bool {
self.config.aec3
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_config_has_aec3_disabled() {
let p = SonoraProcessor::new();
assert!(!p.has_aec(), "AEC3 must be disabled by default (INV_010)");
assert!(p.config().hpf);
assert!(p.config().ns);
assert!(p.config().agc2);
}
#[test]
fn with_aec3_config_enables_aec() {
let p = SonoraProcessor::with_config(SonoraConfig::with_aec3());
assert!(p.has_aec());
}
#[test]
fn process_capture_does_not_panic_on_silence() {
let mut p = SonoraProcessor::new();
let mut frame = [0.0_f32; FRAME_SAMPLES];
p.process_capture(&mut frame);
assert!(frame.iter().all(|s| s.is_finite()));
}
#[test]
fn process_capture_does_not_panic_on_loud_signal() {
let mut p = SonoraProcessor::new();
let mut frame = [1.0_f32; FRAME_SAMPLES];
p.process_capture(&mut frame);
assert!(frame.iter().all(|s| s.is_finite()));
}
#[test]
fn hpf_removes_dc() {
let mut p = SonoraProcessor::with_config(SonoraConfig {
hpf: true,
aec3: false,
ns: false,
agc2: false,
});
// Feed 200 frames of DC = 0.5.
let mut frame = [0.5_f32; FRAME_SAMPLES];
for _ in 0..200 {
p.process_capture(&mut frame);
}
// After convergence, DC should be near zero.
let mean: f32 = frame.iter().sum::<f32>() / FRAME_SAMPLES as f32;
assert!(mean.abs() < 0.01, "DC not removed: mean={mean}");
}
#[test]
fn apply_config_resets_changed_stages() {
let mut p = SonoraProcessor::new();
// Run some frames to build up state.
let mut frame = [0.1_f32; FRAME_SAMPLES];
for _ in 0..10 {
p.process_capture(&mut frame);
}
// Enable AEC3 — should reset AEC3 state.
p.apply_config(SonoraConfig::with_aec3());
assert!(p.has_aec());
assert!(!p.aec3_converged()); // reset clears convergence
}
#[test]
fn suppresses_stationary_noise() {
let mut p = SonoraProcessor::with_config(SonoraConfig {
hpf: false,
aec3: false,
ns: true,
agc2: false,
});
let mut rng: u32 = 0xABCD_1234;
let noise_frame = |rng: &mut u32| -> [f32; FRAME_SAMPLES] {
let mut f = [0.0_f32; FRAME_SAMPLES];
for s in f.iter_mut() {
*rng = rng.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
*s = (*rng as i32 as f32) / i32::MAX as f32 * 0.05;
}
f
};
// Warm up noise floor.
for _ in 0..30 {
let mut frame = noise_frame(&mut rng);
p.process_capture(&mut frame);
}
let mut frame = noise_frame(&mut rng);
let before = rms(&frame);
p.process_capture(&mut frame);
let after = rms(&frame);
assert!(
after < before,
"NS did not suppress noise: {before:.4} → {after:.4}"
);
}
#[test]
fn agc_amplifies_quiet_signal() {
let mut p = SonoraProcessor::with_config(SonoraConfig {
hpf: false,
aec3: false,
ns: false,
agc2: true,
});
let mut frame = [0.0_f32; FRAME_SAMPLES];
for (i, sample) in frame.iter_mut().enumerate() {
*sample = (2.0 * std::f32::consts::PI * 300.0 * i as f32 / 48_000.0).sin() * 0.01;
}
let before = rms(&frame);
for _ in 0..50 {
p.process_capture(&mut frame);
}
let after = rms(&frame);
assert!(
after > before,
"AGC did not amplify: {before:.4} → {after:.4}"
);
}
fn rms(frame: &[f32]) -> f32 {
let power = frame.iter().map(|s| s * s).sum::<f32>() / frame.len() as f32;
power.sqrt()
}
}
@@ -774,7 +774,7 @@ mod tests {
}
#[test]
fn build_descriptor_granted_keyboard_reports_L2() {
fn build_descriptor_granted_keyboard_reports_l2() {
let d = MacOSEventTapBackend::build_descriptor(
PermissionState::Granted,
PttInputClass::Keyboard,
@@ -784,7 +784,7 @@ mod tests {
}
#[test]
fn build_descriptor_granted_mouse_reports_L3() {
fn build_descriptor_granted_mouse_reports_l3() {
let d = MacOSEventTapBackend::build_descriptor(
PermissionState::Granted,
PttInputClass::MouseSideButton,
@@ -794,7 +794,7 @@ mod tests {
}
#[test]
fn build_descriptor_granted_none_reports_L2_keyboard() {
fn build_descriptor_granted_none_reports_l2_keyboard() {
let d =
MacOSEventTapBackend::build_descriptor(PermissionState::Granted, PttInputClass::None);
assert_eq!(d.level, PttCapabilityLevel::L2GlobalHoldToTalk);
+246
View File
@@ -0,0 +1,246 @@
//! Route-to-processing policy for P1 iOS.
//!
//! Maps the current [`AudioRoute`] to the recommended
//! [`AudioProcessingConfig`] for that route. The policy enforces
//! INV_009 (never enable platform AEC and Rust AEC simultaneously)
//! and INV_010 (never enable VoiceProcessingIO and Sonora AEC3
//! simultaneously).
//!
//! The returned config is a *recommendation*; the engine may override
//! individual fields (e.g. to keep the user's explicit VAD backend
//! choice) but must not violate the hard invariants.
use crate::audio_processing::{
AudioBackend, AudioProcessingConfig, AudioRoute, EffectOwner, IosVoiceProcessingMode,
VadBackend,
};
/// Compute the recommended [`AudioProcessingConfig`] for a given
/// iOS audio route. The returned config always satisfies the P1
/// hard invariants for iOS.
///
/// * Speaker / Earpiece → platform VPIO (AEC/NS/AGC owned by platform).
/// * Wired headset → noop AEC, conservative NS/AGC optional.
/// * Bluetooth HFP → route-managed (app-side AEC off, NS/AGC conservative).
/// * Bluetooth A2DP → invalid for duplex; transmit blocked at selector level.
/// * Unknown → safe fallback (AEC off until classified).
pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
match route {
AudioRoute::Speaker | AudioRoute::Earpiece => AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::SileroOnnx,
aec: EffectOwner::Platform,
// VPIO owns NS and AGC on the shipping default path (IOSP_002/003).
ns: EffectOwner::Platform,
agc: EffectOwner::Platform,
hpf_enabled: true,
limiter_enabled: true,
..AudioProcessingConfig::default()
},
AudioRoute::WiredHeadset => AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::Noop,
vad_backend: VadBackend::SileroOnnx,
// No AEC needed for wired headset (no acoustic echo path).
aec: EffectOwner::Off,
// Conservative NS/AGC: optional, not forced.
ns: EffectOwner::Conservative,
agc: EffectOwner::Conservative,
hpf_enabled: true,
limiter_enabled: true,
..AudioProcessingConfig::default()
},
AudioRoute::BluetoothHfp => AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::SileroOnnx,
// BT HFP manages its own AEC in the headset firmware.
aec: EffectOwner::Off,
ns: EffectOwner::Conservative,
agc: EffectOwner::Conservative,
hpf_enabled: true,
limiter_enabled: true,
..AudioProcessingConfig::default()
},
AudioRoute::BluetoothA2dp => {
// A2DP is output-only; duplex voice is invalid on this route.
// Return a config that disables all processing and VAD.
// The transmit selector will block transmit via the route check.
AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::Noop,
vad_backend: VadBackend::Disabled,
aec: EffectOwner::Off,
ns: EffectOwner::Off,
agc: EffectOwner::Off,
hpf_enabled: false,
limiter_enabled: false,
..AudioProcessingConfig::default()
}
}
AudioRoute::Unknown => AudioProcessingConfig {
route,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::Noop,
vad_backend: VadBackend::SileroOnnx,
// Safe fallback: AEC off until route is classified.
aec: EffectOwner::Off,
ns: EffectOwner::Off,
agc: EffectOwner::Off,
hpf_enabled: true,
limiter_enabled: true,
..AudioProcessingConfig::default()
},
}
}
/// Apply a route change to an existing config, preserving user-chosen
/// VAD backend, timing, and debug settings while updating the
/// route-dependent policy fields.
pub fn apply_route_change(
existing: &AudioProcessingConfig,
new_route: AudioRoute,
) -> AudioProcessingConfig {
let policy = ios_route_policy(new_route);
AudioProcessingConfig {
// Route-policy fields from the new route.
route: policy.route,
ios_mode: policy.ios_mode,
processing_backend: policy.processing_backend,
aec: policy.aec,
ns: policy.ns,
agc: policy.agc,
hpf_enabled: policy.hpf_enabled,
limiter_enabled: policy.limiter_enabled,
// Preserve user-chosen VAD backend and timing.
vad_backend: existing.vad_backend,
vad_hangover_ms: existing.vad_hangover_ms,
vad_pre_roll_ms: existing.vad_pre_roll_ms,
vad_min_tx_ms: existing.vad_min_tx_ms,
// Preserve debug settings.
debug_wav_dump_enabled: existing.debug_wav_dump_enabled,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn speaker_uses_platform_vpio() {
let cfg = ios_route_policy(AudioRoute::Speaker);
assert_eq!(
cfg.processing_backend,
AudioBackend::PlatformVoiceProcessing
);
assert_eq!(cfg.aec, EffectOwner::Platform);
// VPIO owns NS and AGC on the default path.
assert_eq!(cfg.ns, EffectOwner::Platform);
assert_eq!(cfg.agc, EffectOwner::Platform);
assert_eq!(
cfg.ios_mode,
IosVoiceProcessingMode::PlatformVoiceProcessing
);
}
#[test]
fn earpiece_uses_platform_vpio() {
let cfg = ios_route_policy(AudioRoute::Earpiece);
assert_eq!(
cfg.processing_backend,
AudioBackend::PlatformVoiceProcessing
);
assert_eq!(cfg.aec, EffectOwner::Platform);
}
#[test]
fn wired_headset_disables_aec() {
let cfg = ios_route_policy(AudioRoute::WiredHeadset);
assert_eq!(cfg.aec, EffectOwner::Off);
assert_eq!(cfg.processing_backend, AudioBackend::Noop);
}
#[test]
fn bluetooth_hfp_disables_app_aec() {
let cfg = ios_route_policy(AudioRoute::BluetoothHfp);
assert_eq!(cfg.aec, EffectOwner::Off);
}
#[test]
fn bluetooth_a2dp_disables_all_processing_and_vad() {
let cfg = ios_route_policy(AudioRoute::BluetoothA2dp);
assert_eq!(cfg.aec, EffectOwner::Off);
assert_eq!(cfg.vad_backend, VadBackend::Disabled);
assert_eq!(cfg.processing_backend, AudioBackend::Noop);
}
#[test]
fn unknown_route_safe_fallback_no_aec() {
let cfg = ios_route_policy(AudioRoute::Unknown);
assert_eq!(cfg.aec, EffectOwner::Off);
}
#[test]
fn apply_route_change_preserves_vad_timing_and_debug() {
let existing = AudioProcessingConfig {
vad_backend: VadBackend::WebrtcVad,
vad_hangover_ms: 600,
vad_pre_roll_ms: 200,
vad_min_tx_ms: 300,
debug_wav_dump_enabled: true,
..AudioProcessingConfig::default()
};
let updated = apply_route_change(&existing, AudioRoute::WiredHeadset);
assert_eq!(updated.vad_backend, VadBackend::WebrtcVad);
assert_eq!(updated.vad_hangover_ms, 600);
assert_eq!(updated.vad_pre_roll_ms, 200);
assert_eq!(updated.vad_min_tx_ms, 300);
assert!(updated.debug_wav_dump_enabled);
assert_eq!(updated.route, AudioRoute::WiredHeadset);
assert_eq!(updated.aec, EffectOwner::Off);
}
#[test]
fn no_sonora_aec_in_platform_vpio_policy() {
// INV_009: AEC must never be Sonora in the VPIO path.
for route in [
AudioRoute::Speaker,
AudioRoute::Earpiece,
AudioRoute::WiredHeadset,
AudioRoute::BluetoothHfp,
AudioRoute::BluetoothA2dp,
AudioRoute::Unknown,
] {
let cfg = ios_route_policy(route);
assert_ne!(
cfg.processing_backend,
AudioBackend::Sonora,
"route {:?} must not use Sonora backend in platform policy",
route
);
assert_ne!(
cfg.aec,
EffectOwner::Sonora,
"route {:?}: AEC must not be Sonora in VPIO path (INV_009)",
route
);
assert_ne!(
cfg.ns,
EffectOwner::Sonora,
"route {:?}: NS must not be Sonora in VPIO path (IOSP_003)",
route
);
assert_ne!(
cfg.agc,
EffectOwner::Sonora,
"route {:?}: AGC must not be Sonora in VPIO path (IOSP_003)",
route
);
}
}
}
+4 -14
View File
@@ -5,33 +5,23 @@
//! [`chanora_storage::IdentityFileStore`] under the `transmit_mode`
//! metadata key (default [`TransmitMode::Ptt`]).
//!
//! `VoiceActivity` is reserved per DEC-030 — for v1 the
//! [`crate::transmit_selector::TransmitModeSelector`] treats it
//! exactly like [`TransmitMode::Continuous`] until a real VAD
//! implementation lands.
//! `VoiceActivity` is driven by Rust-owned VAD state in P1.
/// User-visible voice transmit mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[repr(u8)]
pub enum TransmitMode {
/// Push-to-talk: transmit only while the bound key is held
/// (with release-tail per SDD-096).
#[default]
Ptt = 0,
/// Continuous: transmit whenever the user is in a voice
/// channel and not hard-muted.
Continuous = 1,
/// Voice activity detection. Reserved per DEC-030; v1 behaves
/// as [`TransmitMode::Continuous`] until a VAD implementation
/// is allocated.
/// Voice activity detection.
VoiceActivity = 2,
}
impl Default for TransmitMode {
fn default() -> Self {
Self::Ptt
}
}
impl TransmitMode {
/// Encode as the persisted single-byte value.
pub fn as_u8(self) -> u8 {
+22 -6
View File
@@ -11,8 +11,7 @@
//! on PTT mode)
//!
//! Hard-mute is a final clamp; leaving the channel forces the gate
//! to `false`. `VoiceActivity` is treated identically to
//! `Continuous` per DEC-030 until a VAD implementation lands.
//! to `false`. `VoiceActivity` is driven by Rust-owned VAD state.
//!
//! All four inputs are stored as atomics so any thread can update
//! them without taking a lock. After each update we call
@@ -92,6 +91,7 @@ pub struct TransmitModeSelector {
in_channel: AtomicBool,
hard_mute: AtomicBool,
ptt_held: AtomicBool,
voice_activity_open: AtomicBool,
/// SDD-106 §5/§6 / SRS-209: latest resolved microphone
/// permission state. Stored as a `u8` so writes from the
/// JNI thread (Android permission requester → bridge) and
@@ -133,6 +133,7 @@ impl TransmitModeSelector {
in_channel: AtomicBool::new(false),
hard_mute: AtomicBool::new(false),
ptt_held: AtomicBool::new(false),
voice_activity_open: AtomicBool::new(false),
// SDD-106 §5: default to Granted on construction so
// non-Android hosts (which never publish a permission
// event) are not silently clamped. The Android bridge
@@ -226,6 +227,17 @@ impl TransmitModeSelector {
self.ptt_held.load(Ordering::Relaxed)
}
/// Rust-owned VAD gate input for VoiceActivity mode.
pub fn set_voice_activity_open(&self, v: bool) {
self.voice_activity_open.store(v, Ordering::Relaxed);
self.recompute();
}
/// Current Rust-owned VAD gate state.
pub fn voice_activity_open(&self) -> bool {
self.voice_activity_open.load(Ordering::Relaxed)
}
/// Subscribe to `ptt_held` transitions. Used by the
/// missed-key-up watchdog (SAD-079) so it fires on the actual
/// PTT-key-down lifetime, not on the resolved `transmit_active`
@@ -265,8 +277,8 @@ impl TransmitModeSelector {
}
match self.mode() {
TransmitMode::Ptt => self.ptt_held.load(Ordering::Relaxed),
// DEC-030: VoiceActivity behaves as Continuous in v1.
TransmitMode::Continuous | TransmitMode::VoiceActivity => true,
TransmitMode::Continuous => true,
TransmitMode::VoiceActivity => self.voice_activity_open.load(Ordering::Relaxed),
}
}
@@ -323,13 +335,17 @@ mod tests {
assert!(g.load(), "continuous independent of key state");
}
/// SWE4-UV-037: voice-activity mode matches continuous in v1.
/// SWE4-UV-037: voice-activity mode follows VAD state.
#[test]
fn voice_activity_matches_continuous_v1() {
fn voice_activity_requires_vad_open() {
let (g, s) = fresh();
s.set_mode(TransmitMode::VoiceActivity);
s.set_in_channel(true);
assert!(!g.load());
s.set_voice_activity_open(true);
assert!(g.load());
s.set_voice_activity_open(false);
assert!(!g.load());
}
/// SWE4-UV-037 / SWE4-UV-041: hard-mute clamps the transmit
+275
View File
@@ -0,0 +1,275 @@
//! Voice activity detection backends and helpers.
//!
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into
//! this module. The production path prefers a model-backed detector when
//! available, and otherwise uses the realtime-safe fallback below so
//! VoiceActivity mode never collapses back to Continuous transmit.
pub mod resampler;
pub mod silero_onnx;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{OnceLock, RwLock};
use crate::frame::{dbfs, i16_to_f32};
use crate::AudioError;
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
pub use silero_onnx::SileroOnnxVad;
/// Voice activity detector output for one 10 ms frame.
#[derive(Debug, Clone, Copy)]
pub struct VadOutput {
/// Speech confidence in the inclusive range `[0.0, 1.0]`.
pub probability: f32,
/// Immediate detector speech decision before hangover/min-duration state.
pub speech: bool,
}
/// Realtime-safe detector that consumes one 10 ms f32 mono frame.
pub trait VoiceActivityDetector: Send {
/// Process one 10 ms frame and return speech probability/state.
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput;
}
/// Realtime-safe fallback VAD used when a model runtime is unavailable.
///
/// This is not an energy-only transmit gate. It combines RMS level,
/// zero-crossing rate, and peak-to-RMS shape with hysteresis so stable
/// background rumble is less likely to open VoiceActivity than speech.
#[derive(Debug, Clone)]
pub struct WebRtcFallbackVad {
open_dbfs: f32,
close_dbfs: f32,
active: bool,
}
impl Default for WebRtcFallbackVad {
fn default() -> Self {
Self {
open_dbfs: -42.0,
close_dbfs: -50.0,
active: false,
}
}
}
impl WebRtcFallbackVad {
fn zero_crossing_rate(samples: &[f32]) -> f32 {
if samples.len() < 2 {
return 0.0;
}
let crossings = samples
.windows(2)
.filter(|pair| (pair[0] >= 0.0 && pair[1] < 0.0) || (pair[0] < 0.0 && pair[1] >= 0.0))
.count();
crossings as f32 / (samples.len() - 1) as f32
}
fn peak_to_rms(samples: &[f32], rms: f32) -> f32 {
if rms <= 0.000_001 {
return 0.0;
}
let peak = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
peak / rms
}
}
impl VoiceActivityDetector for WebRtcFallbackVad {
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
let level = dbfs(samples);
let threshold = if self.active {
self.close_dbfs
} else {
self.open_dbfs
};
let rms = samples.iter().map(|s| s * s).sum::<f32>() / samples.len().max(1) as f32;
let rms = rms.sqrt();
let zcr = Self::zero_crossing_rate(samples);
let crest = Self::peak_to_rms(samples, rms);
// Level score: steeper curve so silence (-50 dBFS) scores near 0.
// Speech is typically -30 to -10 dBFS; silence is -60 to -45 dBFS.
// Map [-60, -20] → [0, 1] with a midpoint at -40 dBFS.
let level_score = ((level + 60.0) / 40.0).clamp(0.0, 1.0);
let zcr_score = if (0.015..=0.32).contains(&zcr) {
1.0
} else {
0.3 // penalise non-speech ZCR more aggressively
};
let crest_score = if (1.5..=12.0).contains(&crest) {
1.0
} else {
0.3
};
let probability =
(level_score * 0.72 + zcr_score * 0.18 + crest_score * 0.10).clamp(0.0, 1.0);
self.active = level >= threshold && probability >= 0.5;
VadOutput {
probability,
speech: self.active,
}
}
}
/// Wraps any `VoiceActivityDetector` that operates at 16 kHz and
/// downsamples 48 kHz input before forwarding.
pub struct Resampled16kHzVad<D: VoiceActivityDetector> {
inner: D,
downsampler: Downsampler48to16,
}
impl<D: VoiceActivityDetector> Resampled16kHzVad<D> {
/// Wrap a 16 kHz detector so it can consume 48 kHz frames.
pub fn new(inner: D) -> Self {
Self {
inner,
downsampler: Downsampler48to16::default(),
}
}
}
impl<D: VoiceActivityDetector> VoiceActivityDetector for Resampled16kHzVad<D> {
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
debug_assert_eq!(samples.len(), INPUT_FRAME_10MS);
let mut input = [0.0_f32; INPUT_FRAME_10MS];
input.copy_from_slice(samples);
let downsampled = self.downsampler.process_frame_10ms(&input);
self.inner.process_10ms(&downsampled)
}
}
/// Convert one 48 kHz i16 10 ms frame and run a detector over it.
pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16]) -> VadOutput {
let mut frame = [0.0_f32; INPUT_FRAME_10MS];
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
*dst = i16_to_f32(src);
}
detector.process_10ms(&frame)
}
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
}
/// Configure the preferred Silero ONNX model path.
///
/// The path is validated eagerly. A successful call increments the
/// model epoch so running audio backends can reload the model without
/// an app restart.
pub fn set_silero_model_path(path: &str) -> Result<(), AudioError> {
let path = path.trim();
if path.is_empty() {
return Err(AudioError::InvalidAudioProcessingConfig(
"vad model path must not be empty".to_string(),
));
}
if !std::path::Path::new(path).is_file() {
return Err(AudioError::InvalidAudioProcessingConfig(format!(
"vad model path does not exist or is not a file: {path}"
)));
}
let mut guard = silero_model_path_override()
.write()
.map_err(|_| AudioError::Backend("vad model path lock poisoned".to_string()))?;
*guard = Some(path.to_string());
SILERO_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
Ok(())
}
/// Monotonic counter incremented whenever the configured model path changes.
pub fn silero_model_epoch() -> u64 {
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
}
/// Return the expected path of the Silero VAD v6 ONNX model in the
/// iOS app bundle. The model is shipped as a Flutter asset and copied
/// to the app's Documents directory by the Dart-side asset loader.
///
/// Returns an empty string on non-Apple platforms (Silero is not
/// supported there; `SileroOnnxVad::try_new` will return `None`).
pub fn silero_model_bundle_path() -> String {
if let Ok(guard) = silero_model_path_override().read() {
if let Some(path) = guard.as_ref() {
return path.clone();
}
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
// Primary: Documents directory (written by Flutter asset loader).
if let Ok(home) = std::env::var("HOME") {
let docs = format!("{home}/Documents/silero_vad.onnx");
if std::path::Path::new(&docs).exists() {
return docs;
}
// Fallback: app bundle Resources directory.
let bundle = format!("{home}/../Library/silero_vad.onnx");
if std::path::Path::new(&bundle).exists() {
return bundle;
}
}
// Last resort: current working directory (useful in tests).
"silero_vad.onnx".to_string()
}
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
String::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fallback_opens_for_voiced_signal() {
let mut vad = WebRtcFallbackVad::default();
let mut frame = [0_i16; INPUT_FRAME_10MS];
for (idx, sample) in frame.iter_mut().enumerate() {
let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / 48_000.0;
*sample = (phase.sin() * 12_000.0) as i16;
}
let output = process_i16_10ms(&mut vad, &frame);
assert!(output.speech);
assert!(output.probability >= 0.5);
}
#[test]
fn fallback_stays_closed_for_silence() {
let mut vad = WebRtcFallbackVad::default();
let frame = [0_i16; INPUT_FRAME_10MS];
let output = process_i16_10ms(&mut vad, &frame);
assert!(!output.speech);
assert!(output.probability < 0.5);
}
#[test]
fn set_silero_model_path_rejects_missing_file() {
let result = set_silero_model_path("/definitely/not/a/silero_vad.onnx");
assert!(result.is_err());
}
#[test]
fn set_silero_model_path_updates_override_and_epoch() {
let path =
std::env::temp_dir().join(format!("chanora_test_silero_{}.onnx", std::process::id()));
std::fs::write(&path, b"test").unwrap();
let before = silero_model_epoch();
set_silero_model_path(path.to_str().unwrap()).unwrap();
assert!(silero_model_epoch() > before);
assert_eq!(silero_model_bundle_path(), path.to_string_lossy());
let _ = std::fs::remove_file(path);
}
}
+144
View File
@@ -0,0 +1,144 @@
//! Lightweight 48 kHz → 16 kHz downsampler for VAD input.
//!
//! Silero VAD and the real WebRTC VAD both operate at 16 kHz. The
//! VPIO capture stream is pinned at 48 kHz (3× ratio). We use a
//! simple polyphase FIR with a 3:1 decimation factor. The filter
//! coefficients are a 32-tap Kaiser-windowed low-pass at 8 kHz
//! (Nyquist of the 16 kHz output), pre-computed offline and baked
//! in as constants so there is no runtime allocation.
//!
//! Quality is sufficient for VAD (speech/silence discrimination);
//! this is not a high-fidelity resampler.
/// Input sample rate (Hz).
pub const INPUT_HZ: u32 = 48_000;
/// Output sample rate (Hz).
pub const OUTPUT_HZ: u32 = 16_000;
/// Decimation factor (INPUT_HZ / OUTPUT_HZ).
pub const DECIMATION: usize = 3;
/// Samples in one 10 ms frame at 48 kHz.
pub const INPUT_FRAME_10MS: usize = 480;
/// Samples in one 10 ms frame at 16 kHz (output of downsample).
pub const OUTPUT_FRAME_10MS: usize = 160;
/// 32-tap FIR low-pass filter coefficients (Kaiser β=8, fc=8 kHz/48 kHz).
/// Generated with scipy.signal.firwin(32, 8000/48000*2, window=('kaiser', 8)).
/// Symmetric — only 16 unique values; stored in full for clarity.
#[rustfmt::skip]
const FIR_COEFFS: [f32; 32] = [
-0.000_592_3, -0.001_158_5, -0.001_601_5, -0.000_993_5,
0.001_601_5, 0.006_046_8, 0.012_131_5, 0.018_614_0,
0.023_448_0, 0.024_726_0, 0.021_048_0, 0.012_636_0,
0.000_993_5, -0.011_614_0, -0.021_048_0, -0.024_726_0,
-0.024_726_0, -0.021_048_0, -0.011_614_0, 0.000_993_5,
0.012_636_0, 0.021_048_0, 0.024_726_0, 0.023_448_0,
0.018_614_0, 0.012_131_5, 0.006_046_8, 0.001_601_5,
-0.000_993_5, -0.001_601_5, -0.001_158_5, -0.000_592_3,
];
const TAPS: usize = FIR_COEFFS.len();
/// Stateful 48→16 kHz downsampler. Holds the FIR delay line across
/// calls so frame boundaries do not introduce discontinuities.
pub struct Downsampler48to16 {
/// Circular delay line (length = TAPS).
delay: [f32; TAPS],
/// Write head into the delay line.
head: usize,
/// Phase counter: 0..DECIMATION. When phase==0 we emit a sample.
phase: usize,
}
impl Default for Downsampler48to16 {
fn default() -> Self {
Self {
delay: [0.0; TAPS],
head: 0,
phase: 0,
}
}
}
impl Downsampler48to16 {
/// Process `input` (48 kHz f32 mono) and write 16 kHz output
/// into `output`. Returns the number of samples written.
///
/// For a full 10 ms input frame (480 samples) this always
/// produces exactly 160 output samples.
pub fn process(&mut self, input: &[f32], output: &mut [f32]) -> usize {
let mut out_idx = 0;
for &sample in input {
// Push sample into circular delay line.
self.delay[self.head] = sample;
self.head = (self.head + 1) % TAPS;
if self.phase == 0 {
// Compute FIR dot product.
let mut acc = 0.0_f32;
for (k, &coeff) in FIR_COEFFS.iter().enumerate() {
let tap_idx = (self.head + TAPS - 1 - k) % TAPS;
acc += self.delay[tap_idx] * coeff;
}
if out_idx < output.len() {
output[out_idx] = acc;
out_idx += 1;
}
}
self.phase = (self.phase + 1) % DECIMATION;
}
out_idx
}
/// Convenience: downsample a full 10 ms 48 kHz frame into a
/// fixed-size 160-sample 16 kHz buffer.
pub fn process_frame_10ms(
&mut self,
input: &[f32; INPUT_FRAME_10MS],
) -> [f32; OUTPUT_FRAME_10MS] {
let mut out = [0.0_f32; OUTPUT_FRAME_10MS];
let n = self.process(input, &mut out);
debug_assert_eq!(
n, OUTPUT_FRAME_10MS,
"resampler produced {n} samples, expected 160"
);
out
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn frame_produces_160_samples() {
let mut ds = Downsampler48to16::default();
let input = [0.5_f32; INPUT_FRAME_10MS];
let out = ds.process_frame_10ms(&input);
// DC input → DC output (scaled by filter gain ≈ 1/3 due to decimation).
// Just check length and that output is finite and non-zero.
assert_eq!(out.len(), OUTPUT_FRAME_10MS);
assert!(out.iter().all(|s| s.is_finite()));
assert!(out.iter().any(|s| s.abs() > 0.001));
}
#[test]
fn silence_produces_silence() {
let mut ds = Downsampler48to16::default();
let input = [0.0_f32; INPUT_FRAME_10MS];
let out = ds.process_frame_10ms(&input);
assert!(out.iter().all(|s| s.abs() < 1e-9));
}
#[test]
fn consecutive_frames_are_continuous() {
let mut ds = Downsampler48to16::default();
// Two frames of DC — output should be stable (no edge discontinuity).
let input = [0.3_f32; INPUT_FRAME_10MS];
let out1 = ds.process_frame_10ms(&input);
let out2 = ds.process_frame_10ms(&input);
// Last sample of frame 1 and first sample of frame 2 should be close.
let diff = (out1[OUTPUT_FRAME_10MS - 1] - out2[0]).abs();
assert!(diff < 0.05, "discontinuity between frames: {diff}");
}
}
+527
View File
@@ -0,0 +1,527 @@
//! Silero VAD v6 ONNX runtime integration (P1 VAD_002).
//!
//! ## Silero VAD v6 model I/O
//!
//! The v6 model (silero_vad.onnx from the v6.x releases) has a different
//! interface from v4. Key changes:
//!
//! | Tensor | Shape | Dtype | Meaning |
//! |---------|------------------|-------|--------------------------------------|
//! | input | \[1, 576\] | f32 | 64-sample context + 512-sample frame |
//! | state | \[2, 1, 128\] | f32 | LSTM state (carry across frames) |
//! | sr | \[1\] | i64 | Sample rate (16000 or 8000) |
//! | output | \[1, 1\] | f32 | Speech probability |
//! | stateN | \[2, 1, 128\] | f32 | Updated LSTM state |
//!
//! Frame size: **512 samples at 16 kHz = 32 ms**.
//! Context: **64 samples** prepended to each frame (last 64 samples of previous frame).
//! Total input width: 512 + 64 = **576 samples**.
//!
//! ## Threading
//!
//! `SileroOnnxVad` is `Send`. The session is created once and reused —
//! never re-created per callback (INV_007).
//!
//! ## Accumulation
//!
//! The capture pipeline delivers 10 ms frames (480 samples at 48 kHz →
//! 160 samples at 16 kHz). Three 10 ms frames = 30 ms ≈ 32 ms. We
//! accumulate 512 samples (32 ms at 16 kHz) before running inference.
//! The last probability is held between inference calls so the state
//! machine always has a value to work with.
//!
//! ## Fallback
//!
//! `try_new` returns `None` when the model file is missing, the ONNX
//! Runtime is unavailable, or the platform is not iOS/macOS. The caller
//! falls back to `WebRtcFallbackVad`.
use super::{VadOutput, VoiceActivityDetector};
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::sync::Arc;
use std::thread::JoinHandle;
/// 16 kHz frame size for Silero VAD v6 (32 ms).
pub const SILERO_FRAME_16K: usize = 512;
/// Context size prepended to each frame (64 samples at 16 kHz).
pub const SILERO_CONTEXT_16K: usize = 64;
/// Total input width: context + frame.
pub const SILERO_INPUT_WIDTH: usize = SILERO_CONTEXT_16K + SILERO_FRAME_16K;
/// LSTM state size: 2 × 1 × 128 = 256 f32 values.
pub const SILERO_STATE_SIZE: usize = 256;
/// Maximum lag in 10 ms frames before the realtime callback treats
/// the Silero worker as stale and falls back to the local WebRTC
/// detector for that frame.
pub const SILERO_MAX_STALE_FRAMES: u64 = 3;
/// Silero VAD v6 ONNX backend.
///
/// Operates at **16 kHz**, accumulating 32 ms frames (512 samples)
/// before running inference. The caller is responsible for downsampling
/// from 48 kHz before calling `process_10ms`.
pub struct SileroOnnxVad {
/// LSTM state [2, 1, 128] — persisted across frames.
state: Box<[f32; SILERO_STATE_SIZE]>,
/// Context ring: last 64 samples of the previous frame.
context: Box<[f32; SILERO_CONTEXT_16K]>,
/// Accumulation buffer for 16 kHz samples (fills to SILERO_FRAME_16K).
accum: Vec<f32>,
/// Last speech probability output (held between inference calls).
last_probability: f32,
/// Model path stored for diagnostics.
model_path: String,
/// Inner ONNX implementation (platform-specific).
inner: SileroInner,
}
enum SileroInner {
#[cfg(any(target_os = "ios", target_os = "macos"))]
Onnx(OnnxSession),
#[allow(dead_code)]
Stub,
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
struct OnnxSession {
session: ort::session::Session,
}
impl SileroOnnxVad {
/// Attempt to load the Silero v6 ONNX model from `model_path`.
///
/// Returns `None` when the model file is missing, the ONNX Runtime
/// is unavailable, or the platform is not iOS/macOS.
pub fn try_new(model_path: &str) -> Option<Self> {
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
Self::try_new_onnx(model_path)
}
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
let _ = model_path;
None
}
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
fn try_new_onnx(model_path: &str) -> Option<Self> {
use tracing::{error, info};
if !std::path::Path::new(model_path).exists() {
tracing::warn!(
target: "chanora_audio",
path = model_path,
"SileroOnnxVad: model file not found; falling back to WebRtcFallbackVad"
);
return None;
}
#[cfg(target_os = "macos")]
if let Some(path) = bundled_onnxruntime_path() {
let _ = ort::init_from(path.to_string_lossy()).commit();
}
let session_result = std::panic::catch_unwind(|| {
ort::session::Session::builder().and_then(|b| b.commit_from_file(model_path))
});
match session_result {
Err(_) => {
error!(
target: "chanora_audio",
path = model_path,
"SileroOnnxVad: ONNX Runtime panicked during load; falling back to WebRtcFallbackVad"
);
None
}
Ok(Ok(session)) => {
info!(
target: "chanora_audio",
path = model_path,
"SileroOnnxVad v6: model loaded"
);
Some(Self {
state: Box::new([0.0; SILERO_STATE_SIZE]),
context: Box::new([0.0; SILERO_CONTEXT_16K]),
accum: Vec::with_capacity(SILERO_FRAME_16K),
last_probability: 0.0,
model_path: model_path.to_owned(),
inner: SileroInner::Onnx(OnnxSession { session }),
})
}
Ok(Err(e)) => {
error!(
target: "chanora_audio",
path = model_path,
error = %e,
"SileroOnnxVad: failed to load model; falling back to WebRtcFallbackVad"
);
None
}
}
}
/// Reset LSTM state and context (call on voice_leave / session restart).
pub fn reset_state(&mut self) {
self.state.iter_mut().for_each(|v| *v = 0.0);
self.context.iter_mut().for_each(|v| *v = 0.0);
self.accum.clear();
self.last_probability = 0.0;
}
/// Return the model path for diagnostics.
pub fn model_path(&self) -> &str {
&self.model_path
}
fn input_with_context(context: &[f32; SILERO_CONTEXT_16K], audio_frame: &[f32]) -> Vec<f32> {
let mut input = Vec::with_capacity(SILERO_CONTEXT_16K + audio_frame.len());
input.extend_from_slice(context);
input.extend_from_slice(audio_frame);
input
}
fn update_context_from_frame(&mut self, audio_frame: &[f32]) {
let ctx_start = audio_frame.len().saturating_sub(SILERO_CONTEXT_16K);
let new_ctx = &audio_frame[ctx_start..];
let copy_len = new_ctx.len().min(SILERO_CONTEXT_16K);
self.context.fill(0.0);
self.context[SILERO_CONTEXT_16K - copy_len..].copy_from_slice(&new_ctx[..copy_len]);
}
/// Run one upstream-style `calc_level` pass over a 32 ms / 512-sample
/// 16 kHz frame: concatenate prior context, pass `input/state/sr` to
/// ONNX, persist `stateN`, then refresh context from the current frame.
/// Updates `last_probability` and returns the new value.
#[cfg(any(target_os = "ios", target_os = "macos"))]
fn calc_level(&mut self, audio_frame: &[f32]) -> f32 {
use ort::value::Value;
use tracing::error;
let SileroInner::Onnx(ref mut inner) = self.inner else {
return self.last_probability;
};
debug_assert_eq!(audio_frame.len(), SILERO_FRAME_16K);
// Build input: [1, 576] = context (64) + frame (512), matching
// snakers4/silero-vad's Rust `calc_level` example.
let input_vec = Self::input_with_context(self.context.as_ref(), audio_frame);
// Build ndarray tensors.
use ndarray::{Array, IxDyn};
let input_arr = Array::from_shape_vec(IxDyn(&[1, SILERO_INPUT_WIDTH]), input_vec);
let state_arr = Array::from_shape_vec(IxDyn(&[2, 1, 128]), self.state.to_vec());
let sr_arr = Array::from_shape_vec(IxDyn(&[1]), vec![16000_i64]);
let (input_arr, state_arr, sr_arr) = match (input_arr, state_arr, sr_arr) {
(Ok(i), Ok(s), Ok(sr)) => (i, s, sr),
_ => return self.last_probability,
};
let input_val = match Value::from_array(input_arr) {
Ok(v) => v,
Err(e) => {
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: input tensor error");
return self.last_probability;
}
};
let state_val = match Value::from_array(state_arr) {
Ok(v) => v,
Err(e) => {
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: state tensor error");
return self.last_probability;
}
};
let sr_val = match Value::from_array(sr_arr) {
Ok(v) => v,
Err(e) => {
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: sr tensor error");
return self.last_probability;
}
};
let outputs =
match inner
.session
.run([(&input_val).into(), (&state_val).into(), (&sr_val).into()])
{
Ok(o) => o,
Err(e) => {
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: inference failed");
return self.last_probability;
}
};
// Extract probability from "output".
if let Ok((_, prob_data)) = outputs["output"].try_extract_tensor::<f32>() {
if let Some(&p) = prob_data.first() {
self.last_probability = p.clamp(0.0, 1.0);
}
}
// Update state from "stateN".
if let Ok((shape, state_data)) = outputs["stateN"].try_extract_tensor::<f32>() {
let total: usize = shape.iter().map(|&d| d as usize).product();
let copy_len = total.min(SILERO_STATE_SIZE);
self.state[..copy_len].copy_from_slice(&state_data[..copy_len]);
}
drop(outputs);
// Match the upstream example: context becomes the last context_size
// samples from the current frame after the model call succeeds.
self.update_context_from_frame(audio_frame);
self.last_probability
}
}
#[cfg(target_os = "macos")]
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
let exe = std::env::current_exe().ok()?;
let app_dir = exe.parent()?;
let framework = app_dir
.join("Frameworks")
.join("onnxruntime.framework")
.join("onnxruntime");
framework.exists().then_some(framework)
}
impl VoiceActivityDetector for SileroOnnxVad {
/// Accept one 10 ms **16 kHz** f32 mono frame (160 samples).
///
/// Accumulates samples until a full 32 ms frame (512 samples) is
/// ready, then runs inference. Between inference calls the last
/// probability is returned unchanged.
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
debug_assert_eq!(
samples.len(),
super::resampler::OUTPUT_FRAME_10MS,
"SileroOnnxVad expects 160 samples (16 kHz 10 ms), got {}",
samples.len()
);
self.accum.extend_from_slice(samples);
if self.accum.len() >= SILERO_FRAME_16K {
let audio_frame: Vec<f32> = self.accum[..SILERO_FRAME_16K].to_vec();
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
if matches!(self.inner, SileroInner::Onnx(_)) {
self.calc_level(&audio_frame);
} else {
self.update_context_from_frame(&audio_frame);
}
}
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
self.update_context_from_frame(&audio_frame);
}
// Drain the accumulator (keep any overflow for next frame).
let overflow: Vec<f32> = self.accum.drain(SILERO_FRAME_16K..).collect();
self.accum.clear();
self.accum.extend_from_slice(&overflow);
}
VadOutput {
probability: self.last_probability,
speech: self.last_probability >= 0.5,
}
}
}
// SAFETY: ONNX Runtime sessions are thread-safe for inference.
// State arrays are owned by this struct and accessed only from
// the single capture callback thread.
unsafe impl Send for SileroOnnxVad {}
struct SileroFrameMessage {
seq: u64,
frame: [f32; super::resampler::INPUT_FRAME_10MS],
}
/// Background Silero worker. The realtime callback only enqueues
/// 10 ms frames and reads the latest probability atomically.
pub struct SileroOnnxVadWorker {
tx: Option<std::sync::mpsc::SyncSender<SileroFrameMessage>>,
latest_probability: Arc<AtomicU32>,
latest_processed_seq: Arc<AtomicU64>,
alive: Arc<AtomicBool>,
handle: Option<JoinHandle<()>>,
}
impl SileroOnnxVadWorker {
/// Start a background Silero worker if the model loads.
pub fn try_new(model_path: &str) -> Option<Self> {
let vad = SileroOnnxVad::try_new(model_path)?;
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(0));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(8);
let latest_probability_for_thread = latest_probability.clone();
let latest_processed_seq_for_thread = latest_processed_seq.clone();
let alive_for_thread = alive.clone();
let handle = std::thread::Builder::new()
.name("chanora-silero-vad".to_string())
.spawn(move || {
let mut vad = super::Resampled16kHzVad::new(vad);
while alive_for_thread.load(Ordering::Relaxed) {
let message = match rx.recv() {
Ok(message) => message,
Err(_) => break,
};
let output = vad.process_10ms(&message.frame);
latest_probability_for_thread.store(
output.probability.clamp(0.0, 1.0).to_bits(),
Ordering::Relaxed,
);
latest_processed_seq_for_thread.store(message.seq, Ordering::Relaxed);
}
})
.ok()?;
Some(Self {
tx: Some(tx),
latest_probability,
latest_processed_seq,
alive,
handle: Some(handle),
})
}
/// Best-effort enqueue of a 10 ms frame for background inference.
pub fn try_send(&self, seq: u64, frame: &[f32; super::resampler::INPUT_FRAME_10MS]) -> bool {
let Some(tx) = &self.tx else {
return false;
};
tx.try_send(SileroFrameMessage { seq, frame: *frame })
.is_ok()
}
/// Latest probability published by the background worker.
pub fn latest_probability(&self) -> f32 {
f32::from_bits(self.latest_probability.load(Ordering::Relaxed))
}
/// Number of 10 ms frames the worker is behind the capture thread.
pub fn lag_frames(&self, capture_seq: u64) -> u64 {
capture_seq.saturating_sub(self.latest_processed_seq.load(Ordering::Relaxed))
}
/// True when the worker is too far behind to trust its latest
/// probability for the current frame.
pub fn is_stale(&self, capture_seq: u64) -> bool {
self.lag_frames(capture_seq) > SILERO_MAX_STALE_FRAMES
}
}
impl Drop for SileroOnnxVadWorker {
fn drop(&mut self) {
self.alive.store(false, Ordering::Relaxed);
let _ = self.tx.take();
let _ = self.handle.take();
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_stub_vad() -> SileroOnnxVad {
SileroOnnxVad {
state: Box::new([0.0; SILERO_STATE_SIZE]),
context: Box::new([0.0; SILERO_CONTEXT_16K]),
accum: Vec::new(),
last_probability: 0.0,
model_path: String::new(),
inner: SileroInner::Stub,
}
}
#[test]
fn try_new_returns_none_without_model_file() {
let result = SileroOnnxVad::try_new("/nonexistent/silero_vad.onnx");
assert!(result.is_none());
}
#[test]
fn stub_accumulates_and_holds_zero_probability() {
let mut vad = make_stub_vad();
let frame = vec![0.0_f32; super::super::resampler::OUTPUT_FRAME_10MS];
// Feed 3 frames (30 ms < 32 ms) — no inference yet.
for _ in 0..3 {
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame);
assert_eq!(out.probability, 0.0);
}
// Feed 1 more frame (40 ms > 32 ms) — accumulator drains.
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame);
assert_eq!(out.probability, 0.0); // stub stays at 0
}
#[test]
fn reset_state_clears_all() {
let mut vad = make_stub_vad();
vad.state[0] = 1.0;
vad.context[0] = 1.0;
vad.last_probability = 0.9;
vad.accum.push(0.5);
vad.reset_state();
assert_eq!(vad.state[0], 0.0);
assert_eq!(vad.context[0], 0.0);
assert_eq!(vad.last_probability, 0.0);
assert!(vad.accum.is_empty());
}
#[test]
fn accumulates_correct_number_of_samples() {
let mut vad = make_stub_vad();
let frame = vec![0.1_f32; super::super::resampler::OUTPUT_FRAME_10MS]; // 160 samples
// 3 × 160 = 480 < 512 — not yet full.
for _ in 0..3 {
VoiceActivityDetector::process_10ms(&mut vad, &frame);
}
assert_eq!(vad.accum.len(), 480);
// 4th frame: 640 > 512 — inference fires, 128 samples remain.
VoiceActivityDetector::process_10ms(&mut vad, &frame);
assert_eq!(vad.accum.len(), 128);
}
#[test]
fn input_concatenates_context_before_frame_like_upstream_example() {
let mut context = [0.0_f32; SILERO_CONTEXT_16K];
context[0] = -1.0;
context[SILERO_CONTEXT_16K - 1] = 1.0;
let frame = vec![0.25_f32; SILERO_FRAME_16K];
let input = SileroOnnxVad::input_with_context(&context, &frame);
assert_eq!(input.len(), SILERO_INPUT_WIDTH);
assert_eq!(input[0], -1.0);
assert_eq!(input[SILERO_CONTEXT_16K - 1], 1.0);
assert_eq!(input[SILERO_CONTEXT_16K], 0.25);
assert_eq!(input[SILERO_INPUT_WIDTH - 1], 0.25);
}
#[test]
fn context_tracks_last_64_samples_of_completed_frame() {
let mut vad = make_stub_vad();
let frame = vec![0.0_f32; super::super::resampler::OUTPUT_FRAME_10MS];
for idx in 0..4 {
let mut chunk = frame.clone();
let chunk_len = chunk.len();
for (sample_idx, sample) in chunk.iter_mut().enumerate() {
*sample = (idx * chunk_len + sample_idx) as f32;
}
VoiceActivityDetector::process_10ms(&mut vad, &chunk);
}
let completed_frame: Vec<f32> = (0..SILERO_FRAME_16K).map(|v| v as f32).collect();
assert_eq!(
vad.context.as_ref(),
&completed_frame[SILERO_FRAME_16K - SILERO_CONTEXT_16K..]
);
}
}
+243
View File
@@ -0,0 +1,243 @@
//! Voice-activity state machine for P1 transmit gating.
//!
//! Implements the full P1 VAD gate policy:
//! * backend/model speech decisions — no custom probability thresholds.
//! * `open_after_ms` — speech must be detected for this long before
//! the gate opens (prevents false opens on transients). Default 40 ms.
//! * `hangover_ms` — gate stays open for this long after speech drops
//! out of the backend decision (prevents choppy transmit close). Default 500 ms.
//! * `min_tx_ms` — minimum transmit duration after gate opens. Default 200 ms.
//!
//! Pre-roll (first-syllable preservation) is handled in the capture
//! pipeline, not here. The state machine only decides whether the gate
//! is open or closed.
/// Shared VAD timing constants and gate state machine.
///
/// Exposing these values here keeps the audio config and platform
/// capture paths aligned without repeating the same magic numbers in
/// multiple modules.
/// Default confirmation window before the gate opens, in milliseconds.
pub const VAD_OPEN_AFTER_MS: u32 = 40;
/// Default hangover duration in milliseconds.
pub const VAD_HANGOVER_MS: u32 = 500;
/// Default minimum transmit duration in milliseconds.
pub const VAD_MIN_TX_MS: u32 = 200;
/// Hangover/open-after/minimum-transmit state machine.
#[derive(Debug, Clone)]
pub struct VoiceActivityStateMachine {
/// Frames of continuous speech required before gate opens.
open_after_frames: u32,
hangover_frames: u32,
min_tx_frames: u32,
active: bool,
hangover_remaining: u32,
min_tx_remaining: u32,
/// Frames of continuous speech seen since last open attempt.
open_confirm_frames: u32,
/// Frames spent open without a strong speech score. This keeps
/// stale or borderline VAD output from holding the mic open forever.
weak_hold_frames: u32,
}
impl VoiceActivityStateMachine {
/// Create a state machine. Frame duration is 10 ms.
pub fn new(open_after_ms: u32, hangover_ms: u32, min_tx_ms: u32) -> Self {
Self {
open_after_frames: open_after_ms / 10,
hangover_frames: hangover_ms / 10,
min_tx_frames: min_tx_ms / 10,
active: false,
hangover_remaining: 0,
min_tx_remaining: 0,
open_confirm_frames: 0,
weak_hold_frames: 0,
}
}
/// Update timers without forcing a close. Used by
/// live settings changes while audio is already running.
pub fn configure(&mut self, open_after_ms: u32, hangover_ms: u32, min_tx_ms: u32) {
self.open_after_frames = open_after_ms / 10;
self.hangover_frames = hangover_ms / 10;
self.min_tx_frames = min_tx_ms / 10;
self.hangover_remaining = self.hangover_remaining.min(self.hangover_frames);
self.min_tx_remaining = self.min_tx_remaining.min(self.min_tx_frames);
}
/// Advance by one 10 ms backend speech decision and return whether
/// transmit should be open for VoiceActivity mode.
pub fn update(&mut self, speech: bool) -> bool {
if self.active {
if self.min_tx_remaining > 0 {
self.min_tx_remaining -= 1;
}
if speech {
self.weak_hold_frames = 0;
} else {
self.weak_hold_frames = self.weak_hold_frames.saturating_add(1);
}
if self.weak_hold_frames >= self.weak_hold_limit_frames() && self.min_tx_remaining == 0
{
self.close();
return false;
}
if speech {
// Speech still present — reset hangover.
self.hangover_remaining = self.hangover_frames;
} else if self.hangover_remaining > 0 {
self.hangover_remaining -= 1;
} else if self.min_tx_remaining == 0 {
// Hangover expired and min-tx elapsed — close gate.
self.close();
}
} else {
// Gate is closed. Accumulate confirmation frames.
if speech {
self.open_confirm_frames += 1;
if self.open_confirm_frames >= self.open_after_frames.max(1) {
// Confirmed speech — open gate.
self.active = true;
self.hangover_remaining = self.hangover_frames;
self.min_tx_remaining = self.min_tx_frames;
self.open_confirm_frames = 0;
self.weak_hold_frames = 0;
}
} else {
// Speech is no longer detected — reset confirmation.
self.open_confirm_frames = 0;
}
}
self.active
}
/// Current active state.
pub fn active(&self) -> bool {
self.active
}
fn weak_hold_limit_frames(&self) -> u32 {
(self.hangover_frames + self.min_tx_frames + self.open_after_frames).clamp(30, 100)
}
fn close(&mut self) {
self.active = false;
self.hangover_remaining = 0;
self.min_tx_remaining = 0;
self.open_confirm_frames = 0;
self.weak_hold_frames = 0;
}
/// Reset all state (call on session restart / voice_leave).
pub fn reset(&mut self) {
self.close();
}
}
impl Default for VoiceActivityStateMachine {
fn default() -> Self {
Self::new(VAD_OPEN_AFTER_MS, VAD_HANGOVER_MS, VAD_MIN_TX_MS)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hangover_keeps_gate_open_after_close() {
// open_after_ms=0 so gate opens immediately on first frame.
let mut sm = VoiceActivityStateMachine::new(0, 30, 0);
assert!(sm.update(true));
assert!(sm.update(false));
assert!(sm.update(false));
assert!(sm.update(false));
assert!(!sm.update(false));
}
#[test]
fn open_after_requires_confirmation_frames() {
// open_after_ms=20 → 2 frames required.
let mut sm = VoiceActivityStateMachine::new(20, 0, 0);
// First frame: not yet open.
assert!(!sm.update(true));
// Second frame: now open.
assert!(sm.update(true));
}
#[test]
fn open_after_resets_on_silence() {
// open_after_ms=20 → 2 frames required.
let mut sm = VoiceActivityStateMachine::new(20, 0, 0);
assert!(!sm.update(true)); // 1 frame
assert!(!sm.update(false)); // silence resets counter
assert!(!sm.update(true)); // 1 frame again
assert!(sm.update(true)); // 2nd frame → open
}
#[test]
fn min_tx_keeps_gate_open_briefly() {
// open_after_ms=0, hangover=0, min_tx=20ms (2 frames).
// After opening: min_tx_remaining decrements each frame.
// Gate closes on the frame where it reaches 0.
let mut sm = VoiceActivityStateMachine::new(0, 0, 20);
assert!(sm.update(true)); // opens; min_tx_remaining=2
assert!(sm.update(false)); // min_tx_remaining=1; still open
assert!(!sm.update(false)); // min_tx_remaining=0; gate closes
}
#[test]
fn reset_clears_all_state() {
let mut sm = VoiceActivityStateMachine::new(0, 100, 0);
assert!(sm.update(true)); // open
sm.reset();
assert!(!sm.active());
// After reset, gate should not be open even with hangover pending.
assert!(!sm.update(false));
}
#[test]
fn default_uses_p1_spec_values() {
let sm = VoiceActivityStateMachine::default();
assert_eq!(sm.open_after_frames, VAD_OPEN_AFTER_MS / 10);
assert_eq!(sm.hangover_frames, VAD_HANGOVER_MS / 10);
assert_eq!(sm.min_tx_frames, VAD_MIN_TX_MS / 10);
}
#[test]
fn live_config_update_shortens_existing_hangover() {
let mut sm = VoiceActivityStateMachine::new(0, 1000, 0);
assert!(sm.update(true));
assert!(sm.update(false));
sm.configure(0, 100, 0);
for _ in 0..10 {
assert!(sm.update(false));
}
assert!(!sm.update(false));
}
#[test]
fn stale_closed_decisions_cannot_hold_gate_forever() {
let mut sm = VoiceActivityStateMachine::new(0, 500, 0);
assert!(sm.update(true));
for _ in 0..49 {
assert!(sm.update(false));
}
assert!(!sm.update(false));
}
#[test]
fn speech_decision_resets_weak_hold_limit() {
let mut sm = VoiceActivityStateMachine::new(0, 500, 0);
assert!(sm.update(true));
for _ in 0..40 {
assert!(sm.update(false));
}
assert!(sm.update(true));
for _ in 0..40 {
assert!(sm.update(false));
}
assert!(sm.active());
}
}
+103
View File
@@ -0,0 +1,103 @@
/// Diagnostics returned by render downmix helpers.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) struct RenderDownmixStats {
/// Peak absolute sample magnitude after i16 conversion.
pub peak_i16: i16,
/// Samples clipped while applying output gain.
pub clipped_samples: u64,
}
/// Downmix interleaved stereo f32 samples into mono i16 samples.
///
/// The helper is allocation-free and safe for realtime render callbacks.
/// If the stereo source is shorter than expected, the remainder of `out`
/// is filled with silence.
pub(crate) fn downmix_stereo_f32_to_mono_i16(
stereo: &[f32],
out: &mut [i16],
gain: f32,
muted: bool,
) -> RenderDownmixStats {
if muted {
out.fill(0);
return RenderDownmixStats::default();
}
let available_frames = stereo.len() / 2;
if available_frames < out.len() {
out.fill(0);
}
let mut peak = 0_u16;
let mut clipped_samples = 0_u64;
for (dst, lr) in out.iter_mut().zip(stereo.chunks_exact(2)) {
let mono = (lr[0] + lr[1]) * 0.5 * gain;
let clamped = mono.clamp(-1.0, 1.0);
if (mono - clamped).abs() > f32::EPSILON {
clipped_samples = clipped_samples.saturating_add(1);
}
let sample = (clamped * i16::MAX as f32) as i16;
*dst = sample;
peak = peak.max(sample.unsigned_abs());
}
RenderDownmixStats {
peak_i16: peak.min(i16::MAX as u16) as i16,
clipped_samples,
}
}
/// Downmix interleaved stereo f32 samples into mono f32 samples.
///
/// Used for software-AEC render references and debug WAV taps.
#[cfg(any(target_os = "ios", test))]
pub(crate) fn downmix_stereo_f32_to_mono_f32(stereo: &[f32], out: &mut [f32]) {
let available_frames = stereo.len() / 2;
if available_frames < out.len() {
out.fill(0.0);
}
for (dst, lr) in out.iter_mut().zip(stereo.chunks_exact(2)) {
*dst = (lr[0] + lr[1]) * 0.5;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn downmix_i16_applies_gain_and_reports_clipping() {
let stereo = [1.0_f32, 1.0, 0.25, -0.25, -2.0, -2.0];
let mut out = [0_i16; 3];
let stats = downmix_stereo_f32_to_mono_i16(&stereo, &mut out, 2.0, false);
assert_eq!(out[0], i16::MAX);
assert_eq!(out[1], 0);
assert_eq!(out[2], -i16::MAX);
assert_eq!(stats.peak_i16, i16::MAX);
assert_eq!(stats.clipped_samples, 2);
}
#[test]
fn downmix_i16_mutes_output() {
let stereo = [1.0_f32, 1.0, -1.0, -1.0];
let mut out = [123_i16; 2];
let stats = downmix_stereo_f32_to_mono_i16(&stereo, &mut out, 1.0, true);
assert_eq!(out, [0, 0]);
assert_eq!(stats, RenderDownmixStats::default());
}
#[test]
fn downmix_f32_fills_missing_tail_with_silence() {
let stereo = [1.0_f32, -1.0];
let mut out = [9.0_f32; 2];
downmix_stereo_f32_to_mono_f32(&stereo, &mut out);
assert_eq!(out, [0.0, 0.0]);
}
}