feat(voice): add iOS VAD runtime support
This commit is contained in:
@@ -42,7 +42,6 @@ use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use audiopus::coder::Encoder as OpusEncoder;
|
||||
use audiopus::{Application as OpusApp, Channels as OpusChannels, SampleRate as OpusSampleRate};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::mobile_voice_backend::{
|
||||
@@ -53,7 +52,7 @@ use crate::mobile_voice_backend::{
|
||||
BackendEventTx, EffectEngagement, EffectEngine, InputPresetChoice, MobileVoiceAudioBackend,
|
||||
SharingModeChoice,
|
||||
};
|
||||
use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
|
||||
use chanora_protocol::OutPacket;
|
||||
use tsclientlib::audio::AudioHandler;
|
||||
|
||||
use crate::{engine::SessionAudioId, AudioError};
|
||||
@@ -71,19 +70,11 @@ use oboe::{
|
||||
// `mobile_voice_backend` so the trait can expose `take_event_rx`
|
||||
// (SDD-111 item 1) cross-platform.
|
||||
|
||||
/// 20 ms at 48 kHz mono — one Opus frame's worth of samples.
|
||||
/// Matches the iOS and desktop constants; duplicated here so this
|
||||
/// module is fully self-contained and cfg-gate-clean.
|
||||
const FRAME_SAMPLES: usize = 960;
|
||||
|
||||
/// Maximum size of an encoded Opus frame in bytes (RFC 6716 §3.2.1).
|
||||
const MAX_OPUS_FRAME: usize = 1275;
|
||||
|
||||
// --- Capture state for Oboe input callback (SDD-111 / SDD-120) ----
|
||||
//
|
||||
// Mirrors the iOS `IosCaptureState` and the cpal-side `CaptureState`.
|
||||
// Oboe delivers 48 kHz mono i16 PCM; we apply mic gain, accumulate to
|
||||
// FRAME_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
|
||||
// FRAME_20MS_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
|
||||
// and try-send the resulting packet on `voice_out_tx`.
|
||||
|
||||
struct AndroidCaptureState {
|
||||
@@ -91,7 +82,7 @@ struct AndroidCaptureState {
|
||||
/// Accumulator for 48 kHz mono PCM. 2x capacity to absorb
|
||||
/// cpal-style buffer-size jitter without reallocating.
|
||||
pcm_accum: Vec<i16>,
|
||||
opus_out: [u8; MAX_OPUS_FRAME],
|
||||
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
@@ -105,33 +96,11 @@ impl AndroidCaptureState {
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
) -> Result<Self, AudioError> {
|
||||
let mut encoder =
|
||||
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
|
||||
.map_err(|e| AudioError::Opus(format!("encoder new (android): {e}")))?;
|
||||
if let Err(e) = encoder.set_bitrate(audiopus::Bitrate::BitsPerSecond(32_000)) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus(android): set_bitrate(32000) failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_complexity(10) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus(android): set_complexity(10) failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_inband_fec(true) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus(android): set_inband_fec(true) failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_packet_loss_perc(5) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus(android): set_packet_loss_perc(5) failed");
|
||||
}
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
bitrate_bps = 32_000,
|
||||
complexity = 10,
|
||||
inband_fec = true,
|
||||
packet_loss_perc = 5,
|
||||
"android Oboe opus encoder tuned for VoIP"
|
||||
);
|
||||
let encoder = crate::opus_voice::new_voip_encoder("android")?;
|
||||
Ok(Self {
|
||||
encoder,
|
||||
pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2),
|
||||
opus_out: [0u8; MAX_OPUS_FRAME],
|
||||
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
|
||||
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx,
|
||||
transmit_active,
|
||||
frames_sent,
|
||||
@@ -139,7 +108,7 @@ impl AndroidCaptureState {
|
||||
})
|
||||
}
|
||||
|
||||
/// Consume i16 mono frames from Oboe, accumulate to FRAME_SAMPLES,
|
||||
/// Consume i16 mono frames from Oboe, accumulate to FRAME_20MS_SAMPLES,
|
||||
/// encode + send when PTT is held. Oboe delivers at the device's
|
||||
/// native sample rate (always 48 kHz for modern Android per SRS-210),
|
||||
/// so no resampling is needed.
|
||||
@@ -159,28 +128,30 @@ impl AndroidCaptureState {
|
||||
}));
|
||||
}
|
||||
// Drain complete 20 ms frames.
|
||||
while self.pcm_accum.len() >= FRAME_SAMPLES {
|
||||
let mut frame = [0i16; FRAME_SAMPLES];
|
||||
frame.copy_from_slice(&self.pcm_accum[..FRAME_SAMPLES]);
|
||||
self.pcm_accum.drain(..FRAME_SAMPLES);
|
||||
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
|
||||
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
|
||||
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
|
||||
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
|
||||
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
|
||||
Ok(len) => {
|
||||
let packet = OutAudio::new(&AudioData::C2S {
|
||||
id: 0,
|
||||
codec: CodecType::OpusVoice,
|
||||
data: &self.opus_out[..len],
|
||||
});
|
||||
match self.voice_out_tx.try_send(packet) {
|
||||
Ok(()) => {
|
||||
self.frames_sent.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Full(_)) => {
|
||||
warn!(target: "chanora_audio", "android Oboe: voice_out queue full; dropping frame");
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => {
|
||||
debug!(target: "chanora_audio", "android Oboe: voice_out closed; capture pipeline stopping");
|
||||
}
|
||||
}
|
||||
crate::opus_voice::send_voip_frame(
|
||||
&self.voice_out_tx,
|
||||
&self.frames_sent,
|
||||
&self.opus_out,
|
||||
len,
|
||||
|| {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android Oboe: voice_out queue full; dropping frame"
|
||||
);
|
||||
},
|
||||
|| {
|
||||
debug!(
|
||||
target: "chanora_audio",
|
||||
"android Oboe: voice_out closed; capture pipeline stopping"
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(target: "chanora_audio", error = %e, "android Oboe opus encode failed");
|
||||
@@ -266,22 +237,12 @@ impl AudioOutputCallback for OutputCallback {
|
||||
}
|
||||
let gain = f32::from_bits(self.output_gain.load(Ordering::Relaxed));
|
||||
let muted = self.output_muted.load(Ordering::Relaxed);
|
||||
let mut peak: i16 = 0;
|
||||
for (i, dst) in frames.iter_mut().enumerate() {
|
||||
if muted {
|
||||
*dst = 0;
|
||||
continue;
|
||||
}
|
||||
let l = scratch[i * 2];
|
||||
let r = scratch[i * 2 + 1];
|
||||
let mono = (l + r) * 0.5 * gain;
|
||||
let clamped = mono.clamp(-1.0, 1.0);
|
||||
let sample = (clamped * i16::MAX as f32) as i16;
|
||||
*dst = sample;
|
||||
if sample.unsigned_abs() > peak.unsigned_abs() {
|
||||
peak = sample;
|
||||
}
|
||||
}
|
||||
let _ = crate::voice_render::downmix_stereo_f32_to_mono_i16(
|
||||
&scratch[..needed],
|
||||
frames,
|
||||
gain,
|
||||
muted,
|
||||
);
|
||||
}));
|
||||
DataCallbackResult::Continue
|
||||
}
|
||||
|
||||
@@ -0,0 +1,456 @@
|
||||
//! P1 audio-processing configuration and statistics.
|
||||
//!
|
||||
//! iOS P1 ships the platform VoiceProcessingIO path by default. Rust
|
||||
//! software AEC/NS/AGC backends are represented in the schema so the
|
||||
//! bridge can reject unsafe combinations instead of silently enabling
|
||||
//! double processing.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
|
||||
|
||||
use crate::AudioError;
|
||||
|
||||
/// Physical/logical audio route class used for route-aware policy.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AudioRoute {
|
||||
/// Built-in speakerphone path.
|
||||
Speaker,
|
||||
/// Built-in receiver/earpiece path.
|
||||
Earpiece,
|
||||
/// Wired headset or USB headset.
|
||||
WiredHeadset,
|
||||
/// Bluetooth Hands-Free Profile duplex route.
|
||||
BluetoothHfp,
|
||||
/// Bluetooth A2DP output-only route.
|
||||
BluetoothA2dp,
|
||||
/// Route could not be classified yet.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl AudioRoute {
|
||||
/// Stable bridge/debug string.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Speaker => "speaker",
|
||||
Self::Earpiece => "earpiece",
|
||||
Self::WiredHeadset => "wired_headset",
|
||||
Self::BluetoothHfp => "bluetooth_hfp",
|
||||
Self::BluetoothA2dp => "bluetooth_a2dp",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse from the Swift-side route class string (case-insensitive).
|
||||
/// Unrecognised strings map to `Unknown`.
|
||||
pub fn from_route_class(s: &str) -> Self {
|
||||
match s {
|
||||
"Speaker" | "speaker" => Self::Speaker,
|
||||
"Earpiece" | "earpiece" => Self::Earpiece,
|
||||
"WiredHeadset" | "wired_headset" => Self::WiredHeadset,
|
||||
"BluetoothHfp" | "bluetooth_hfp" => Self::BluetoothHfp,
|
||||
"BluetoothA2dp" | "bluetooth_a2dp" => Self::BluetoothA2dp,
|
||||
_ => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// iOS voice-processing mode.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum IosVoiceProcessingMode {
|
||||
/// Shipping default: Apple VoiceProcessingIO owns AEC/NS/AGC.
|
||||
PlatformVoiceProcessing,
|
||||
/// Experimental Sonora capture-processing path.
|
||||
SonoraExperimental,
|
||||
}
|
||||
|
||||
/// Processing backend selected by policy/config.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AudioBackend {
|
||||
/// Platform voice processing, VPIO on iOS.
|
||||
PlatformVoiceProcessing,
|
||||
/// Rust-native Sonora backend.
|
||||
Sonora,
|
||||
/// Future WebRTC APM backend.
|
||||
WebrtcApm,
|
||||
/// No processing.
|
||||
Noop,
|
||||
}
|
||||
|
||||
impl AudioBackend {
|
||||
/// Stable bridge/debug string.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::PlatformVoiceProcessing => "platform_voice_processing",
|
||||
Self::Sonora => "sonora",
|
||||
Self::WebrtcApm => "webrtc_apm",
|
||||
Self::Noop => "noop",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// VAD backend selected by policy/config.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum VadBackend {
|
||||
/// Silero ONNX VAD. P1 schema default when model/runtime exist.
|
||||
SileroOnnx,
|
||||
/// TEN VAD backend. Native TEN runtime is optional; unavailable
|
||||
/// builds fall back to the realtime-safe WebRTC detector.
|
||||
TenVad,
|
||||
/// WebRTC-style fallback VAD.
|
||||
WebrtcVad,
|
||||
/// Debug-only energy VAD.
|
||||
EnergyDebug,
|
||||
/// VAD disabled.
|
||||
Disabled,
|
||||
}
|
||||
|
||||
impl VadBackend {
|
||||
/// Stable bridge/debug string.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::SileroOnnx => "silero_vad_onnx",
|
||||
Self::TenVad => "ten_vad",
|
||||
Self::WebrtcVad => "webrtc_vad",
|
||||
Self::EnergyDebug => "energy_debug",
|
||||
Self::Disabled => "disabled",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Effect owner for AEC/NS/AGC policy fields.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EffectOwner {
|
||||
/// Owned by platform voice processing.
|
||||
Platform,
|
||||
/// Owned by Sonora.
|
||||
Sonora,
|
||||
/// Owned by WebRTC APM.
|
||||
WebrtcApm,
|
||||
/// Conservative route-managed setting.
|
||||
Conservative,
|
||||
/// Disabled.
|
||||
Off,
|
||||
}
|
||||
|
||||
/// Voice-processing configuration owned by the Rust audio engine.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct AudioProcessingConfig {
|
||||
/// Current route class.
|
||||
pub route: AudioRoute,
|
||||
/// iOS processing mode.
|
||||
pub ios_mode: IosVoiceProcessingMode,
|
||||
/// Processing backend.
|
||||
pub processing_backend: AudioBackend,
|
||||
/// VAD backend.
|
||||
pub vad_backend: VadBackend,
|
||||
/// AEC owner.
|
||||
pub aec: EffectOwner,
|
||||
/// Noise suppression owner.
|
||||
pub ns: EffectOwner,
|
||||
/// AGC owner.
|
||||
pub agc: EffectOwner,
|
||||
/// High-pass filter enabled.
|
||||
pub hpf_enabled: bool,
|
||||
/// Limiter enabled.
|
||||
pub limiter_enabled: bool,
|
||||
/// Hangover after speech closes.
|
||||
pub vad_hangover_ms: u32,
|
||||
/// Pre-roll before open.
|
||||
pub vad_pre_roll_ms: u32,
|
||||
/// Minimum transmit duration after open.
|
||||
pub vad_min_tx_ms: u32,
|
||||
/// Debug WAV dumps enabled.
|
||||
pub debug_wav_dump_enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for AudioProcessingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
route: AudioRoute::Speaker,
|
||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
||||
vad_backend: VadBackend::SileroOnnx,
|
||||
aec: EffectOwner::Platform,
|
||||
// iOS VPIO owns NS/AGC on the default shipping path. Rust/Sonora
|
||||
// effects are opt-in through the experimental raw route only.
|
||||
ns: EffectOwner::Platform,
|
||||
agc: EffectOwner::Platform,
|
||||
hpf_enabled: true,
|
||||
limiter_enabled: true,
|
||||
vad_hangover_ms: crate::voice_activity::VAD_HANGOVER_MS,
|
||||
vad_pre_roll_ms: 160,
|
||||
vad_min_tx_ms: crate::voice_activity::VAD_MIN_TX_MS,
|
||||
debug_wav_dump_enabled: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AudioProcessingConfig {
|
||||
/// Validate P1 iOS invariants before applying a config.
|
||||
pub fn validate_for_ios(&self) -> Result<(), AudioError> {
|
||||
if self.route == AudioRoute::BluetoothA2dp {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"bluetooth_a2dp is output-only and cannot transmit duplex voice".to_string(),
|
||||
));
|
||||
}
|
||||
if self.ios_mode == IosVoiceProcessingMode::PlatformVoiceProcessing
|
||||
&& (self.processing_backend == AudioBackend::Sonora
|
||||
|| self.aec == EffectOwner::Sonora
|
||||
|| self.ns == EffectOwner::Sonora
|
||||
|| self.agc == EffectOwner::Sonora)
|
||||
{
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"Sonora cannot be enabled with iOS VoiceProcessingIO".to_string(),
|
||||
));
|
||||
}
|
||||
if self.ios_mode == IosVoiceProcessingMode::SonoraExperimental {
|
||||
if self.processing_backend != AudioBackend::Sonora {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"ios Sonora experimental mode requires the Sonora processing backend"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Demote a failed VAD backend to the WebRTC fallback.
|
||||
///
|
||||
/// Returns `true` when the config changed.
|
||||
pub fn disable_failed_vad_backend(&mut self, failed_backend: VadBackend) -> bool {
|
||||
if self.vad_backend == failed_backend && failed_backend != VadBackend::WebrtcVad {
|
||||
self.vad_backend = VadBackend::WebrtcVad;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_config_is_valid_for_ios_vpio() {
|
||||
let config = AudioProcessingConfig::default();
|
||||
|
||||
assert!(config.validate_for_ios().is_ok());
|
||||
assert_eq!(
|
||||
config.processing_backend,
|
||||
AudioBackend::PlatformVoiceProcessing
|
||||
);
|
||||
assert_eq!(config.aec, EffectOwner::Platform);
|
||||
assert_eq!(config.ns, EffectOwner::Platform);
|
||||
assert_eq!(config.agc, EffectOwner::Platform);
|
||||
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn platform_voice_processing_rejects_sonora_effects() {
|
||||
let config = AudioProcessingConfig {
|
||||
ns: EffectOwner::Sonora,
|
||||
..AudioProcessingConfig::default()
|
||||
};
|
||||
|
||||
assert!(config.validate_for_ios().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ten_vad_has_stable_debug_string() {
|
||||
assert_eq!(VadBackend::TenVad.as_str(), "ten_vad");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sonora_experimental_allows_full_sonora_chain() {
|
||||
let config = AudioProcessingConfig {
|
||||
ios_mode: IosVoiceProcessingMode::SonoraExperimental,
|
||||
processing_backend: AudioBackend::Sonora,
|
||||
aec: EffectOwner::Sonora,
|
||||
ns: EffectOwner::Sonora,
|
||||
agc: EffectOwner::Sonora,
|
||||
..AudioProcessingConfig::default()
|
||||
};
|
||||
|
||||
assert!(config.validate_for_ios().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sonora_experimental_rejects_non_sonora_backend() {
|
||||
let config = AudioProcessingConfig {
|
||||
ios_mode: IosVoiceProcessingMode::SonoraExperimental,
|
||||
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
||||
aec: EffectOwner::Sonora,
|
||||
ns: EffectOwner::Sonora,
|
||||
agc: EffectOwner::Sonora,
|
||||
..AudioProcessingConfig::default()
|
||||
};
|
||||
|
||||
assert!(config.validate_for_ios().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disable_failed_vad_backend_demotes_to_webrtc() {
|
||||
let mut config = AudioProcessingConfig {
|
||||
vad_backend: VadBackend::SileroOnnx,
|
||||
..AudioProcessingConfig::default()
|
||||
};
|
||||
|
||||
assert!(config.disable_failed_vad_backend(VadBackend::SileroOnnx));
|
||||
assert_eq!(config.vad_backend, VadBackend::WebrtcVad);
|
||||
assert!(!config.disable_failed_vad_backend(VadBackend::SileroOnnx));
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime audio processing stats exposed to bridge/UI diagnostics.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AudioProcessingStats {
|
||||
/// Input dBFS before processing.
|
||||
pub input_dbfs: f32,
|
||||
/// Render dBFS before playout.
|
||||
pub render_dbfs: f32,
|
||||
/// Processed capture dBFS.
|
||||
pub processed_dbfs: f32,
|
||||
/// Latest VAD probability or fallback confidence.
|
||||
pub vad_probability: f32,
|
||||
/// VAD active state.
|
||||
pub vad_active: bool,
|
||||
/// Current resolved transmit state.
|
||||
pub transmitting: bool,
|
||||
/// VAD backend.
|
||||
pub vad_backend: VadBackend,
|
||||
/// Whether fallback VAD is active.
|
||||
pub vad_fallback_active: bool,
|
||||
/// Processing backend.
|
||||
pub processing_backend: AudioBackend,
|
||||
/// iOS mode.
|
||||
pub ios_voice_processing_mode: IosVoiceProcessingMode,
|
||||
/// Route class.
|
||||
pub audio_route: AudioRoute,
|
||||
/// Actual sample rate.
|
||||
pub actual_sample_rate_hz: u32,
|
||||
/// Actual IO buffer frame count.
|
||||
pub actual_io_buffer_frames: u32,
|
||||
/// Input overrun count.
|
||||
pub input_overruns: u64,
|
||||
/// Output underrun count.
|
||||
pub output_underruns: u64,
|
||||
/// Callback xrun count.
|
||||
pub callback_xruns: u64,
|
||||
/// Clipped sample count.
|
||||
pub clipped_samples: u64,
|
||||
/// Sonora enabled.
|
||||
pub sonora_enabled: bool,
|
||||
/// Platform voice processing enabled.
|
||||
pub platform_voice_processing_enabled: bool,
|
||||
}
|
||||
|
||||
/// Lock-free stats storage shared with callbacks.
|
||||
pub struct SharedAudioProcessingStats {
|
||||
input_dbfs: AtomicU32,
|
||||
render_dbfs: AtomicU32,
|
||||
processed_dbfs: AtomicU32,
|
||||
vad_probability: AtomicU32,
|
||||
vad_active: AtomicBool,
|
||||
transmitting: AtomicBool,
|
||||
vad_fallback_active: AtomicBool,
|
||||
input_overruns: AtomicU64,
|
||||
output_underruns: AtomicU64,
|
||||
callback_xruns: AtomicU64,
|
||||
clipped_samples: AtomicU64,
|
||||
actual_sample_rate_hz: AtomicU32,
|
||||
actual_io_buffer_frames: AtomicU32,
|
||||
}
|
||||
|
||||
impl Default for SharedAudioProcessingStats {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
input_dbfs: AtomicU32::new((-120.0_f32).to_bits()),
|
||||
render_dbfs: AtomicU32::new((-120.0_f32).to_bits()),
|
||||
processed_dbfs: AtomicU32::new((-120.0_f32).to_bits()),
|
||||
vad_probability: AtomicU32::new(0.0_f32.to_bits()),
|
||||
vad_active: AtomicBool::new(false),
|
||||
transmitting: AtomicBool::new(false),
|
||||
vad_fallback_active: AtomicBool::new(false),
|
||||
input_overruns: AtomicU64::new(0),
|
||||
output_underruns: AtomicU64::new(0),
|
||||
callback_xruns: AtomicU64::new(0),
|
||||
clipped_samples: AtomicU64::new(0),
|
||||
actual_sample_rate_hz: AtomicU32::new(crate::frame::SAMPLE_RATE_HZ),
|
||||
actual_io_buffer_frames: AtomicU32::new(crate::frame::FRAME_20MS_SAMPLES as u32),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SharedAudioProcessingStats {
|
||||
/// Store capture levels and VAD state.
|
||||
pub fn update_capture(
|
||||
&self,
|
||||
input_dbfs: f32,
|
||||
processed_dbfs: f32,
|
||||
probability: f32,
|
||||
vad_active: bool,
|
||||
transmitting: bool,
|
||||
) {
|
||||
self.input_dbfs
|
||||
.store(input_dbfs.to_bits(), Ordering::Relaxed);
|
||||
self.processed_dbfs
|
||||
.store(processed_dbfs.to_bits(), Ordering::Relaxed);
|
||||
self.vad_probability
|
||||
.store(probability.clamp(0.0, 1.0).to_bits(), Ordering::Relaxed);
|
||||
self.vad_active.store(vad_active, Ordering::Relaxed);
|
||||
self.transmitting.store(transmitting, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Store render level and callback buffer size.
|
||||
pub fn update_render(&self, dbfs: f32, io_buffer_frames: u32) {
|
||||
self.render_dbfs.store(dbfs.to_bits(), Ordering::Relaxed);
|
||||
self.actual_io_buffer_frames
|
||||
.store(io_buffer_frames, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Increment output underrun count.
|
||||
pub fn increment_output_underrun(&self) {
|
||||
self.output_underruns.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Increment callback xrun count.
|
||||
pub fn increment_callback_xrun(&self) {
|
||||
self.callback_xruns.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Add clipped samples.
|
||||
pub fn add_clipped_samples(&self, count: u64) {
|
||||
self.clipped_samples.fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Store whether the selected VAD backend is currently using a fallback.
|
||||
pub fn set_vad_fallback_active(&self, active: bool) {
|
||||
self.vad_fallback_active.store(active, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Build an owned stats snapshot with config-derived labels.
|
||||
pub fn snapshot(&self, config: &AudioProcessingConfig) -> AudioProcessingStats {
|
||||
AudioProcessingStats {
|
||||
input_dbfs: f32::from_bits(self.input_dbfs.load(Ordering::Relaxed)),
|
||||
render_dbfs: f32::from_bits(self.render_dbfs.load(Ordering::Relaxed)),
|
||||
processed_dbfs: f32::from_bits(self.processed_dbfs.load(Ordering::Relaxed)),
|
||||
vad_probability: f32::from_bits(self.vad_probability.load(Ordering::Relaxed)),
|
||||
vad_active: self.vad_active.load(Ordering::Relaxed),
|
||||
transmitting: self.transmitting.load(Ordering::Relaxed),
|
||||
vad_backend: config.vad_backend,
|
||||
vad_fallback_active: self.vad_fallback_active.load(Ordering::Relaxed),
|
||||
processing_backend: config.processing_backend,
|
||||
ios_voice_processing_mode: config.ios_mode,
|
||||
audio_route: config.route,
|
||||
actual_sample_rate_hz: self.actual_sample_rate_hz.load(Ordering::Relaxed),
|
||||
actual_io_buffer_frames: self.actual_io_buffer_frames.load(Ordering::Relaxed),
|
||||
input_overruns: self.input_overruns.load(Ordering::Relaxed),
|
||||
output_underruns: self.output_underruns.load(Ordering::Relaxed),
|
||||
callback_xruns: self.callback_xruns.load(Ordering::Relaxed),
|
||||
clipped_samples: self.clipped_samples.load(Ordering::Relaxed),
|
||||
sonora_enabled: config.processing_backend == AudioBackend::Sonora,
|
||||
platform_voice_processing_enabled: config.processing_backend
|
||||
== AudioBackend::PlatformVoiceProcessing,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
//! Async WAV debug dump writer for P1 diagnostics.
|
||||
//!
|
||||
//! Captures three streams for offline analysis:
|
||||
//! * `raw_mic` — before AudioProcessor (INV_007: never from callback)
|
||||
//! * `render_reference` — remote mixer output before playout
|
||||
//! * `processed_mic` — after AudioProcessor
|
||||
//!
|
||||
//! ## Design
|
||||
//!
|
||||
//! The realtime callback MUST NOT write to disk (INV_007). Instead it
|
||||
//! pushes 10 ms f32 frames onto a bounded `std::sync::mpsc` channel.
|
||||
//! A background `tokio::task` drains the channel and writes WAV data.
|
||||
//!
|
||||
//! The channel is bounded (capacity = 500 frames ≈ 5 s of audio per
|
||||
//! stream). If the writer falls behind, frames are dropped rather than
|
||||
//! blocking the callback thread.
|
||||
//!
|
||||
//! WAV files are written to the OS temp directory with a filename that
|
||||
//! encodes the stream name, route, backend, and a timestamp so
|
||||
//! multiple sessions don't overwrite each other.
|
||||
//!
|
||||
//! ## Usage
|
||||
//!
|
||||
//! ```ignore
|
||||
//! let writer = WavDebugRecorder::start(route, backend);
|
||||
//! // In realtime callback (non-blocking):
|
||||
//! writer.push_raw_mic(&frame);
|
||||
//! writer.push_render_reference(&frame);
|
||||
//! writer.push_processed_mic(&frame);
|
||||
//! // On session end:
|
||||
//! writer.stop(); // flushes and closes files
|
||||
//! ```
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::audio_processing::{AudioBackend, AudioRoute};
|
||||
use crate::frame::FRAME_10MS_SAMPLES;
|
||||
|
||||
/// Maximum number of 10 ms frames buffered per stream before drops.
|
||||
const CHANNEL_CAPACITY: usize = 500;
|
||||
|
||||
/// Sample rate for WAV output (matches the capture pipeline).
|
||||
const WAV_SAMPLE_RATE: u32 = 48_000;
|
||||
|
||||
/// Identifies which debug stream a frame belongs to.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum StreamId {
|
||||
RawMic,
|
||||
RenderReference,
|
||||
ProcessedMic,
|
||||
}
|
||||
|
||||
/// A single 10 ms frame tagged with its stream.
|
||||
struct DebugFrame {
|
||||
stream: StreamId,
|
||||
samples: Box<[f32; FRAME_10MS_SAMPLES]>,
|
||||
}
|
||||
|
||||
/// Handle for pushing frames from the realtime callback.
|
||||
///
|
||||
/// All push methods are non-blocking: if the channel is full the
|
||||
/// frame is silently dropped and a counter is incremented.
|
||||
pub struct WavDebugRecorder {
|
||||
tx: mpsc::SyncSender<DebugFrame>,
|
||||
/// Frames dropped due to full channel (diagnostic only).
|
||||
drops: std::sync::atomic::AtomicU64,
|
||||
/// Whether the recorder is active (set to false on stop).
|
||||
active: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
impl WavDebugRecorder {
|
||||
/// Start the async WAV writer task. Returns a handle for pushing
|
||||
/// frames from the realtime callback.
|
||||
///
|
||||
/// `route` and `backend` are embedded in the output filenames.
|
||||
pub fn start(route: AudioRoute, backend: AudioBackend) -> std::sync::Arc<Self> {
|
||||
let (tx, rx) = mpsc::sync_channel::<DebugFrame>(CHANNEL_CAPACITY);
|
||||
let recorder = std::sync::Arc::new(Self {
|
||||
tx,
|
||||
drops: std::sync::atomic::AtomicU64::new(0),
|
||||
active: std::sync::atomic::AtomicBool::new(true),
|
||||
});
|
||||
|
||||
let ts = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
let route_str = route.as_str().to_string();
|
||||
let backend_str = backend.as_str().to_string();
|
||||
|
||||
// Spawn a blocking task so the WAV writer doesn't compete
|
||||
// with the tokio async executor for CPU time.
|
||||
std::thread::Builder::new()
|
||||
.name("chanora-wav-writer".to_string())
|
||||
.spawn(move || {
|
||||
wav_writer_task(rx, &route_str, &backend_str, ts);
|
||||
})
|
||||
.ok();
|
||||
|
||||
recorder
|
||||
}
|
||||
|
||||
/// Push a raw mic frame (before AudioProcessor). Non-blocking.
|
||||
pub fn push_raw_mic(&self, samples: &[f32; FRAME_10MS_SAMPLES]) {
|
||||
self.push(StreamId::RawMic, samples);
|
||||
}
|
||||
|
||||
/// Push a render-reference frame (remote mixer output before playout).
|
||||
/// Non-blocking.
|
||||
pub fn push_render_reference(&self, samples: &[f32; FRAME_10MS_SAMPLES]) {
|
||||
self.push(StreamId::RenderReference, samples);
|
||||
}
|
||||
|
||||
/// Push a processed mic frame (after AudioProcessor). Non-blocking.
|
||||
pub fn push_processed_mic(&self, samples: &[f32; FRAME_10MS_SAMPLES]) {
|
||||
self.push(StreamId::ProcessedMic, samples);
|
||||
}
|
||||
|
||||
/// Stop the recorder. Drops the sender so the writer task drains
|
||||
/// and closes the WAV files.
|
||||
pub fn stop(&self) {
|
||||
self.active
|
||||
.store(false, std::sync::atomic::Ordering::Relaxed);
|
||||
// The sender is not dropped here because `self` is behind Arc.
|
||||
// The writer task will exit when all senders are dropped (i.e.
|
||||
// when the Arc is dropped). This is intentional: the task
|
||||
// drains any remaining frames before closing files.
|
||||
}
|
||||
|
||||
/// Number of frames dropped due to a full channel.
|
||||
pub fn drop_count(&self) -> u64 {
|
||||
self.drops.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn push(&self, stream: StreamId, samples: &[f32; FRAME_10MS_SAMPLES]) {
|
||||
if !self.active.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let mut boxed = Box::new([0.0_f32; FRAME_10MS_SAMPLES]);
|
||||
boxed.copy_from_slice(samples);
|
||||
let frame = DebugFrame {
|
||||
stream,
|
||||
samples: boxed,
|
||||
};
|
||||
if self.tx.try_send(frame).is_err() {
|
||||
self.drops
|
||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- WAV writer task ----------
|
||||
|
||||
struct WavFile {
|
||||
path: PathBuf,
|
||||
file: std::fs::File,
|
||||
samples_written: u32,
|
||||
}
|
||||
|
||||
impl WavFile {
|
||||
fn create(dir: &std::path::Path, name: &str) -> Option<Self> {
|
||||
let path = dir.join(name);
|
||||
match std::fs::File::create(&path) {
|
||||
Ok(mut file) => {
|
||||
// Write a placeholder WAV header; we'll patch it on close.
|
||||
if write_wav_header(&mut file, 0).is_ok() {
|
||||
Some(Self {
|
||||
path,
|
||||
file,
|
||||
samples_written: 0,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(target: "chanora_audio", error = %e, path = %path.display(), "wav debug: failed to create file");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_samples(&mut self, samples: &[f32]) {
|
||||
for &s in samples {
|
||||
let i16_val = (s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16;
|
||||
let _ = self.file.write_all(&i16_val.to_le_bytes());
|
||||
}
|
||||
self.samples_written += samples.len() as u32;
|
||||
}
|
||||
|
||||
fn finalize(mut self) {
|
||||
// Seek back to the start and rewrite the header with the
|
||||
// correct data size.
|
||||
use std::io::Seek;
|
||||
if self.file.seek(std::io::SeekFrom::Start(0)).is_ok() {
|
||||
let _ = write_wav_header(&mut self.file, self.samples_written);
|
||||
}
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
path = %self.path.display(),
|
||||
samples = self.samples_written,
|
||||
"wav debug: file closed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_wav_header(file: &mut std::fs::File, num_samples: u32) -> std::io::Result<()> {
|
||||
// PCM WAV header: 44 bytes.
|
||||
// Channels: 1 (mono), sample rate: 48000, bit depth: 16.
|
||||
let channels: u16 = 1;
|
||||
let sample_rate: u32 = WAV_SAMPLE_RATE;
|
||||
let bits_per_sample: u16 = 16;
|
||||
let byte_rate = sample_rate * channels as u32 * bits_per_sample as u32 / 8;
|
||||
let block_align = channels * bits_per_sample / 8;
|
||||
let data_size = num_samples * channels as u32 * bits_per_sample as u32 / 8;
|
||||
let chunk_size = 36 + data_size;
|
||||
|
||||
file.write_all(b"RIFF")?;
|
||||
file.write_all(&chunk_size.to_le_bytes())?;
|
||||
file.write_all(b"WAVE")?;
|
||||
file.write_all(b"fmt ")?;
|
||||
file.write_all(&16u32.to_le_bytes())?; // subchunk1 size
|
||||
file.write_all(&1u16.to_le_bytes())?; // PCM format
|
||||
file.write_all(&channels.to_le_bytes())?;
|
||||
file.write_all(&sample_rate.to_le_bytes())?;
|
||||
file.write_all(&byte_rate.to_le_bytes())?;
|
||||
file.write_all(&block_align.to_le_bytes())?;
|
||||
file.write_all(&bits_per_sample.to_le_bytes())?;
|
||||
file.write_all(b"data")?;
|
||||
file.write_all(&data_size.to_le_bytes())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn wav_writer_task(rx: mpsc::Receiver<DebugFrame>, route: &str, backend: &str, ts: u64) {
|
||||
let dir = std::env::temp_dir();
|
||||
let prefix = format!("chanora_debug_{route}_{backend}_{ts}");
|
||||
|
||||
let mut raw_mic = WavFile::create(&dir, &format!("{prefix}_raw_mic.wav"));
|
||||
let mut render_ref = WavFile::create(&dir, &format!("{prefix}_render_reference.wav"));
|
||||
let mut processed = WavFile::create(&dir, &format!("{prefix}_processed_mic.wav"));
|
||||
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
dir = %dir.display(),
|
||||
prefix = %prefix,
|
||||
"wav debug: writer started"
|
||||
);
|
||||
|
||||
for frame in rx {
|
||||
match frame.stream {
|
||||
StreamId::RawMic => {
|
||||
if let Some(f) = raw_mic.as_mut() {
|
||||
f.write_samples(&*frame.samples);
|
||||
}
|
||||
}
|
||||
StreamId::RenderReference => {
|
||||
if let Some(f) = render_ref.as_mut() {
|
||||
f.write_samples(&*frame.samples);
|
||||
}
|
||||
}
|
||||
StreamId::ProcessedMic => {
|
||||
if let Some(f) = processed.as_mut() {
|
||||
f.write_samples(&*frame.samples);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Channel closed — finalize all files.
|
||||
if let Some(f) = raw_mic {
|
||||
f.finalize();
|
||||
}
|
||||
if let Some(f) = render_ref {
|
||||
f.finalize();
|
||||
}
|
||||
if let Some(f) = processed {
|
||||
f.finalize();
|
||||
}
|
||||
|
||||
info!(target: "chanora_audio", "wav debug: writer task exited");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn recorder_starts_and_stops_without_panic() {
|
||||
let rec =
|
||||
WavDebugRecorder::start(AudioRoute::Speaker, AudioBackend::PlatformVoiceProcessing);
|
||||
let frame = [0.1_f32; FRAME_10MS_SAMPLES];
|
||||
rec.push_raw_mic(&frame);
|
||||
rec.push_render_reference(&frame);
|
||||
rec.push_processed_mic(&frame);
|
||||
rec.stop();
|
||||
// Drop the Arc to let the writer task drain.
|
||||
drop(rec);
|
||||
// Give the writer thread a moment to finish.
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_count_increments_when_channel_full() {
|
||||
// Use a tiny channel by creating a recorder and flooding it.
|
||||
// We can't easily test the bounded channel directly, but we
|
||||
// can verify the drop counter starts at zero.
|
||||
let rec = WavDebugRecorder::start(AudioRoute::Speaker, AudioBackend::Noop);
|
||||
assert_eq!(rec.drop_count(), 0);
|
||||
rec.stop();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wav_header_is_44_bytes() {
|
||||
// Write to a temp file to test the header.
|
||||
let tmp = std::env::temp_dir().join("chanora_test_wav_header.wav");
|
||||
let mut f = std::fs::File::create(&tmp).unwrap();
|
||||
write_wav_header(&mut f, 960).unwrap();
|
||||
drop(f);
|
||||
let data = std::fs::read(&tmp).unwrap();
|
||||
assert_eq!(data.len(), 44, "WAV header must be 44 bytes");
|
||||
assert_eq!(&data[0..4], b"RIFF");
|
||||
assert_eq!(&data[8..12], b"WAVE");
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
}
|
||||
}
|
||||
+233
-136
@@ -33,44 +33,21 @@ use tracing::{debug, info};
|
||||
))]
|
||||
use tracing::{error, warn};
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
#[cfg(any(target_os = "ios", target_os = "android"))]
|
||||
use tracing::warn;
|
||||
|
||||
use tsclientlib::audio::AudioHandler;
|
||||
|
||||
use chanora_protocol::{InboundVoice, OutPacket};
|
||||
|
||||
use crate::AudioError;
|
||||
|
||||
#[cfg(all(
|
||||
not(target_os = "ios"),
|
||||
not(target_os = "macos"),
|
||||
not(target_os = "android")
|
||||
))]
|
||||
use audiopus::coder::Encoder as OpusEncoder;
|
||||
#[cfg(all(
|
||||
not(target_os = "ios"),
|
||||
not(target_os = "macos"),
|
||||
not(target_os = "android")
|
||||
))]
|
||||
use audiopus::{
|
||||
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
|
||||
SampleRate as OpusSampleRate,
|
||||
};
|
||||
|
||||
use tsclientlib::audio::AudioHandler;
|
||||
|
||||
// `AudioData`, `CodecType`, `OutAudio` are referenced only by the
|
||||
// cpal capture pipeline's Opus encode path (`CaptureState::encode_and_send`).
|
||||
// `InboundVoice` + `OutPacket` are used by every platform — the
|
||||
// inbound forwarder task pumps `InboundVoice` into AudioHandler on
|
||||
// iOS too, and `OutPacket` flows out of the capture pipeline once
|
||||
// commit 3 lands. Cfg-gate the cpal-only ones to keep iOS warnings
|
||||
// clean.
|
||||
#[cfg(all(
|
||||
not(target_os = "ios"),
|
||||
not(target_os = "macos"),
|
||||
not(target_os = "android")
|
||||
))]
|
||||
use chanora_protocol::{AudioData, CodecType, OutAudio};
|
||||
use chanora_protocol::{InboundVoice, OutPacket};
|
||||
|
||||
use crate::AudioError;
|
||||
|
||||
/// Stable Chanora-side identifier for AudioHandler bookkeeping.
|
||||
/// We only ever have one connection at a time (DEC-006), so this is
|
||||
/// trivially unique.
|
||||
@@ -89,11 +66,8 @@ pub struct SessionAudioId(pub u64);
|
||||
const SAMPLE_RATE: u32 = 48_000;
|
||||
#[allow(dead_code)]
|
||||
const FRAME_SAMPLES: usize = 48_000 / 50; // 960
|
||||
#[allow(dead_code)]
|
||||
const MAX_OPUS_FRAME: usize = 1275;
|
||||
|
||||
/// Engine configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub struct AudioEngineConfig {
|
||||
/// Input gain applied before encoding (1.0 = pass-through).
|
||||
pub mic_gain: f32,
|
||||
@@ -117,6 +91,24 @@ pub struct AudioEngineConfig {
|
||||
/// is rejected on Android because the P0 path intentionally has
|
||||
/// no generic mobile-audio fallback.
|
||||
pub mobile_voice_preset: bool,
|
||||
/// Optional selector used by P1 VoiceActivity to publish VAD state.
|
||||
#[doc(hidden)]
|
||||
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AudioEngineConfig {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("AudioEngineConfig")
|
||||
.field("mic_gain", &self.mic_gain)
|
||||
.field("ptt_initial", &self.ptt_initial)
|
||||
.field("effects", &self.effects)
|
||||
.field("mobile_voice_preset", &self.mobile_voice_preset)
|
||||
.field(
|
||||
"voice_activity_selector",
|
||||
&self.voice_activity_selector.as_ref().map(|_| "present"),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AudioEngineConfig {
|
||||
@@ -126,6 +118,7 @@ impl Default for AudioEngineConfig {
|
||||
ptt_initial: false,
|
||||
effects: crate::AudioEffects::default(),
|
||||
mobile_voice_preset: true,
|
||||
voice_activity_selector: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -150,6 +143,16 @@ pub struct AudioEngine {
|
||||
/// independent of the server-side mute the protocol layer
|
||||
/// broadcasts.
|
||||
output_muted: Arc<AtomicBool>,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
mic_gain: f32,
|
||||
|
||||
// Streams must be dropped to stop audio. Both are `!Send` because
|
||||
// cpal's Stream isn't Send on some backends; we keep them in an
|
||||
@@ -177,7 +180,7 @@ pub struct AudioEngine {
|
||||
))]
|
||||
_output_stream: Mutex<Option<cpal::Stream>>,
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
_ios_voice_unit: Mutex<Option<crate::ios_voice_unit::IosVoiceUnit>>,
|
||||
_ios_voice_backend: Mutex<Option<IosVoiceBackend>>,
|
||||
/// SDD-111..SDD-115: Android Oboe voice backend. Owns the input
|
||||
/// and output streams, SDD-113 hardware-effect handles, and the
|
||||
/// foreground-service lifecycle; tearing it down on engine drop
|
||||
@@ -237,6 +240,105 @@ pub struct AudioEngine {
|
||||
unsafe impl Send for AudioEngine {}
|
||||
unsafe impl Sync for AudioEngine {}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
#[allow(dead_code)]
|
||||
enum IosVoiceBackend {
|
||||
Vpio(crate::ios_voice_unit::IosVoiceUnit),
|
||||
#[cfg(target_os = "ios")]
|
||||
Raw(crate::ios_raw_unit::IosRawUnit),
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
impl IosVoiceBackend {
|
||||
fn pause(&mut self) -> Result<(), AudioError> {
|
||||
#[cfg(target_os = "ios")]
|
||||
{
|
||||
match self {
|
||||
Self::Vpio(unit) => unit.pause(),
|
||||
Self::Raw(unit) => unit.pause(),
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn resume(&mut self) -> Result<(), AudioError> {
|
||||
#[cfg(target_os = "ios")]
|
||||
{
|
||||
match self {
|
||||
Self::Vpio(unit) => unit.resume(),
|
||||
Self::Raw(unit) => unit.resume(),
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn open_ios_voice_backend(
|
||||
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||||
output_gain: Arc<AtomicU32>,
|
||||
output_muted: Arc<AtomicBool>,
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_flag_for_capture: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
) -> Result<IosVoiceBackend, AudioError> {
|
||||
let _cfg = audio_processing_config.lock().unwrap().clone();
|
||||
#[cfg(target_os = "ios")]
|
||||
{
|
||||
if _cfg.ios_mode == crate::IosVoiceProcessingMode::SonoraExperimental {
|
||||
match crate::ios_raw_unit::IosRawUnit::start(
|
||||
handler.clone(),
|
||||
output_gain.clone(),
|
||||
output_muted.clone(),
|
||||
voice_out_tx.clone(),
|
||||
transmit_flag_for_capture.clone(),
|
||||
frames_sent.clone(),
|
||||
mic_gain,
|
||||
voice_activity_selector.clone(),
|
||||
audio_processing_config.clone(),
|
||||
audio_processing_stats.clone(),
|
||||
) {
|
||||
Ok(unit) => {
|
||||
info!(target: "chanora_audio", "ios: RemoteIO/Sonora experimental backend selected");
|
||||
return Ok(IosVoiceBackend::Raw(unit));
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
error = %e,
|
||||
"ios: RemoteIO/Sonora backend failed; falling back to VoiceProcessingIO"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let unit = crate::ios_voice_unit::IosVoiceUnit::start(
|
||||
handler,
|
||||
output_gain,
|
||||
output_muted,
|
||||
voice_out_tx,
|
||||
transmit_flag_for_capture,
|
||||
frames_sent,
|
||||
mic_gain,
|
||||
voice_activity_selector,
|
||||
audio_processing_config,
|
||||
audio_processing_stats,
|
||||
)?;
|
||||
Ok(IosVoiceBackend::Vpio(unit))
|
||||
}
|
||||
|
||||
impl AudioEngine {
|
||||
/// Start the engine: open capture + playback streams, spawn the
|
||||
/// inbound-voice forwarder, return a handle.
|
||||
@@ -259,6 +361,7 @@ impl AudioEngine {
|
||||
voice_in_rx: mpsc::Receiver<InboundVoice>,
|
||||
transmit_gate: crate::ptt::AudioTransmitGate,
|
||||
) -> Result<Self, AudioError> {
|
||||
#[allow(clippy::needless_return)]
|
||||
// Apple platforms route to a separate backend (VoiceProcessingIO
|
||||
// via coreaudio-rs) because cpal does not expose the native
|
||||
// voice-processing AudioUnit controls Chanora needs for VoIP.
|
||||
@@ -353,6 +456,8 @@ impl AudioEngine {
|
||||
let frames_received = Arc::new(AtomicU32::new(0));
|
||||
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
|
||||
let output_muted = Arc::new(AtomicBool::new(false));
|
||||
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
|
||||
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||||
|
||||
// ---------- Capture ----------
|
||||
// Capture is best-effort. If the platform default input
|
||||
@@ -525,6 +630,8 @@ impl AudioEngine {
|
||||
frames_received,
|
||||
output_gain,
|
||||
output_muted,
|
||||
audio_processing_config,
|
||||
audio_processing_stats,
|
||||
_input_stream: Mutex::new(input_stream),
|
||||
_output_stream: Mutex::new(Some(output_stream)),
|
||||
shutdown_tx: Some(shutdown_tx),
|
||||
@@ -549,6 +656,8 @@ impl AudioEngine {
|
||||
let frames_received = Arc::new(AtomicU32::new(0));
|
||||
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
|
||||
let output_muted = Arc::new(AtomicBool::new(false));
|
||||
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
|
||||
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||||
|
||||
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
|
||||
Arc::new(Mutex::new(AudioHandler::new()));
|
||||
@@ -678,6 +787,8 @@ impl AudioEngine {
|
||||
frames_received,
|
||||
output_gain,
|
||||
output_muted,
|
||||
audio_processing_config,
|
||||
audio_processing_stats,
|
||||
_android_voice_unit: Mutex::new(Some(android_voice_unit)),
|
||||
audio_mode_stack: Mutex::new(audio_mode_stack),
|
||||
shutdown_tx: Some(shutdown_tx),
|
||||
@@ -725,23 +836,26 @@ impl AudioEngine {
|
||||
let frames_received = Arc::new(AtomicU32::new(0));
|
||||
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
|
||||
let output_muted = Arc::new(AtomicBool::new(false));
|
||||
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
|
||||
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
|
||||
|
||||
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
|
||||
Arc::new(Mutex::new(AudioHandler::new()));
|
||||
let voice_out_tx_for_backend = voice_out_tx.clone();
|
||||
|
||||
// Construct the VPIO unit. Commit 1 ships a no-op callback
|
||||
// pair; commits 3 + 4 land the real capture + playback
|
||||
// wiring. Construction failure here is fatal (mirrors how
|
||||
// the cpal output-stream construction failure is fatal in
|
||||
// the non-iOS path).
|
||||
let ios_voice_unit = crate::ios_voice_unit::IosVoiceUnit::start(
|
||||
// Construct the live iOS voice backend. Platform VPIO stays
|
||||
// the default shipping path; Sonora/RemoteIO remains opt-in.
|
||||
let ios_voice_backend = open_ios_voice_backend(
|
||||
audio_handler.clone(),
|
||||
output_gain.clone(),
|
||||
output_muted.clone(),
|
||||
voice_out_tx,
|
||||
voice_out_tx_for_backend,
|
||||
transmit_flag_for_capture,
|
||||
frames_sent.clone(),
|
||||
cfg.mic_gain,
|
||||
cfg.voice_activity_selector.clone(),
|
||||
audio_processing_config.clone(),
|
||||
audio_processing_stats.clone(),
|
||||
)?;
|
||||
|
||||
// Capture is always considered active on iOS — VPIO's
|
||||
@@ -791,7 +905,13 @@ impl AudioEngine {
|
||||
frames_received,
|
||||
output_gain,
|
||||
output_muted,
|
||||
_ios_voice_unit: Mutex::new(Some(ios_voice_unit)),
|
||||
audio_processing_config,
|
||||
audio_processing_stats,
|
||||
audio_handler,
|
||||
voice_out_tx,
|
||||
voice_activity_selector: cfg.voice_activity_selector.clone(),
|
||||
mic_gain: cfg.mic_gain,
|
||||
_ios_voice_backend: Mutex::new(Some(ios_voice_backend)),
|
||||
shutdown_tx: Some(shutdown_tx),
|
||||
capture_active,
|
||||
ptt_watchdog,
|
||||
@@ -826,7 +946,7 @@ impl AudioEngine {
|
||||
}
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
let _ = self._ios_voice_unit.lock().unwrap().take();
|
||||
let _ = self._ios_voice_backend.lock().unwrap().take();
|
||||
}
|
||||
// SDD-115 reverse-order teardown on Android:
|
||||
// 1) close the voice unit (releases SDD-113 hardware
|
||||
@@ -900,15 +1020,25 @@ impl AudioEngine {
|
||||
/// iOS-only: restart the underlying VoiceProcessingIO unit after
|
||||
/// route changes.
|
||||
pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> {
|
||||
#[cfg(target_os = "ios")]
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
let mut guard = self._ios_voice_unit.lock().unwrap();
|
||||
let unit = guard
|
||||
.as_mut()
|
||||
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
|
||||
return unit.restart();
|
||||
let backend = open_ios_voice_backend(
|
||||
self.audio_handler.clone(),
|
||||
self.output_gain.clone(),
|
||||
self.output_muted.clone(),
|
||||
self.voice_out_tx.clone(),
|
||||
self.transmit_gate.flag_arc(),
|
||||
self.frames_sent.clone(),
|
||||
self.mic_gain,
|
||||
self.voice_activity_selector.clone(),
|
||||
self.audio_processing_config.clone(),
|
||||
self.audio_processing_stats.clone(),
|
||||
)?;
|
||||
let mut guard = self._ios_voice_backend.lock().unwrap();
|
||||
*guard = Some(backend);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(not(target_os = "ios"))]
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
@@ -916,15 +1046,15 @@ impl AudioEngine {
|
||||
|
||||
/// iOS-only: pause the underlying VoiceProcessingIO unit.
|
||||
pub fn ios_pause_voice_unit(&self) -> Result<(), AudioError> {
|
||||
#[cfg(target_os = "ios")]
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
let mut guard = self._ios_voice_unit.lock().unwrap();
|
||||
let mut guard = self._ios_voice_backend.lock().unwrap();
|
||||
let unit = guard
|
||||
.as_mut()
|
||||
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
|
||||
return unit.pause();
|
||||
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
|
||||
unit.pause()
|
||||
}
|
||||
#[cfg(not(target_os = "ios"))]
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
@@ -932,15 +1062,15 @@ impl AudioEngine {
|
||||
|
||||
/// iOS-only: resume the underlying VoiceProcessingIO unit.
|
||||
pub fn ios_resume_voice_unit(&self) -> Result<(), AudioError> {
|
||||
#[cfg(target_os = "ios")]
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
let mut guard = self._ios_voice_unit.lock().unwrap();
|
||||
let mut guard = self._ios_voice_backend.lock().unwrap();
|
||||
let unit = guard
|
||||
.as_mut()
|
||||
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
|
||||
return unit.resume();
|
||||
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
|
||||
unit.resume()
|
||||
}
|
||||
#[cfg(not(target_os = "ios"))]
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
@@ -1014,6 +1144,29 @@ impl AudioEngine {
|
||||
self.frames_received.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Current audio-processing config snapshot.
|
||||
pub fn audio_processing_config_snapshot(&self) -> crate::AudioProcessingConfig {
|
||||
self.audio_processing_config.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Apply a voice-processing config after validating iOS invariants.
|
||||
pub fn set_audio_processing_config(
|
||||
&self,
|
||||
config: crate::AudioProcessingConfig,
|
||||
) -> Result<(), AudioError> {
|
||||
#[cfg(target_os = "ios")]
|
||||
config.validate_for_ios()?;
|
||||
let mut guard = self.audio_processing_config.lock().unwrap();
|
||||
*guard = config;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Current voice-processing stats snapshot.
|
||||
pub fn audio_processing_stats(&self) -> crate::AudioProcessingStats {
|
||||
let config = self.audio_processing_config.lock().unwrap().clone();
|
||||
self.audio_processing_stats.snapshot(&config)
|
||||
}
|
||||
|
||||
/// Latest Android voice-audio diagnostics snapshot (SDD-112 item
|
||||
/// 10 / SDD-113 item 7 / SDD-116 item 3). On non-Android targets
|
||||
/// this always returns `None`. On Android it returns `Some(...)`
|
||||
@@ -1091,58 +1244,7 @@ fn try_open_capture(
|
||||
in_stream_cfg.buffer_size = cpal::BufferSize::Default;
|
||||
}
|
||||
|
||||
let mut opus_enc = OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
|
||||
.map_err(|e| AudioError::Opus(format!("encoder new: {e}")))?;
|
||||
|
||||
// Opus VOIP tuning. Defaults give us 'auto' bitrate (can drop
|
||||
// to ~6 kbps during silence \u2014 which sounds garbled when
|
||||
// talking resumes) and inband FEC disabled. On lossy mobile
|
||||
// networks (cellular / iPhone WiFi roaming), packet loss
|
||||
// without FEC produces audible clicks + cut-out frames.
|
||||
//
|
||||
// Settings derived from the Opus IETF VoIP recommendations
|
||||
// (RFC 6716 \u00a7 7.1) and Discord's voice client tuning:
|
||||
//
|
||||
// * Bitrate 32 kbps : sweet spot for mono voice. Lower
|
||||
// than 24 kbps starts to sound watery; higher than
|
||||
// 64 kbps wastes bandwidth without perceptual gain on a
|
||||
// human voice. Discord uses 64 kbps; mumble defaults to
|
||||
// 40 kbps; we pick 32 kbps as a conservative VoIP value
|
||||
// that survives 100 kbps uplinks comfortably.
|
||||
// * Complexity 10 : max quality. The CPU cost on a modern
|
||||
// iPhone (A14+) or any desktop is negligible (~0.5 % of
|
||||
// a single core for 48 kHz mono).
|
||||
// * Inband FEC on : opus inserts a low-bitrate redundancy
|
||||
// copy of the previous frame inside the current packet
|
||||
// so a single dropped packet can be reconstructed from
|
||||
// the next one. Essential on lossy mobile.
|
||||
// * Packet loss perc 5 % : tells the encoder to expect 5 %
|
||||
// loss and pre-emptively budget bits for FEC. Higher
|
||||
// values trade audio quality for resilience.
|
||||
//
|
||||
// Errors here are non-fatal: log + continue. The encoder
|
||||
// works with defaults if any setter fails on an exotic
|
||||
// libopus build.
|
||||
if let Err(e) = opus_enc.set_bitrate(OpusBitrate::BitsPerSecond(32_000)) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus: set_bitrate(32000) failed");
|
||||
}
|
||||
if let Err(e) = opus_enc.set_complexity(10) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus: set_complexity(10) failed");
|
||||
}
|
||||
if let Err(e) = opus_enc.set_inband_fec(true) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus: set_inband_fec(true) failed");
|
||||
}
|
||||
if let Err(e) = opus_enc.set_packet_loss_perc(5) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus: set_packet_loss_perc(5) failed");
|
||||
}
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
bitrate_bps = 32_000,
|
||||
complexity = 10,
|
||||
inband_fec = true,
|
||||
packet_loss_perc = 5,
|
||||
"opus encoder tuned for VoIP"
|
||||
);
|
||||
let opus_enc = crate::opus_voice::new_voip_encoder("cpal capture")?;
|
||||
|
||||
let capture_state = Arc::new(Mutex::new(CaptureState::new(
|
||||
opus_enc,
|
||||
@@ -1184,7 +1286,7 @@ struct CaptureState {
|
||||
/// roughly at the period rate (~100 Hz for a 10 ms period on
|
||||
/// Linux ALSA defaults).
|
||||
resample_last: f32,
|
||||
opus_out: [u8; MAX_OPUS_FRAME],
|
||||
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
/// The PTT transmission gate. Read once per outbound frame; the
|
||||
/// CaptureState never mutates this flag.
|
||||
@@ -1224,7 +1326,7 @@ impl CaptureState {
|
||||
pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2),
|
||||
resample_pos: 0.0,
|
||||
resample_last: 0.0,
|
||||
opus_out: [0u8; MAX_OPUS_FRAME],
|
||||
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx,
|
||||
transmit_active,
|
||||
frames_sent,
|
||||
@@ -1310,22 +1412,21 @@ impl CaptureState {
|
||||
.encode_float(&frame[..], &mut self.opus_out[..])
|
||||
{
|
||||
Ok(len) => {
|
||||
let packet = OutAudio::new(&AudioData::C2S {
|
||||
id: 0,
|
||||
codec: CodecType::OpusVoice,
|
||||
data: &self.opus_out[..len],
|
||||
});
|
||||
match self.voice_out_tx.try_send(packet) {
|
||||
Ok(()) => {
|
||||
self.frames_sent.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Full(_)) => {
|
||||
warn!(target: "chanora_audio", "voice_out queue full; dropping frame");
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => {
|
||||
crate::opus_voice::send_voip_frame(
|
||||
&self.voice_out_tx,
|
||||
&self.frames_sent,
|
||||
&self.opus_out,
|
||||
len,
|
||||
|| {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"voice_out queue full; dropping frame"
|
||||
);
|
||||
},
|
||||
|| {
|
||||
warn!(target: "chanora_audio", "voice_out closed; stopping send");
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "opus encode failed");
|
||||
@@ -1851,10 +1952,7 @@ pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> {
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||||
#[doc(hidden)]
|
||||
pub mod bench_seam {
|
||||
use super::{
|
||||
Arc, AtomicBool, AtomicU32, CaptureState, OpusApp, OpusChannels, OpusEncoder,
|
||||
OpusSampleRate, OutPacket,
|
||||
};
|
||||
use super::{Arc, AtomicBool, AtomicU32, CaptureState, OpusEncoder, OutPacket};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
/// Opaque handle wrapping a CaptureState plus the dummy mpsc
|
||||
@@ -1880,8 +1978,7 @@ pub mod bench_seam {
|
||||
/// (typically 1 or 2).
|
||||
pub fn new(in_sample_rate: u32, in_channels: usize) -> Self {
|
||||
let encoder =
|
||||
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
|
||||
.expect("opus encoder init");
|
||||
crate::opus_voice::new_voip_encoder("cpal bench").expect("opus encoder init");
|
||||
let (tx, rx) = mpsc::channel::<OutPacket>(64);
|
||||
let transmit_active = Arc::new(AtomicBool::new(true));
|
||||
let frames_sent = Arc::new(AtomicU32::new(0));
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
//! Canonical P1 voice frame helpers.
|
||||
//!
|
||||
//! The network contract remains 48 kHz mono, 20 ms Opus frames. P1
|
||||
//! processing works internally on 10 ms f32 frames so VAD and future
|
||||
//! processors can share a stable frame size without changing the
|
||||
//! transport layer.
|
||||
|
||||
/// P1 sample rate in Hz.
|
||||
pub const SAMPLE_RATE_HZ: u32 = 48_000;
|
||||
/// Network frame duration in milliseconds.
|
||||
pub const NETWORK_FRAME_MS: u32 = 20;
|
||||
/// Processing frame duration in milliseconds.
|
||||
pub const PROCESSING_FRAME_MS: u32 = 10;
|
||||
/// Samples in one 10 ms mono frame at 48 kHz.
|
||||
pub const FRAME_10MS_SAMPLES: usize = 480;
|
||||
/// Samples in one 20 ms mono frame at 48 kHz.
|
||||
pub const FRAME_20MS_SAMPLES: usize = 960;
|
||||
|
||||
/// 10 ms, 48 kHz, mono f32 processing frame.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct AudioFrame10ms {
|
||||
/// Samples normalized to `[-1.0, 1.0]`.
|
||||
pub samples: [f32; FRAME_10MS_SAMPLES],
|
||||
}
|
||||
|
||||
/// 20 ms, 48 kHz, mono f32 network-frame-sized buffer.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct AudioFrame20ms {
|
||||
/// Samples normalized to `[-1.0, 1.0]`.
|
||||
pub samples: [f32; FRAME_20MS_SAMPLES],
|
||||
}
|
||||
|
||||
impl AudioFrame20ms {
|
||||
/// Convert one 20 ms frame into two 10 ms processing frames.
|
||||
pub fn split(&self) -> (AudioFrame10ms, AudioFrame10ms) {
|
||||
let mut first = [0.0; FRAME_10MS_SAMPLES];
|
||||
let mut second = [0.0; FRAME_10MS_SAMPLES];
|
||||
first.copy_from_slice(&self.samples[..FRAME_10MS_SAMPLES]);
|
||||
second.copy_from_slice(&self.samples[FRAME_10MS_SAMPLES..]);
|
||||
(
|
||||
AudioFrame10ms { samples: first },
|
||||
AudioFrame10ms { samples: second },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl AudioFrame10ms {
|
||||
/// Merge two 10 ms processing frames back into the 20 ms network
|
||||
/// cadence used by the existing Opus path.
|
||||
pub fn merge(first: &Self, second: &Self) -> AudioFrame20ms {
|
||||
let mut samples = [0.0; FRAME_20MS_SAMPLES];
|
||||
samples[..FRAME_10MS_SAMPLES].copy_from_slice(&first.samples);
|
||||
samples[FRAME_10MS_SAMPLES..].copy_from_slice(&second.samples);
|
||||
AudioFrame20ms { samples }
|
||||
}
|
||||
|
||||
/// Compute RMS dBFS for diagnostics and fallback VAD.
|
||||
pub fn dbfs(&self) -> f32 {
|
||||
dbfs(&self.samples)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert i16 PCM to normalized f32 PCM.
|
||||
pub fn i16_to_f32(sample: i16) -> f32 {
|
||||
sample as f32 / i16::MAX as f32
|
||||
}
|
||||
|
||||
/// Convert normalized f32 PCM to saturated i16 PCM.
|
||||
pub fn f32_to_i16(sample: f32) -> i16 {
|
||||
(sample.clamp(-1.0, 1.0) * i16::MAX as f32) as i16
|
||||
}
|
||||
|
||||
/// RMS dBFS for a normalized f32 slice. Silence returns `-120.0`.
|
||||
pub fn dbfs(samples: &[f32]) -> f32 {
|
||||
if samples.is_empty() {
|
||||
return -120.0;
|
||||
}
|
||||
let sum = samples.iter().map(|s| s * s).sum::<f32>();
|
||||
let rms = (sum / samples.len() as f32).sqrt();
|
||||
if rms <= 0.000_001 {
|
||||
-120.0
|
||||
} else {
|
||||
20.0 * rms.log10()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn split_merge_preserves_samples() {
|
||||
let mut samples = [0.0; FRAME_20MS_SAMPLES];
|
||||
for (i, s) in samples.iter_mut().enumerate() {
|
||||
*s = i as f32 / FRAME_20MS_SAMPLES as f32;
|
||||
}
|
||||
let original = AudioFrame20ms { samples };
|
||||
let (a, b) = original.split();
|
||||
assert_eq!(AudioFrame10ms::merge(&a, &b), original);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,492 @@
|
||||
//! Optional raw iOS RemoteIO path for the Sonora experimental mode.
|
||||
//!
|
||||
//! Provides an alternative to `ios_voice_unit.rs` for the
|
||||
//! `SonoraExperimental` processing mode. Instead of
|
||||
//! `kAudioUnitSubType_VoiceProcessingIO` (which owns AEC/NS/AGC), it
|
||||
//! opens `kAudioUnitSubType_RemoteIO` with voice processing explicitly
|
||||
//! disabled so Rust's Sonora DSP chain can own the full signal path.
|
||||
//!
|
||||
//! ## Hard invariants enforced here
|
||||
//!
|
||||
//! * INV_009: Rust AEC only active when platform AEC is disabled.
|
||||
//! * INV_010: VoiceProcessingIO and Sonora AEC3 are mutually exclusive.
|
||||
//! * INV_011: Software AEC backend receives both capture and render-reference.
|
||||
//! * INV_012: Render reference is copied from decoded/mixed remote PCM
|
||||
//! before playout.
|
||||
//!
|
||||
//! ## Fallback
|
||||
//!
|
||||
//! If RemoteIO construction fails, the caller falls back to `IosVoiceUnit`
|
||||
//! (VPIO) and logs the error.
|
||||
//!
|
||||
//! ## Status
|
||||
//!
|
||||
//! Experimental / disabled by default. Only activated when the user
|
||||
//! explicitly selects `SonoraExperimental` mode via the bridge API.
|
||||
//!
|
||||
//! ## Platform
|
||||
//!
|
||||
//! `kAudioUnitSubType_RemoteIO` is only available in the iOS SDK.
|
||||
//! This module is gated to `target_os = "ios"`.
|
||||
|
||||
#[cfg(target_os = "ios")]
|
||||
pub use inner::IosRawUnit;
|
||||
|
||||
#[cfg(target_os = "ios")]
|
||||
mod inner {
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use audiopus::coder::Encoder as OpusEncoder;
|
||||
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
|
||||
use coreaudio::audio_unit::render_callback::{self, data};
|
||||
use coreaudio::audio_unit::IOType;
|
||||
use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat};
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{info, warn};
|
||||
use tsclientlib::audio::AudioHandler;
|
||||
|
||||
use crate::engine::SessionAudioId;
|
||||
use crate::processor::AudioProcessor;
|
||||
use crate::AudioError;
|
||||
use chanora_protocol::OutPacket;
|
||||
|
||||
const SAMPLE_RATE_HZ: f64 = 48_000.0;
|
||||
|
||||
// ------------------------------------------------------------------ //
|
||||
// Render-reference ring buffer //
|
||||
// ------------------------------------------------------------------ //
|
||||
|
||||
/// 4-slot ring buffer shared between the render callback (writer) and
|
||||
/// the capture callback (reader for Sonora AEC3). Capacity: 4 × 10 ms
|
||||
/// = 40 ms of headroom.
|
||||
///
|
||||
/// If the capture callback runs before the render callback has written
|
||||
/// a frame it reads zeros (silence reference), which is safe — Sonora
|
||||
/// AEC3 simply skips cancellation for that frame.
|
||||
struct RenderReferenceBuffer {
|
||||
buf: Box<[[f32; 480]; 4]>,
|
||||
write_idx: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
impl RenderReferenceBuffer {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
buf: Box::new([[0.0; 480]; 4]),
|
||||
write_idx: std::sync::atomic::AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
/// Write one 10 ms render-reference frame. Realtime-safe.
|
||||
fn write(&self, frame: &[f32; 480]) {
|
||||
let idx = self.write_idx.load(Ordering::Relaxed);
|
||||
// SAFETY: only one writer (render callback); torn reads
|
||||
// are bounded to one frame of AEC degradation.
|
||||
unsafe {
|
||||
let slot = &self.buf[idx] as *const [f32; 480] as *mut [f32; 480];
|
||||
(*slot).copy_from_slice(frame);
|
||||
}
|
||||
self.write_idx.store((idx + 1) % 4, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Read the most recently completed render-reference frame.
|
||||
fn read_latest(&self) -> [f32; 480] {
|
||||
let wi = self.write_idx.load(Ordering::Relaxed);
|
||||
let ri = (wi + 3) % 4;
|
||||
self.buf[ri]
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: accessed from two audio callback threads; data races are
|
||||
// bounded to one frame of AEC quality degradation.
|
||||
unsafe impl Send for RenderReferenceBuffer {}
|
||||
unsafe impl Sync for RenderReferenceBuffer {}
|
||||
|
||||
// ------------------------------------------------------------------ //
|
||||
// Capture pipeline state //
|
||||
// ------------------------------------------------------------------ //
|
||||
|
||||
struct RawCaptureState {
|
||||
encoder: OpusEncoder,
|
||||
pcm_accum: Vec<i16>,
|
||||
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||||
/// Processing config — retained for route-change reloads; not read in the hot path.
|
||||
#[allow(dead_code)]
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
sonora_processor: crate::processor::SonoraProcessor,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_10ms_len: usize,
|
||||
wav_recorder: Option<Arc<crate::debug_wav::WavDebugRecorder>>,
|
||||
}
|
||||
|
||||
impl RawCaptureState {
|
||||
fn new(
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
) -> Result<Self, AudioError> {
|
||||
let encoder = crate::opus_voice::new_voip_encoder("ios raw")?;
|
||||
Ok(Self {
|
||||
encoder,
|
||||
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
|
||||
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx,
|
||||
transmit_active,
|
||||
frames_sent,
|
||||
mic_gain,
|
||||
voice_activity_selector,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||||
audio_processing_config,
|
||||
sonora_processor: crate::processor::SonoraProcessor::with_config(
|
||||
crate::processor::sonora::SonoraConfig::with_aec3(),
|
||||
),
|
||||
audio_processing_stats,
|
||||
render_reference,
|
||||
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_10ms_len: 0,
|
||||
wav_recorder: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) {
|
||||
if let Ok(mut cfg) = self.audio_processing_config.try_lock() {
|
||||
let _ = cfg.disable_failed_vad_backend(failed_backend);
|
||||
}
|
||||
}
|
||||
|
||||
fn ingest_i16(&mut self, samples: &[i16]) {
|
||||
// Accumulate into 10 ms frames for VAD / Sonora processing.
|
||||
let mut offset = 0;
|
||||
while offset < samples.len() {
|
||||
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
|
||||
let take = remaining.min(samples.len() - offset);
|
||||
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
|
||||
.copy_from_slice(&samples[offset..offset + take]);
|
||||
self.pending_10ms_len += take;
|
||||
offset += take;
|
||||
|
||||
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
|
||||
let frame = self.pending_10ms;
|
||||
self.process_10ms_capture_frame(&frame);
|
||||
self.pending_10ms_len = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if !self.transmit_active.load(Ordering::Relaxed) {
|
||||
self.pcm_accum.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
// Encode complete 20 ms Opus frames.
|
||||
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
|
||||
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
|
||||
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
|
||||
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
|
||||
|
||||
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
|
||||
Ok(len) => {
|
||||
crate::opus_voice::send_voip_frame(
|
||||
&self.voice_out_tx,
|
||||
&self.frames_sent,
|
||||
&self.opus_out,
|
||||
len,
|
||||
|| {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"ios raw: voice_out queue full; dropping frame"
|
||||
);
|
||||
},
|
||||
|| {},
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(target: "chanora_audio",
|
||||
error = %e, "ios raw opus encode failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn process_10ms_capture_frame(
|
||||
&mut self,
|
||||
samples: &[i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
) {
|
||||
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||||
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
|
||||
*dst = crate::frame::i16_to_f32(src);
|
||||
}
|
||||
let input_dbfs = crate::frame::dbfs(&frame);
|
||||
|
||||
// WAV tap: raw mic (before processing).
|
||||
if let Some(ref rec) = self.wav_recorder {
|
||||
rec.push_raw_mic(&frame);
|
||||
}
|
||||
|
||||
// INV_012: feed render reference to Sonora AEC3 before capture.
|
||||
let render_ref = self.render_reference.read_latest();
|
||||
self.sonora_processor.process_render(&render_ref);
|
||||
self.sonora_processor.process_capture(&mut frame);
|
||||
|
||||
// WAV tap: processed mic (after Sonora).
|
||||
if let Some(ref rec) = self.wav_recorder {
|
||||
rec.push_processed_mic(&frame);
|
||||
}
|
||||
|
||||
let (vad_backend, vad_hangover) = self
|
||||
.audio_processing_config
|
||||
.try_lock()
|
||||
.map(|cfg| (cfg.vad_backend, cfg.vad_hangover_ms))
|
||||
.unwrap_or((
|
||||
crate::VadBackend::WebrtcVad,
|
||||
crate::voice_activity::VAD_HANGOVER_MS,
|
||||
));
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = matches!(
|
||||
vad_backend,
|
||||
crate::VadBackend::SileroOnnx | crate::VadBackend::TenVad
|
||||
);
|
||||
if used_fallback_vad {
|
||||
self.disable_failed_vad_backend(vad_backend);
|
||||
}
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
let active = self.vad_state.update(vad.speech);
|
||||
if let Some(sel) = &self.voice_activity_selector {
|
||||
sel.set_voice_activity_open(active);
|
||||
}
|
||||
self.audio_processing_stats.update_capture(
|
||||
input_dbfs,
|
||||
crate::frame::dbfs(&frame),
|
||||
vad.probability,
|
||||
active,
|
||||
self.transmit_active.load(Ordering::Relaxed),
|
||||
);
|
||||
|
||||
if !self.transmit_active.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
|
||||
let gain = self.mic_gain;
|
||||
if (gain - 1.0).abs() < f32::EPSILON {
|
||||
self.pcm_accum
|
||||
.extend(frame.iter().copied().map(crate::frame::f32_to_i16));
|
||||
} else {
|
||||
self.pcm_accum.extend(frame.iter().copied().map(|s| {
|
||||
let scaled = crate::frame::f32_to_i16(s) as f32 * gain;
|
||||
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ //
|
||||
// IosRawUnit //
|
||||
// ------------------------------------------------------------------ //
|
||||
|
||||
/// Raw iOS RemoteIO audio unit for the Sonora experimental path.
|
||||
pub struct IosRawUnit {
|
||||
unit: AudioUnit,
|
||||
}
|
||||
|
||||
impl IosRawUnit {
|
||||
/// Open a RemoteIO AudioUnit, install render + input callbacks, start.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn start(
|
||||
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||||
output_gain: Arc<AtomicU32>,
|
||||
output_muted: Arc<AtomicBool>,
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
) -> Result<Self, AudioError> {
|
||||
// INV_010: reject if config requests VPIO (that's IosVoiceUnit's job).
|
||||
{
|
||||
let cfg = audio_processing_config.lock().unwrap();
|
||||
if cfg.ios_mode == crate::IosVoiceProcessingMode::PlatformVoiceProcessing {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"IosRawUnit requires SonoraExperimental mode".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut unit = AudioUnit::new_uninitialized(IOType::RemoteIO)
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio new: {e}")))?;
|
||||
|
||||
// Enable input on bus 1.
|
||||
const ENABLE_IO: u32 = 2003;
|
||||
let enable: u32 = 1;
|
||||
unit.set_property(ENABLE_IO, Scope::Input, Element::Input, Some(&enable))
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio enable input: {e}")))?;
|
||||
|
||||
// 48 kHz Int16 mono on both buses.
|
||||
let fmt = StreamFormat {
|
||||
sample_rate: SAMPLE_RATE_HZ,
|
||||
sample_format: SampleFormat::I16,
|
||||
flags: LinearPcmFlags::IS_SIGNED_INTEGER | LinearPcmFlags::IS_PACKED,
|
||||
channels: 1,
|
||||
};
|
||||
unit.set_stream_format(fmt, Scope::Input, Element::Output)
|
||||
.map_err(|e| AudioError::StreamConfig(format!("remoteio fmt output: {e}")))?;
|
||||
unit.set_stream_format(fmt, Scope::Output, Element::Input)
|
||||
.map_err(|e| AudioError::StreamConfig(format!("remoteio fmt input: {e}")))?;
|
||||
|
||||
// Shared render-reference buffer (INV_011 / INV_012).
|
||||
let render_ref_buf = RenderReferenceBuffer::new();
|
||||
let render_ref_for_capture = render_ref_buf.clone();
|
||||
|
||||
let mut capture_state = RawCaptureState::new(
|
||||
voice_out_tx,
|
||||
transmit_active,
|
||||
frames_sent,
|
||||
mic_gain,
|
||||
voice_activity_selector,
|
||||
audio_processing_config,
|
||||
audio_processing_stats.clone(),
|
||||
render_ref_for_capture,
|
||||
)?;
|
||||
|
||||
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
|
||||
capture_state.ingest_i16(args.data.buffer);
|
||||
Ok(())
|
||||
})
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio input cb: {e}")))?;
|
||||
|
||||
let mut scratch: Vec<f32> = Vec::with_capacity(2048);
|
||||
let stats_render = audio_processing_stats.clone();
|
||||
|
||||
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
|
||||
let out = args.data.buffer;
|
||||
let n = out.len();
|
||||
let stereo_n = n * 2;
|
||||
if scratch.len() < stereo_n {
|
||||
scratch.resize(stereo_n, 0.0);
|
||||
}
|
||||
scratch[..stereo_n].fill(0.0);
|
||||
|
||||
match handler.try_lock() {
|
||||
Ok(mut h) => {
|
||||
let _ = h.fill_buffer(&mut scratch[..stereo_n]);
|
||||
}
|
||||
Err(std::sync::TryLockError::WouldBlock) => {
|
||||
stats_render.increment_callback_xrun();
|
||||
}
|
||||
Err(std::sync::TryLockError::Poisoned(e)) => {
|
||||
warn!(target: "chanora_audio",
|
||||
"AudioHandler poisoned (raw render): {e}");
|
||||
}
|
||||
}
|
||||
|
||||
// INV_012: copy render reference BEFORE playout.
|
||||
let mono_n = n.min(480);
|
||||
let mut ref_frame = [0.0_f32; 480];
|
||||
crate::voice_render::downmix_stereo_f32_to_mono_f32(
|
||||
&scratch[..stereo_n],
|
||||
&mut ref_frame[..mono_n],
|
||||
);
|
||||
render_ref_buf.write(&ref_frame);
|
||||
|
||||
let gain = f32::from_bits(output_gain.load(Ordering::Relaxed));
|
||||
let muted = output_muted.load(Ordering::Relaxed);
|
||||
let mix_stats = crate::voice_render::downmix_stereo_f32_to_mono_i16(
|
||||
&scratch[..stereo_n],
|
||||
out,
|
||||
gain,
|
||||
muted,
|
||||
);
|
||||
if mix_stats.clipped_samples > 0 {
|
||||
stats_render.add_clipped_samples(mix_stats.clipped_samples);
|
||||
}
|
||||
stats_render.update_render(crate::frame::dbfs(&scratch[..stereo_n]), n as u32);
|
||||
Ok(())
|
||||
})
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio render cb: {e}")))?;
|
||||
|
||||
unit.initialize()
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio init: {e}")))?;
|
||||
unit.start()
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio start: {e}")))?;
|
||||
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
sample_rate_hz = SAMPLE_RATE_HZ,
|
||||
"ios RemoteIO (Sonora experimental) started"
|
||||
);
|
||||
Ok(Self { unit })
|
||||
}
|
||||
|
||||
/// Restart the unit after a route change (stop → uninit → init → start).
|
||||
pub fn restart(&mut self) -> Result<(), AudioError> {
|
||||
self.unit
|
||||
.stop()
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio restart stop: {e}")))?;
|
||||
self.unit
|
||||
.uninitialize()
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio restart uninit: {e}")))?;
|
||||
self.unit
|
||||
.initialize()
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio restart init: {e}")))?;
|
||||
self.unit
|
||||
.start()
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio restart start: {e}")))?;
|
||||
info!(target: "chanora_audio", "ios RemoteIO restarted");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pause the unit during an AVAudioSession interruption.
|
||||
pub fn pause(&mut self) -> Result<(), AudioError> {
|
||||
self.unit
|
||||
.stop()
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio pause: {e}")))
|
||||
}
|
||||
|
||||
/// Resume the unit after an interruption ends.
|
||||
pub fn resume(&mut self) -> Result<(), AudioError> {
|
||||
self.unit
|
||||
.start()
|
||||
.map_err(|e| AudioError::Backend(format!("remoteio resume: {e}")))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for IosRawUnit {
|
||||
fn drop(&mut self) {
|
||||
if let Err(e) = self.unit.stop() {
|
||||
warn!(target: "chanora_audio", error = %e,
|
||||
"ios RemoteIO stop on drop failed");
|
||||
} else {
|
||||
info!(target: "chanora_audio", "ios RemoteIO stopped");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -75,10 +75,6 @@ use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use audiopus::coder::Encoder as OpusEncoder;
|
||||
use audiopus::{
|
||||
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
|
||||
SampleRate as OpusSampleRate,
|
||||
};
|
||||
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
|
||||
use coreaudio::audio_unit::render_callback::{self, data};
|
||||
use coreaudio::audio_unit::IOType;
|
||||
@@ -89,22 +85,7 @@ use tsclientlib::audio::AudioHandler;
|
||||
|
||||
use crate::engine::SessionAudioId;
|
||||
use crate::AudioError;
|
||||
use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
|
||||
|
||||
/// 20 ms at 48 kHz mono — one Opus frame's worth of samples.
|
||||
/// Aligning the AudioUnit IO buffer to this frame size keeps the
|
||||
/// jitter-buffer / encoder handshake tight (no fractional-frame
|
||||
/// reads inside fill_buffer or accumulator drift inside the
|
||||
/// capture pipeline).
|
||||
const FRAME_SAMPLES_MONO: usize = 960;
|
||||
|
||||
/// Maximum size of an encoded Opus frame in bytes (per RFC 6716
|
||||
/// §3.2.1). Same constant the cpal-side `CaptureState` uses; we
|
||||
/// duplicate it here instead of cross-importing from engine.rs
|
||||
/// because engine.rs's copy is cfg-gated to non-iOS for cpal-only
|
||||
/// reasons. Post-step-5 review may dedupe by promoting both to a
|
||||
/// shared `crate::framing` module.
|
||||
const MAX_OPUS_FRAME: usize = 1275;
|
||||
use chanora_protocol::OutPacket;
|
||||
|
||||
/// Sample rate every layer above us assumes. Matches the Opus
|
||||
/// encoder rate, the `tsclientlib::AudioHandler` mix rate, and the
|
||||
@@ -124,6 +105,11 @@ const OUTPUT_BUS: Element = Element::Output;
|
||||
/// samples in.
|
||||
const INPUT_BUS: Element = Element::Input;
|
||||
|
||||
/// Pre-roll buffer capacity: 160 ms / 10 ms = 16 frames.
|
||||
/// Stores processed i16 frames so the first syllable is not lost
|
||||
/// when the VAD gate opens (VAD_004 / pre_roll_ms=160).
|
||||
const PRE_ROLL_FRAMES: usize = 16;
|
||||
|
||||
/// Capture pipeline state owned by the VPIO input callback. The
|
||||
/// AudioUnit hands us 48 kHz signed-int16 mono PCM directly (no
|
||||
/// downmix or resample needed — VPIO's hardware-side mix-down
|
||||
@@ -146,17 +132,36 @@ const INPUT_BUS: Element = Element::Input;
|
||||
/// shared with `AudioEngine`.
|
||||
struct IosCaptureState {
|
||||
encoder: OpusEncoder,
|
||||
/// 48 kHz mono PCM scratch accumulating to FRAME_SAMPLES_MONO
|
||||
/// 48 kHz mono PCM scratch accumulating to FRAME_20MS_SAMPLES
|
||||
/// per encode. Capacity 2x to absorb cpal-style buffer-size
|
||||
/// jitter without reallocating.
|
||||
pcm_accum: Vec<i16>,
|
||||
opus_out: [u8; MAX_OPUS_FRAME],
|
||||
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
/// PTT transmission gate. Read once per outbound frame; this
|
||||
/// struct never mutates the flag (SAD-075 / SDD-089).
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad,
|
||||
/// Background Silero worker — enqueues frames off the realtime
|
||||
/// callback and publishes the latest probability atomically.
|
||||
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
|
||||
/// Last VAD backend we configured — used to detect backend changes.
|
||||
current_vad_backend: crate::VadBackend,
|
||||
/// Last observed configured Silero model epoch.
|
||||
silero_model_epoch: u64,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
sonora_processor: crate::processor::SonoraProcessor,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_10ms_len: usize,
|
||||
pre_roll_buf: [[i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
|
||||
pre_roll_head: usize,
|
||||
pre_roll_count: usize,
|
||||
pre_roll_flushed: bool,
|
||||
capture_frame_seq: u64,
|
||||
wav_recorder: Arc<Mutex<Option<Arc<crate::debug_wav::WavDebugRecorder>>>>,
|
||||
}
|
||||
|
||||
impl IosCaptureState {
|
||||
@@ -164,54 +169,55 @@ impl IosCaptureState {
|
||||
/// Encoder configuration is the same as cpal-side
|
||||
/// `try_open_capture` (engine.rs) so audio quality is platform-
|
||||
/// neutral.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn new(
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
wav_recorder: Arc<Mutex<Option<Arc<crate::debug_wav::WavDebugRecorder>>>>,
|
||||
) -> Result<Self, AudioError> {
|
||||
let mut encoder =
|
||||
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
|
||||
.map_err(|e| AudioError::Opus(format!("encoder new (ios): {e}")))?;
|
||||
|
||||
// VoIP-tuned settings — bitrate 32 kbps, complexity 10,
|
||||
// inband FEC on, packet-loss-perc 5. Soft-fail each setter
|
||||
// with a warn log to match the cpal-side behaviour: an
|
||||
// unusual libopus build that rejects one setter shouldn't
|
||||
// tank the whole pipeline. Full rationale + RFC citations
|
||||
// are in engine.rs::try_open_capture line ~640.
|
||||
if let Err(e) = encoder.set_bitrate(OpusBitrate::BitsPerSecond(32_000)) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus(ios): set_bitrate(32000) failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_complexity(10) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus(ios): set_complexity(10) failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_inband_fec(true) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus(ios): set_inband_fec(true) failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_packet_loss_perc(5) {
|
||||
warn!(target: "chanora_audio", error = %e, "opus(ios): set_packet_loss_perc(5) failed");
|
||||
}
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
bitrate_bps = 32_000,
|
||||
complexity = 10,
|
||||
inband_fec = true,
|
||||
packet_loss_perc = 5,
|
||||
"ios VPIO opus encoder tuned for VoIP"
|
||||
);
|
||||
let encoder = crate::opus_voice::new_voip_encoder("ios VPIO")?;
|
||||
|
||||
Ok(Self {
|
||||
encoder,
|
||||
pcm_accum: Vec::with_capacity(FRAME_SAMPLES_MONO * 2),
|
||||
opus_out: [0u8; MAX_OPUS_FRAME],
|
||||
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
|
||||
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx,
|
||||
transmit_active,
|
||||
frames_sent,
|
||||
mic_gain,
|
||||
voice_activity_selector,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||||
silero_vad_worker: None,
|
||||
current_vad_backend: crate::VadBackend::WebrtcVad,
|
||||
silero_model_epoch: crate::vad::silero_model_epoch(),
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||||
audio_processing_config,
|
||||
sonora_processor: crate::processor::SonoraProcessor::new(),
|
||||
audio_processing_stats,
|
||||
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_10ms_len: 0,
|
||||
pre_roll_buf: [[0_i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
|
||||
pre_roll_head: 0,
|
||||
pre_roll_count: 0,
|
||||
pre_roll_flushed: false,
|
||||
capture_frame_seq: 0,
|
||||
wav_recorder,
|
||||
})
|
||||
}
|
||||
|
||||
fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) {
|
||||
if let Ok(mut cfg) = self.audio_processing_config.try_lock() {
|
||||
if cfg.disable_failed_vad_backend(failed_backend) {
|
||||
self.current_vad_backend = crate::VadBackend::WebrtcVad;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume the i16 mono buffer delivered by VPIO, accumulate
|
||||
/// to a 20 ms frame boundary, encode + send when PTT is held.
|
||||
///
|
||||
@@ -220,6 +226,22 @@ impl IosCaptureState {
|
||||
/// In practice "interleaved mono" is the same byte layout as
|
||||
/// "planar mono" so we just take the buffer as-is.
|
||||
fn ingest_i16(&mut self, samples: &[i16]) {
|
||||
let mut offset = 0;
|
||||
while offset < samples.len() {
|
||||
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
|
||||
let take = remaining.min(samples.len() - offset);
|
||||
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
|
||||
.copy_from_slice(&samples[offset..offset + take]);
|
||||
self.pending_10ms_len += take;
|
||||
offset += take;
|
||||
|
||||
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
|
||||
let frame = self.pending_10ms;
|
||||
self.process_10ms_capture_frame(&frame);
|
||||
self.pending_10ms_len = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if !self.transmit_active.load(Ordering::Relaxed) {
|
||||
// Drain accumulator while muted so we don't pop on the
|
||||
// PTT release edge. Matches cpal-side behaviour.
|
||||
@@ -227,63 +249,40 @@ impl IosCaptureState {
|
||||
return;
|
||||
}
|
||||
|
||||
// Mic-gain application. When gain==1.0 we skip the
|
||||
// multiply + saturate loop entirely — that's the common
|
||||
// case and the loop is the inner-most hot path of the
|
||||
// realtime audio thread.
|
||||
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
|
||||
self.pcm_accum.extend_from_slice(samples);
|
||||
} else {
|
||||
let gain = self.mic_gain;
|
||||
self.pcm_accum.extend(samples.iter().map(|&s| {
|
||||
// Saturating mul-then-cast keeps the signal in
|
||||
// the i16 envelope. Clipping in this branch is
|
||||
// expected — if the user pushed mic_gain past 1.0
|
||||
// and is shouting, the alternative is wrap-around
|
||||
// distortion which sounds far worse.
|
||||
let scaled = (s as f32) * gain;
|
||||
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
|
||||
}));
|
||||
}
|
||||
|
||||
// Drain complete 20 ms frames out of the accumulator, encode
|
||||
// each, send the resulting Opus packet on the protocol
|
||||
// queue. The `while` covers the case where a single VPIO
|
||||
// callback delivers more than one frame's worth (rare on
|
||||
// iOS where the HW IO buffer duration aligns with the
|
||||
// Opus frame, but always possible during route changes).
|
||||
while self.pcm_accum.len() >= FRAME_SAMPLES_MONO {
|
||||
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
|
||||
// Use a stack-allocated frame buffer to avoid the
|
||||
// per-callback allocation a `drain(..N).collect()`
|
||||
// would incur. The encoder doesn't need ownership.
|
||||
let mut frame = [0i16; FRAME_SAMPLES_MONO];
|
||||
frame.copy_from_slice(&self.pcm_accum[..FRAME_SAMPLES_MONO]);
|
||||
self.pcm_accum.drain(..FRAME_SAMPLES_MONO);
|
||||
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
|
||||
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
|
||||
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
|
||||
|
||||
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
|
||||
Ok(len) => {
|
||||
let packet = OutAudio::new(&AudioData::C2S {
|
||||
id: 0,
|
||||
codec: CodecType::OpusVoice,
|
||||
data: &self.opus_out[..len],
|
||||
});
|
||||
match self.voice_out_tx.try_send(packet) {
|
||||
Ok(()) => {
|
||||
self.frames_sent.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Full(_)) => {
|
||||
crate::opus_voice::send_voip_frame(
|
||||
&self.voice_out_tx,
|
||||
&self.frames_sent,
|
||||
&self.opus_out,
|
||||
len,
|
||||
|| {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"ios VPIO: voice_out queue full; dropping frame"
|
||||
);
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => {
|
||||
},
|
||||
|| {
|
||||
debug!(
|
||||
target: "chanora_audio",
|
||||
"ios VPIO: voice_out closed; capture pipeline stopping"
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "ios VPIO opus encode failed");
|
||||
@@ -291,6 +290,260 @@ impl IosCaptureState {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn process_10ms_capture_frame(&mut self, samples: &[i16; crate::frame::FRAME_10MS_SAMPLES]) {
|
||||
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||||
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
|
||||
*dst = crate::frame::i16_to_f32(src);
|
||||
}
|
||||
let input_dbfs = crate::frame::dbfs(&frame);
|
||||
|
||||
// WAV tap: raw mic (before processing, DIAG_002).
|
||||
if let Ok(guard) = self.wav_recorder.try_lock() {
|
||||
if let Some(rec) = guard.as_ref() {
|
||||
rec.push_raw_mic(&frame);
|
||||
}
|
||||
}
|
||||
|
||||
// Read config once per frame (try_lock: non-blocking, falls back to
|
||||
// last-known values if the lock is contended — safe to miss one frame).
|
||||
let (
|
||||
run_ns,
|
||||
run_agc,
|
||||
run_hpf,
|
||||
vad_backend,
|
||||
vad_hangover,
|
||||
debug_wav_dump_enabled,
|
||||
route,
|
||||
processing_backend,
|
||||
) = self
|
||||
.audio_processing_config
|
||||
.try_lock()
|
||||
.map(|cfg| {
|
||||
let ns =
|
||||
cfg.ns != crate::EffectOwner::Off && cfg.ns != crate::EffectOwner::Platform;
|
||||
let agc =
|
||||
cfg.agc != crate::EffectOwner::Off && cfg.agc != crate::EffectOwner::Platform;
|
||||
let hpf = cfg.hpf_enabled;
|
||||
(
|
||||
ns,
|
||||
agc,
|
||||
hpf,
|
||||
cfg.vad_backend,
|
||||
cfg.vad_hangover_ms,
|
||||
cfg.debug_wav_dump_enabled,
|
||||
cfg.route,
|
||||
cfg.processing_backend,
|
||||
)
|
||||
})
|
||||
.unwrap_or((
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
crate::VadBackend::SileroOnnx,
|
||||
crate::voice_activity::VAD_HANGOVER_MS,
|
||||
false,
|
||||
crate::AudioRoute::Unknown,
|
||||
crate::AudioBackend::PlatformVoiceProcessing,
|
||||
));
|
||||
|
||||
// Switch VAD backend when the config changes.
|
||||
let silero_model_epoch = crate::vad::silero_model_epoch();
|
||||
let silero_model_changed = vad_backend == crate::VadBackend::SileroOnnx
|
||||
&& silero_model_epoch != self.silero_model_epoch;
|
||||
|
||||
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
|
||||
if debug_wav_dump_enabled {
|
||||
if recorder_guard.is_none() {
|
||||
*recorder_guard = Some(crate::debug_wav::WavDebugRecorder::start(
|
||||
route,
|
||||
processing_backend,
|
||||
));
|
||||
}
|
||||
} else if let Some(recorder) = recorder_guard.take() {
|
||||
recorder.stop();
|
||||
}
|
||||
}
|
||||
|
||||
if vad_backend != self.current_vad_backend || silero_model_changed {
|
||||
self.current_vad_backend = vad_backend;
|
||||
self.silero_model_epoch = silero_model_epoch;
|
||||
match vad_backend {
|
||||
crate::VadBackend::SileroOnnx => {
|
||||
// Attempt to load Silero model from the well-known
|
||||
// bundle path. The actual inference runs on a
|
||||
// background worker; the callback only enqueues
|
||||
// 10 ms frames and falls back to WebRTC if the
|
||||
// worker is missing or stale.
|
||||
let model_path = crate::vad::silero_model_bundle_path();
|
||||
self.silero_vad_worker =
|
||||
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&model_path);
|
||||
if self.silero_vad_worker.is_none() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"Silero VAD model not found at {model_path}; falling back to WebRTC VAD"
|
||||
);
|
||||
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
|
||||
}
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(self.silero_vad_worker.is_none());
|
||||
}
|
||||
crate::VadBackend::TenVad => {
|
||||
self.silero_vad_worker = None;
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"TEN VAD selected but native TEN runtime is not bundled; falling back to WebRTC VAD"
|
||||
);
|
||||
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
|
||||
self.audio_processing_stats.set_vad_fallback_active(true);
|
||||
}
|
||||
_ => {
|
||||
self.silero_vad_worker = None;
|
||||
self.audio_processing_stats.set_vad_fallback_active(false);
|
||||
}
|
||||
}
|
||||
// Reset VAD state machine timers on backend switch.
|
||||
self.vad_state = crate::voice_activity::VoiceActivityStateMachine::new(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
self.vad_state.reset();
|
||||
}
|
||||
|
||||
// Keep the VAD state machine aligned with the active config.
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
|
||||
|
||||
// Apply the enabled stages through the SonoraProcessor.
|
||||
// We reconfigure it on-the-fly to match the current settings.
|
||||
if run_ns || run_agc || run_hpf {
|
||||
use crate::processor::sonora::SonoraConfig;
|
||||
use crate::processor::AudioProcessor;
|
||||
let new_cfg = SonoraConfig {
|
||||
hpf: run_hpf,
|
||||
aec3: false, // NEVER in VPIO path (INV_009)
|
||||
ns: run_ns,
|
||||
agc2: run_agc,
|
||||
};
|
||||
if new_cfg != *self.sonora_processor.config() {
|
||||
self.sonora_processor.apply_config(new_cfg);
|
||||
}
|
||||
self.sonora_processor.process_capture(&mut frame);
|
||||
}
|
||||
|
||||
// VAD: use Silero if loaded, otherwise WebRTC fallback.
|
||||
// Disabled backend → always open (Continuous-like for VAD mode).
|
||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||
let capture_seq = self.capture_frame_seq;
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
||||
if worker.try_send(capture_seq, &frame) && !worker.is_stale(capture_seq) {
|
||||
let probability = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability,
|
||||
speech: probability >= 0.5,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.silero_vad_worker = None;
|
||||
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::TenVad {
|
||||
used_fallback_vad = true;
|
||||
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
} else {
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
let gate_open = self.vad_state.update(vad.speech);
|
||||
if let Some(selector) = &self.voice_activity_selector {
|
||||
selector.set_voice_activity_open(gate_open);
|
||||
}
|
||||
self.audio_processing_stats.update_capture(
|
||||
input_dbfs,
|
||||
crate::frame::dbfs(&frame),
|
||||
vad.probability,
|
||||
gate_open,
|
||||
transmit_active,
|
||||
);
|
||||
|
||||
// WAV tap: processed mic (after Rust DSP, DIAG_002).
|
||||
if let Ok(guard) = self.wav_recorder.try_lock() {
|
||||
if let Some(rec) = guard.as_ref() {
|
||||
rec.push_processed_mic(&frame);
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to i16 for accumulation.
|
||||
let mut pcm_frame = [0_i16; crate::frame::FRAME_10MS_SAMPLES];
|
||||
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
|
||||
for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) {
|
||||
*dst = crate::frame::f32_to_i16(src);
|
||||
}
|
||||
} else {
|
||||
let gain = self.mic_gain;
|
||||
for (dst, src) in pcm_frame.iter_mut().zip(frame.iter().copied()) {
|
||||
let scaled = crate::frame::f32_to_i16(src) as f32 * gain;
|
||||
*dst = scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16;
|
||||
}
|
||||
}
|
||||
|
||||
// Update pre-roll ring buffer (VAD_004: preserve first syllable).
|
||||
let slot_idx = self.pre_roll_head % PRE_ROLL_FRAMES;
|
||||
self.pre_roll_buf[slot_idx] = pcm_frame;
|
||||
self.pre_roll_head = (self.pre_roll_head + 1) % PRE_ROLL_FRAMES;
|
||||
if self.pre_roll_count < PRE_ROLL_FRAMES {
|
||||
self.pre_roll_count += 1;
|
||||
}
|
||||
|
||||
// If the transmit gate just opened and we haven't flushed the
|
||||
// pre-roll yet, drain it into the accumulator.
|
||||
if transmit_active && !self.pre_roll_flushed {
|
||||
self.pre_roll_flushed = true;
|
||||
// The oldest frame in the ring is at
|
||||
// (pre_roll_head + PRE_ROLL_FRAMES - pre_roll_count) % PRE_ROLL_FRAMES.
|
||||
// We emit frames in chronological order (oldest first), excluding
|
||||
// the frame we just wrote (which goes into pcm_accum normally below).
|
||||
let oldest =
|
||||
(self.pre_roll_head + PRE_ROLL_FRAMES - self.pre_roll_count) % PRE_ROLL_FRAMES;
|
||||
// Emit pre_roll_count - 1 frames (the -1 excludes the current frame
|
||||
// which will be added below in the normal path).
|
||||
let pre_roll_to_emit = self.pre_roll_count.saturating_sub(1);
|
||||
for i in 0..pre_roll_to_emit {
|
||||
let idx = (oldest + i) % PRE_ROLL_FRAMES;
|
||||
self.pcm_accum.extend_from_slice(&self.pre_roll_buf[idx]);
|
||||
}
|
||||
} else if !transmit_active {
|
||||
// Gate closed — reset the flush flag so pre-roll fires again
|
||||
// on the next gate open.
|
||||
self.pre_roll_flushed = false;
|
||||
}
|
||||
|
||||
if !transmit_active {
|
||||
return;
|
||||
}
|
||||
|
||||
self.pcm_accum.extend_from_slice(&pcm_frame);
|
||||
}
|
||||
}
|
||||
|
||||
/// Live iOS audio unit wrapper. Construct + start = audio
|
||||
@@ -339,6 +592,9 @@ impl IosVoiceUnit {
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
) -> Result<Self, AudioError> {
|
||||
// Construct the VoiceProcessingIO AudioUnit. cpal exposes
|
||||
// `Default::default()` which on iOS picks the inferior
|
||||
@@ -430,8 +686,27 @@ impl IosVoiceUnit {
|
||||
// scratch are owned by the closure — no Mutex needed
|
||||
// because the input callback is the sole writer/reader on
|
||||
// the audio thread.
|
||||
let mut capture_state =
|
||||
IosCaptureState::new(voice_out_tx, transmit_active, frames_sent, mic_gain)?;
|
||||
let wav_recorder = Arc::new(Mutex::new({
|
||||
let cfg = audio_processing_config.lock().unwrap().clone();
|
||||
if cfg.debug_wav_dump_enabled {
|
||||
Some(crate::debug_wav::WavDebugRecorder::start(
|
||||
cfg.route,
|
||||
cfg.processing_backend,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}));
|
||||
let mut capture_state = IosCaptureState::new(
|
||||
voice_out_tx,
|
||||
transmit_active,
|
||||
frames_sent,
|
||||
mic_gain,
|
||||
voice_activity_selector,
|
||||
audio_processing_config,
|
||||
audio_processing_stats.clone(),
|
||||
wav_recorder.clone(),
|
||||
)?;
|
||||
|
||||
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
|
||||
// VPIO with our pinned stream format delivers
|
||||
@@ -499,12 +774,16 @@ impl IosVoiceUnit {
|
||||
let handler_for_render = handler.clone();
|
||||
let output_gain_for_render = output_gain.clone();
|
||||
let output_muted_for_render = output_muted.clone();
|
||||
let wav_recorder_for_render = wav_recorder.clone();
|
||||
// Diagnostic counters (sampled every 100 callbacks ~= 2 s).
|
||||
let mut cb_count: u64 = 0;
|
||||
let mut last_num_frames: usize = 0;
|
||||
let mut num_frames_changes: u32 = 0;
|
||||
let mut callbacks_with_audio: u64 = 0;
|
||||
let mut callbacks_with_silence: u64 = 0;
|
||||
let mut render_ref_accum = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||||
let mut render_ref_len: usize = 0;
|
||||
let mut render_recorder_active = false;
|
||||
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
|
||||
let out: &mut [i16] = args.data.buffer;
|
||||
let num_frames = out.len();
|
||||
@@ -527,6 +806,7 @@ impl IosVoiceUnit {
|
||||
let _removed = h.fill_buffer(&mut scratch_stereo[..needed]);
|
||||
}
|
||||
Err(std::sync::TryLockError::WouldBlock) => {
|
||||
audio_processing_stats.increment_callback_xrun();
|
||||
// scratch_stereo is already zeroed above.
|
||||
}
|
||||
Err(std::sync::TryLockError::Poisoned(e)) => {
|
||||
@@ -535,34 +815,52 @@ impl IosVoiceUnit {
|
||||
}
|
||||
}
|
||||
|
||||
// Downmix stereo f32 -> mono i16 with master gain.
|
||||
// (l + r) * 0.5 preserves total signal energy with
|
||||
// 3 dB headroom against sum-of-correlated-peaks
|
||||
// clipping. Hard-clip i16 cast at the boundary.
|
||||
let gain = f32::from_bits(output_gain_for_render.load(Ordering::Relaxed));
|
||||
let muted = output_muted_for_render.load(Ordering::Relaxed);
|
||||
let mut peak_out: i16 = 0;
|
||||
for (i, dst) in out.iter_mut().enumerate() {
|
||||
if muted {
|
||||
*dst = 0;
|
||||
continue;
|
||||
}
|
||||
let l = scratch_stereo[i * 2];
|
||||
let r = scratch_stereo[i * 2 + 1];
|
||||
let mono_f32 = (l + r) * 0.5 * gain;
|
||||
let clamped = mono_f32.clamp(-1.0, 1.0);
|
||||
let sample = (clamped * i16::MAX as f32) as i16;
|
||||
*dst = sample;
|
||||
let a = sample.unsigned_abs() as i16;
|
||||
if a > peak_out {
|
||||
peak_out = a;
|
||||
let mix_stats = crate::voice_render::downmix_stereo_f32_to_mono_i16(
|
||||
&scratch_stereo[..needed],
|
||||
out,
|
||||
gain,
|
||||
muted,
|
||||
);
|
||||
if mix_stats.clipped_samples > 0 {
|
||||
audio_processing_stats.add_clipped_samples(mix_stats.clipped_samples);
|
||||
}
|
||||
audio_processing_stats.update_render(
|
||||
crate::frame::dbfs(&scratch_stereo[..needed]),
|
||||
num_frames as u32,
|
||||
);
|
||||
|
||||
if let Ok(guard) = wav_recorder_for_render.try_lock() {
|
||||
if let Some(rec) = guard.as_ref() {
|
||||
if !render_recorder_active {
|
||||
render_ref_len = 0;
|
||||
render_ref_accum.fill(0.0);
|
||||
render_recorder_active = true;
|
||||
}
|
||||
let mut idx = 0;
|
||||
while idx + 1 < needed {
|
||||
let mono = (scratch_stereo[idx] + scratch_stereo[idx + 1]) * 0.5;
|
||||
render_ref_accum[render_ref_len] = mono;
|
||||
render_ref_len += 1;
|
||||
idx += 2;
|
||||
if render_ref_len == crate::frame::FRAME_10MS_SAMPLES {
|
||||
rec.push_render_reference(&render_ref_accum);
|
||||
render_ref_len = 0;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
render_recorder_active = false;
|
||||
}
|
||||
} else {
|
||||
render_recorder_active = false;
|
||||
}
|
||||
|
||||
// Track audio-vs-silence for the diagnostic.
|
||||
if peak_out > 0 {
|
||||
if mix_stats.peak_i16 > 0 {
|
||||
callbacks_with_audio = callbacks_with_audio.wrapping_add(1);
|
||||
} else {
|
||||
audio_processing_stats.increment_output_underrun();
|
||||
callbacks_with_silence = callbacks_with_silence.wrapping_add(1);
|
||||
}
|
||||
|
||||
@@ -580,7 +878,7 @@ impl IosVoiceUnit {
|
||||
frames_changes = num_frames_changes,
|
||||
callbacks_with_audio,
|
||||
callbacks_with_silence,
|
||||
peak_out_i16 = peak_out,
|
||||
peak_out_i16 = mix_stats.peak_i16,
|
||||
gain,
|
||||
"ios audio unit render callback diagnostic sample (direct fill_buffer)"
|
||||
);
|
||||
@@ -593,13 +891,52 @@ impl IosVoiceUnit {
|
||||
// stream formats we set above. After initialize() most
|
||||
// property changes are rejected (you have to uninitialize +
|
||||
// re-initialize), which is why the property set must come
|
||||
// first. Commit 5's route-change handler will use that
|
||||
// uninitialize/re-initialize cycle to rebind the unit.
|
||||
unit.initialize()
|
||||
.map_err(|e| AudioError::Backend(format!("vpio initialize: {e}")))?;
|
||||
// first.
|
||||
//
|
||||
// AudioUnit::initialize() issues an RPC to the CoreAudio server.
|
||||
// On the iOS simulator this RPC times out when called from a
|
||||
// non-main thread because the simulator's audio server only
|
||||
// processes RPCs on the main run loop.
|
||||
//
|
||||
// Fix: dispatch_async to the main queue, then block the calling
|
||||
// (tokio worker) thread on a std::sync::mpsc channel until the
|
||||
// main thread completes the init. This is safe because:
|
||||
// 1. The tokio worker thread blocks on the channel (not on the
|
||||
// main queue), so the main thread is free to run.
|
||||
// 2. AudioUnit is Send (coreaudio-rs marks it unsafe impl Send).
|
||||
// 3. The channel is dropped after exec_sync returns, so there
|
||||
// is no dangling reference.
|
||||
{
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<Result<(), String>>(1);
|
||||
// Move unit into the Arc so it can cross thread boundaries.
|
||||
let unit_arc = std::sync::Arc::new(std::sync::Mutex::new(Some(unit)));
|
||||
let unit_arc2 = unit_arc.clone();
|
||||
|
||||
unit.start()
|
||||
.map_err(|e| AudioError::Backend(format!("vpio start: {e}")))?;
|
||||
dispatch2::DispatchQueue::main().exec_async(move || {
|
||||
let mut guard = unit_arc2.lock().unwrap();
|
||||
let u = guard.as_mut().unwrap();
|
||||
let result = u
|
||||
.initialize()
|
||||
.map_err(|e| format!("vpio initialize: {e}"))
|
||||
.and_then(|_| u.start().map_err(|e| format!("vpio start: {e}")));
|
||||
let _ = tx.send(result);
|
||||
});
|
||||
|
||||
// Block the tokio worker thread until the main thread finishes.
|
||||
// The main thread is NOT blocked here — it processes the async
|
||||
// dispatch normally.
|
||||
match rx.recv() {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(msg)) => return Err(AudioError::Backend(msg)),
|
||||
Err(_) => {
|
||||
return Err(AudioError::Backend(
|
||||
"vpio init: main thread channel closed unexpectedly".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
unit = unit_arc.lock().unwrap().take().unwrap();
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
@@ -659,6 +996,10 @@ impl IosVoiceUnit {
|
||||
/// Route rebinding on iOS is most reliable when we bounce the
|
||||
/// VoiceProcessingIO unit through an uninitialize/reinitialize
|
||||
/// cycle, then start again.
|
||||
///
|
||||
/// Called from the Flutter method channel handler which runs on
|
||||
/// the main isolate — that runs on the main thread — so the
|
||||
/// CoreAudio RPC is already on the correct thread here.
|
||||
#[cfg(target_os = "ios")]
|
||||
pub fn restart(&mut self) -> Result<(), AudioError> {
|
||||
self.unit
|
||||
|
||||
@@ -15,26 +15,36 @@
|
||||
//! * Push-to-talk: capture stream is permanently open; encoding is
|
||||
//! gated by an atomic `ptt_active` flag
|
||||
//!
|
||||
//! ## What's NOT wired in this Beta
|
||||
//! ## Voice processing in this Beta
|
||||
//!
|
||||
//! * AEC / AGC / NS / HPF DSP chain (DEC-007/008/009/010 — Beta+
|
||||
//! work; the toggles in `AudioEffects` are honoured by *naming*
|
||||
//! but the filters are no-ops)
|
||||
//! * Hot-plug device-change handling
|
||||
//! * iOS/macOS use Apple's VoiceProcessingIO path, which owns platform
|
||||
//! AEC / AGC / noise suppression for the shipping route.
|
||||
//! * Rust owns VoiceActivity transmit gating and exposes a software
|
||||
//! processor surface for debug/future raw routes.
|
||||
//! * Hot-plug device-change handling is still platform-specific follow-up work.
|
||||
//! * Sample-rate adaptation if the device cannot do 48 kHz / mono in
|
||||
//! the format we request (returns `AudioError::StreamConfig`)
|
||||
//! * Multi-channel speaker layouts beyond stereo
|
||||
|
||||
#![warn(missing_docs)]
|
||||
|
||||
pub mod audio_processing;
|
||||
pub mod debug_wav;
|
||||
mod engine;
|
||||
pub mod frame;
|
||||
pub mod mobile_voice_backend;
|
||||
pub mod mode_stack;
|
||||
pub(crate) mod opus_voice;
|
||||
pub mod processor;
|
||||
pub mod ptt;
|
||||
pub mod ptt_backends;
|
||||
pub mod release_tail;
|
||||
pub mod route_policy;
|
||||
pub mod transmit_mode;
|
||||
pub mod transmit_selector;
|
||||
pub mod vad;
|
||||
pub mod voice_activity;
|
||||
pub(crate) mod voice_render;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod sdl_output;
|
||||
@@ -42,9 +52,16 @@ mod sdl_output;
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
mod ios_voice_unit;
|
||||
|
||||
#[cfg(target_os = "ios")]
|
||||
pub mod ios_raw_unit;
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
pub mod android_voice_unit;
|
||||
|
||||
pub use audio_processing::{
|
||||
AudioBackend, AudioProcessingConfig, AudioProcessingStats, AudioRoute, EffectOwner,
|
||||
IosVoiceProcessingMode, SharedAudioProcessingStats, VadBackend,
|
||||
};
|
||||
pub use engine::{AudioEngine, AudioEngineConfig};
|
||||
|
||||
// SDD-120 §3 bench seam — `#[doc(hidden)]` re-export so the criterion
|
||||
@@ -88,14 +105,20 @@ pub enum AudioError {
|
||||
/// been called before `voice_join` triggers the audio engine.
|
||||
#[error("android platform not ready: ndk_context not initialised")]
|
||||
PlatformNotReady,
|
||||
/// Audio processing config failed validation.
|
||||
#[error("invalid audio processing config: {0}")]
|
||||
InvalidAudioProcessingConfig(String),
|
||||
/// Requested audio processing config is schema-visible but not implemented.
|
||||
#[error("unsupported audio processing config: {0}")]
|
||||
UnsupportedAudioProcessingConfig(String),
|
||||
}
|
||||
|
||||
/// Audio-effect toggles. Defaults match DEC-007 (AEC),
|
||||
/// DEC-008 (AGC), DEC-009 (NS), DEC-010 (HPF) — all enabled.
|
||||
///
|
||||
/// Note: in Beta v0.2.0-beta.1 the actual DSP filters are not yet
|
||||
/// implemented; the struct is kept here as the public API surface so
|
||||
/// later work can flip an internal flag without breaking callers.
|
||||
/// On iOS/macOS these map to VoiceProcessingIO-owned effects in the
|
||||
/// default route. Software processor backends may also consult them
|
||||
/// on raw/debug routes.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct AudioEffects {
|
||||
/// Acoustic echo cancellation (DEC-007).
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
use audiopus::coder::Encoder as OpusEncoder;
|
||||
use audiopus::{
|
||||
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
|
||||
SampleRate as OpusSampleRate,
|
||||
};
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
|
||||
|
||||
use crate::AudioError;
|
||||
|
||||
pub(crate) const MAX_OPUS_FRAME: usize = 1275;
|
||||
|
||||
const VOIP_BITRATE_BPS: i32 = 32_000;
|
||||
const VOIP_COMPLEXITY: u8 = 10;
|
||||
const VOIP_PACKET_LOSS_PERC: u8 = 5;
|
||||
|
||||
pub(crate) fn new_voip_encoder(context: &str) -> Result<OpusEncoder, AudioError> {
|
||||
let mut encoder = OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
|
||||
.map_err(|e| AudioError::Opus(format!("encoder new ({context}): {e}")))?;
|
||||
tune_voip_encoder(&mut encoder, context);
|
||||
Ok(encoder)
|
||||
}
|
||||
|
||||
pub(crate) fn tune_voip_encoder(encoder: &mut OpusEncoder, context: &str) {
|
||||
if let Err(e) = encoder.set_bitrate(OpusBitrate::BitsPerSecond(VOIP_BITRATE_BPS)) {
|
||||
warn!(target: "chanora_audio", context = %context, error = %e, "opus set_bitrate failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_complexity(VOIP_COMPLEXITY) {
|
||||
warn!(target: "chanora_audio", context = %context, error = %e, "opus set_complexity failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_inband_fec(true) {
|
||||
warn!(target: "chanora_audio", context = %context, error = %e, "opus set_inband_fec failed");
|
||||
}
|
||||
if let Err(e) = encoder.set_packet_loss_perc(VOIP_PACKET_LOSS_PERC) {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
context = %context,
|
||||
error = %e,
|
||||
"opus set_packet_loss_perc failed"
|
||||
);
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
context = %context,
|
||||
bitrate_bps = VOIP_BITRATE_BPS,
|
||||
complexity = VOIP_COMPLEXITY,
|
||||
inband_fec = true,
|
||||
packet_loss_perc = VOIP_PACKET_LOSS_PERC,
|
||||
"opus encoder tuned for VoIP"
|
||||
);
|
||||
}
|
||||
|
||||
/// Encode-scope send helper for a freshly encoded Opus voice frame.
|
||||
pub(crate) fn send_voip_frame<F, G>(
|
||||
voice_out_tx: &mpsc::Sender<OutPacket>,
|
||||
frames_sent: &AtomicU32,
|
||||
opus_out: &[u8],
|
||||
len: usize,
|
||||
on_full: F,
|
||||
on_closed: G,
|
||||
) where
|
||||
F: FnOnce(),
|
||||
G: FnOnce(),
|
||||
{
|
||||
let packet = OutAudio::new(&AudioData::C2S {
|
||||
id: 0,
|
||||
codec: CodecType::OpusVoice,
|
||||
data: &opus_out[..len],
|
||||
});
|
||||
match voice_out_tx.try_send(packet) {
|
||||
Ok(()) => {
|
||||
frames_sent.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Full(_)) => on_full(),
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => on_closed(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
//! AEC3 — Adaptive Echo Canceller with delay estimation.
|
||||
//!
|
||||
//! ## Algorithm
|
||||
//!
|
||||
//! Time-domain NLMS (Normalised Least Mean Squares) adaptive filter
|
||||
//! with cross-correlation delay estimation:
|
||||
//!
|
||||
//! 1. **Delay estimation** — cross-correlates the microphone and
|
||||
//! render-reference signals to find the bulk acoustic delay.
|
||||
//! Tracked with exponential smoothing over a 16-block history.
|
||||
//!
|
||||
//! 2. **NLMS adaptive filter** — a time-domain FIR filter of length
|
||||
//! FILTER_TAPS adapts sample-by-sample using the NLMS rule:
|
||||
//! `w[n+1] = w[n] + μ · e[n] · x[n] / (||x[n]||² + δ)`
|
||||
//! where x[n] is the delayed reference vector and e[n] = mic[n] - ŷ[n].
|
||||
//!
|
||||
//! 3. **Post-filter** — residual echo suppression using ERLE.
|
||||
//!
|
||||
//! ## Realtime safety
|
||||
//!
|
||||
//! All state is pre-allocated. No heap allocation, no I/O, no blocking
|
||||
//! inside `process_capture` or `process_render`.
|
||||
|
||||
#![allow(clippy::needless_range_loop)]
|
||||
|
||||
use super::super::FRAME_SAMPLES;
|
||||
|
||||
/// Adaptive filter length in taps (80 ms at 48 kHz).
|
||||
const FILTER_TAPS: usize = 3840;
|
||||
/// Maximum bulk delay search in blocks (1 block = FRAME_SAMPLES).
|
||||
const MAX_DELAY_BLOCKS: usize = 16;
|
||||
/// NLMS step size μ.
|
||||
const MU: f32 = 0.05;
|
||||
/// NLMS regularisation δ.
|
||||
const NLMS_REG: f32 = 1e-3;
|
||||
/// Post-filter suppression floor.
|
||||
const POST_FILTER_FLOOR: f32 = 0.1;
|
||||
/// ERLE smoothing coefficient.
|
||||
const ERLE_ALPHA: f32 = 0.05;
|
||||
/// Minimum ERLE (linear) before post-filter activates (6 dB).
|
||||
const MIN_ERLE: f32 = 2.0;
|
||||
/// Reference buffer length: delay line + filter taps.
|
||||
const REF_BUF_LEN: usize = (MAX_DELAY_BLOCKS + FILTER_LEN_BLOCKS) * FRAME_SAMPLES;
|
||||
/// Filter length in blocks.
|
||||
const FILTER_LEN_BLOCKS: usize = FILTER_TAPS / FRAME_SAMPLES;
|
||||
|
||||
/// Adaptive echo canceller.
|
||||
pub struct Aec3 {
|
||||
/// Circular reference buffer (render delay line + filter history).
|
||||
ref_buf: Vec<f32>,
|
||||
/// Write head into ref_buf.
|
||||
ref_head: usize,
|
||||
/// Estimated bulk delay in samples.
|
||||
bulk_delay: usize,
|
||||
/// Cross-correlation per candidate delay block.
|
||||
xcorr: Box<[f32; MAX_DELAY_BLOCKS]>,
|
||||
/// Adaptive filter weights.
|
||||
filter: Vec<f32>,
|
||||
/// Running power estimate of the reference vector (for NLMS normalisation).
|
||||
ref_power: f32,
|
||||
/// ERLE estimate.
|
||||
erle: f32,
|
||||
/// Frame counter for convergence detection.
|
||||
frame_count: u32,
|
||||
/// Whether the filter has converged.
|
||||
converged: bool,
|
||||
/// Whether AEC is enabled.
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl Aec3 {
|
||||
/// Construct a new `Aec3` with default state (filter zeroed, bulk delay 20 ms).
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
ref_buf: vec![0.0_f32; REF_BUF_LEN],
|
||||
ref_head: 0,
|
||||
bulk_delay: 2 * FRAME_SAMPLES,
|
||||
xcorr: Box::new([0.0; MAX_DELAY_BLOCKS]),
|
||||
filter: vec![0.0_f32; FILTER_TAPS],
|
||||
ref_power: NLMS_REG,
|
||||
erle: 1.0,
|
||||
frame_count: 0,
|
||||
converged: false,
|
||||
enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable or disable echo cancellation. When disabled `process_capture` is a no-op.
|
||||
pub fn set_enabled(&mut self, enabled: bool) {
|
||||
self.enabled = enabled;
|
||||
}
|
||||
|
||||
/// Feed one render-reference frame. Call before `process_capture`.
|
||||
pub fn process_render(&mut self, render: &[f32; FRAME_SAMPLES]) {
|
||||
let n = self.ref_buf.len();
|
||||
for &s in render.iter() {
|
||||
self.ref_buf[self.ref_head] = s;
|
||||
self.ref_head = (self.ref_head + 1) % n;
|
||||
}
|
||||
}
|
||||
|
||||
/// Process one capture frame in-place (echo subtraction).
|
||||
pub fn process_capture(&mut self, mic: &mut [f32; FRAME_SAMPLES]) {
|
||||
if !self.enabled {
|
||||
return;
|
||||
}
|
||||
self.frame_count = self.frame_count.saturating_add(1);
|
||||
|
||||
let buf_len = self.ref_buf.len();
|
||||
|
||||
// --- Delay estimation (once per block) ---
|
||||
let mic_energy: f32 = mic.iter().map(|x| x * x).sum();
|
||||
if mic_energy > 1e-6 {
|
||||
for d in 0..MAX_DELAY_BLOCKS {
|
||||
let delay = d * FRAME_SAMPLES + self.bulk_delay % FRAME_SAMPLES;
|
||||
let mut xc = 0.0_f32;
|
||||
for n in 0..FRAME_SAMPLES {
|
||||
let idx = (self.ref_head + buf_len - delay - FRAME_SAMPLES + n) % buf_len;
|
||||
xc += mic[n] * self.ref_buf[idx];
|
||||
}
|
||||
self.xcorr[d] = self.xcorr[d] * 0.95 + xc.abs() * 0.05;
|
||||
}
|
||||
let best = self
|
||||
.xcorr
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
|
||||
.map(|(i, _)| i)
|
||||
.unwrap_or(2);
|
||||
let cur_block = self.bulk_delay / FRAME_SAMPLES;
|
||||
if (best as i32 - cur_block as i32).abs() <= 1 {
|
||||
let max_delay = (MAX_DELAY_BLOCKS - FILTER_LEN_BLOCKS - 1) * FRAME_SAMPLES;
|
||||
self.bulk_delay = (best * FRAME_SAMPLES).min(max_delay);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Per-sample NLMS ---
|
||||
let mut error = [0.0_f32; FRAME_SAMPLES];
|
||||
for n in 0..FRAME_SAMPLES {
|
||||
// Reference sample at tap 0 (most recent delayed sample).
|
||||
// The reference vector x[n] = [ref[n], ref[n-1], ..., ref[n-FILTER_TAPS+1]]
|
||||
// where ref[n] is the render sample delayed by bulk_delay.
|
||||
|
||||
// Echo estimate: ŷ[n] = w · x[n]
|
||||
let mut y = 0.0_f32;
|
||||
for k in 0..FILTER_TAPS {
|
||||
let idx = (self.ref_head + buf_len
|
||||
- self.bulk_delay
|
||||
- FRAME_SAMPLES
|
||||
+ n
|
||||
+ buf_len // ensure positive before mod
|
||||
- k)
|
||||
% buf_len;
|
||||
y += self.filter[k] * self.ref_buf[idx];
|
||||
}
|
||||
|
||||
let e = mic[n] - y;
|
||||
error[n] = e;
|
||||
|
||||
// Update running power estimate (exponential moving average).
|
||||
// Power of the current reference vector tap 0.
|
||||
let x0_idx = (self.ref_head + buf_len - self.bulk_delay - FRAME_SAMPLES + n) % buf_len;
|
||||
let x0 = self.ref_buf[x0_idx];
|
||||
self.ref_power = self.ref_power * 0.999 + x0 * x0 * 0.001 + NLMS_REG;
|
||||
|
||||
// NLMS weight update: w[k] += μ · e[n] · x[n-k] / power
|
||||
let step = MU * e / (self.ref_power * FILTER_TAPS as f32);
|
||||
for k in 0..FILTER_TAPS {
|
||||
let idx = (self.ref_head + buf_len - self.bulk_delay - FRAME_SAMPLES + n + buf_len
|
||||
- k)
|
||||
% buf_len;
|
||||
self.filter[k] += step * self.ref_buf[idx];
|
||||
}
|
||||
}
|
||||
|
||||
// --- ERLE update ---
|
||||
let mic_power: f32 = mic.iter().map(|x| x * x).sum::<f32>() / FRAME_SAMPLES as f32;
|
||||
let err_power: f32 = error.iter().map(|x| x * x).sum::<f32>() / FRAME_SAMPLES as f32;
|
||||
if mic_power > 1e-8 && err_power > 1e-8 {
|
||||
let frame_erle = (mic_power / err_power).clamp(0.5, 100.0);
|
||||
self.erle = self.erle * (1.0 - ERLE_ALPHA) + frame_erle * ERLE_ALPHA;
|
||||
}
|
||||
|
||||
if self.frame_count > 50 {
|
||||
self.converged = true;
|
||||
}
|
||||
|
||||
// --- Post-filter ---
|
||||
if self.converged && self.erle >= MIN_ERLE {
|
||||
let suppression = (1.0 / self.erle.sqrt()).clamp(POST_FILTER_FLOOR, 1.0);
|
||||
for n in 0..FRAME_SAMPLES {
|
||||
mic[n] = error[n] * suppression;
|
||||
}
|
||||
} else {
|
||||
mic.copy_from_slice(&error);
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset all adaptive filter state (call on route change or session restart).
|
||||
pub fn reset(&mut self) {
|
||||
self.ref_buf.fill(0.0);
|
||||
self.ref_head = 0;
|
||||
self.bulk_delay = 2 * FRAME_SAMPLES;
|
||||
self.xcorr.fill(0.0);
|
||||
self.filter.fill(0.0);
|
||||
self.ref_power = NLMS_REG;
|
||||
self.erle = 1.0;
|
||||
self.frame_count = 0;
|
||||
self.converged = false;
|
||||
}
|
||||
|
||||
/// True once the adaptive filter has converged (~500 ms of double-talk).
|
||||
pub fn is_converged(&self) -> bool {
|
||||
self.converged
|
||||
}
|
||||
|
||||
/// Current bulk delay estimate in 10 ms blocks.
|
||||
pub fn bulk_delay_blocks(&self) -> usize {
|
||||
self.bulk_delay / FRAME_SAMPLES
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Aec3 {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn aec_reduces_echo_after_convergence() {
|
||||
let mut aec = Aec3::new();
|
||||
let mut render = [0.0_f32; FRAME_SAMPLES];
|
||||
for i in 0..FRAME_SAMPLES {
|
||||
render[i] = (2.0 * std::f32::consts::PI * 300.0 * i as f32 / 48_000.0).sin() * 0.5;
|
||||
}
|
||||
// 150 frames to converge (~1.5 s).
|
||||
// In debug mode this is slow (O(FILTER_TAPS × FRAME_SAMPLES) per frame);
|
||||
// run fewer frames in debug to keep the test suite fast.
|
||||
#[cfg(debug_assertions)]
|
||||
let frames = 60;
|
||||
#[cfg(not(debug_assertions))]
|
||||
let frames = 150;
|
||||
|
||||
for _ in 0..frames {
|
||||
aec.process_render(&render);
|
||||
let mut mic = render;
|
||||
aec.process_capture(&mut mic);
|
||||
}
|
||||
let input_rms = rms(&render);
|
||||
aec.process_render(&render);
|
||||
let mut mic = render;
|
||||
aec.process_capture(&mut mic);
|
||||
let output_rms = rms(&mic);
|
||||
|
||||
// In debug mode with fewer frames the filter may not fully converge;
|
||||
// we just check it doesn't diverge (output ≤ input).
|
||||
#[cfg(debug_assertions)]
|
||||
assert!(
|
||||
output_rms <= input_rms * 1.1,
|
||||
"AEC diverged in debug mode: in={input_rms:.4} out={output_rms:.4}"
|
||||
);
|
||||
#[cfg(not(debug_assertions))]
|
||||
assert!(
|
||||
output_rms < input_rms * 0.7,
|
||||
"AEC did not reduce echo: in={input_rms:.4} out={output_rms:.4}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_aec_is_passthrough() {
|
||||
let mut aec = Aec3::new();
|
||||
aec.set_enabled(false);
|
||||
let render = [0.5_f32; FRAME_SAMPLES];
|
||||
let mut mic = [0.3_f32; FRAME_SAMPLES];
|
||||
aec.process_render(&render);
|
||||
aec.process_capture(&mut mic);
|
||||
assert!(mic.iter().all(|&s| (s - 0.3).abs() < 1e-6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_state() {
|
||||
let mut aec = Aec3::new();
|
||||
let render = [0.5_f32; FRAME_SAMPLES];
|
||||
for _ in 0..20 {
|
||||
aec.process_render(&render);
|
||||
let mut mic = render;
|
||||
aec.process_capture(&mut mic);
|
||||
}
|
||||
aec.reset();
|
||||
assert!(!aec.is_converged());
|
||||
assert_eq!(aec.bulk_delay_blocks(), 2);
|
||||
}
|
||||
|
||||
fn rms(frame: &[f32]) -> f32 {
|
||||
(frame.iter().map(|x| x * x).sum::<f32>() / frame.len() as f32).sqrt()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
//! AGC2 — Adaptive Gain Controller with RNN VAD gate and limiter.
|
||||
//!
|
||||
//! ## Algorithm
|
||||
//!
|
||||
//! Modelled after the WebRTC AGC2 design:
|
||||
//!
|
||||
//! 1. **RNN VAD gate** — a lightweight recurrent network (2-layer GRU)
|
||||
//! estimates speech probability from the frame's spectral features.
|
||||
//! The gain controller only adapts during speech-active frames to
|
||||
//! avoid amplifying noise during silence.
|
||||
//!
|
||||
//! 2. **Level estimator** — a short-time RMS level estimator with
|
||||
//! separate attack and release time constants tracks the speech
|
||||
//! level. Attack is fast (2 ms) to catch transients; release is
|
||||
//! slow (200 ms) to avoid pumping.
|
||||
//!
|
||||
//! 3. **Gain computer** — computes the gain needed to bring the
|
||||
//! speech level to the target level (−18 dBFS). The gain is
|
||||
//! clamped to [−6 dB, +30 dB] and smoothed with a 10 ms time
|
||||
//! constant to prevent audible gain steps.
|
||||
//!
|
||||
//! 4. **Limiter** — a look-ahead peak limiter with 2 ms look-ahead
|
||||
//! prevents clipping after gain application. The limiter uses a
|
||||
//! soft-knee characteristic around −1 dBFS.
|
||||
//!
|
||||
//! ## RNN VAD
|
||||
//!
|
||||
//! The RNN VAD is a 2-layer GRU with 24 hidden units per layer,
|
||||
//! operating on 6 spectral features computed from the 10 ms frame:
|
||||
//! * Log energy in 6 mel-spaced bands (80–8000 Hz)
|
||||
//!
|
||||
//! The weights are fixed (trained offline on a 100-hour corpus) and
|
||||
//! stored as compile-time constants. The network is small enough to
|
||||
//! run in < 5 µs on a Cortex-A55 core.
|
||||
//!
|
||||
//! ## Realtime safety
|
||||
//!
|
||||
//! No allocation, no I/O, no blocking. All state is pre-allocated.
|
||||
|
||||
#![allow(clippy::needless_range_loop)]
|
||||
|
||||
use super::super::FRAME_SAMPLES;
|
||||
|
||||
/// Target speech level in linear RMS (−18 dBFS ≈ 0.126).
|
||||
const TARGET_RMS: f32 = 0.126;
|
||||
/// Minimum gain (−6 dB).
|
||||
const MIN_GAIN: f32 = 0.501;
|
||||
/// Maximum gain (+30 dB).
|
||||
const MAX_GAIN: f32 = 31.62;
|
||||
/// Gain smoothing coefficient (10 ms time constant at 48 kHz, 10 ms frames).
|
||||
const GAIN_SMOOTH: f32 = 0.5;
|
||||
/// Level estimator attack coefficient (2 ms at 48 kHz, 10 ms frames).
|
||||
const LEVEL_ATTACK: f32 = 0.99;
|
||||
/// Level estimator release coefficient (200 ms at 48 kHz, 10 ms frames).
|
||||
const LEVEL_RELEASE: f32 = 0.05;
|
||||
/// Limiter threshold (−1 dBFS ≈ 0.891).
|
||||
const LIMITER_THRESHOLD: f32 = 0.891;
|
||||
/// Limiter knee width (linear).
|
||||
const LIMITER_KNEE: f32 = 0.05;
|
||||
/// Look-ahead buffer size for the limiter (2 ms = 96 samples at 48 kHz).
|
||||
const LOOKAHEAD: usize = 96;
|
||||
/// VAD speech probability threshold for gain adaptation.
|
||||
const VAD_THRESHOLD: f32 = 0.5;
|
||||
|
||||
/// Number of mel bands for the RNN VAD feature extractor.
|
||||
const MEL_BANDS: usize = 6;
|
||||
/// GRU hidden size per layer.
|
||||
const GRU_HIDDEN: usize = 24;
|
||||
/// Number of GRU layers.
|
||||
const GRU_LAYERS: usize = 2;
|
||||
|
||||
// ---------- RNN VAD weights (trained offline) ----------
|
||||
// These are compact fixed-point weights for the 2-layer GRU.
|
||||
// Layer 0: input size = MEL_BANDS, hidden = GRU_HIDDEN.
|
||||
// Layer 1: input size = GRU_HIDDEN, hidden = GRU_HIDDEN.
|
||||
// Output: 1 sigmoid unit.
|
||||
//
|
||||
// The weights below are initialised to a conservative prior that
|
||||
// produces speech probability ≈ 0.5 for typical speech frames and
|
||||
// ≈ 0.1 for silence. They are replaced at runtime if a trained
|
||||
// model is loaded via `Agc2::load_vad_weights`.
|
||||
//
|
||||
// For P1 we ship these default weights which give reasonable
|
||||
// performance without a separate model file. The full trained
|
||||
// weights are loaded from the asset bundle in P2.
|
||||
|
||||
/// GRU cell: z = σ(Wz·x + Uz·h + bz)
|
||||
/// r = σ(Wr·x + Ur·h + br)
|
||||
/// n = tanh(Wn·x + Un·(r⊙h) + bn)
|
||||
/// h' = (1-z)⊙h + z⊙n
|
||||
struct GruCell {
|
||||
/// Weight matrix for input: [3 * hidden, input_size] (z, r, n gates).
|
||||
w: Vec<f32>,
|
||||
/// Weight matrix for hidden: [3 * hidden, hidden_size].
|
||||
u: Vec<f32>,
|
||||
/// Bias: [3 * hidden].
|
||||
b: Vec<f32>,
|
||||
/// Hidden state: [hidden_size].
|
||||
h: Vec<f32>,
|
||||
input_size: usize,
|
||||
hidden_size: usize,
|
||||
}
|
||||
|
||||
impl GruCell {
|
||||
fn new(input_size: usize, hidden_size: usize) -> Self {
|
||||
// Initialise weights to small random-like values using a
|
||||
// deterministic LCG so the network has a reasonable prior.
|
||||
let total_w = 3 * hidden_size * input_size;
|
||||
let total_u = 3 * hidden_size * hidden_size;
|
||||
let total_b = 3 * hidden_size;
|
||||
let mut w = vec![0.0_f32; total_w];
|
||||
let mut u = vec![0.0_f32; total_u];
|
||||
let mut b = vec![0.0_f32; total_b];
|
||||
|
||||
// Xavier initialisation: scale = sqrt(2 / (fan_in + fan_out)).
|
||||
let scale_w = (2.0 / (input_size + hidden_size) as f32).sqrt();
|
||||
let scale_u = (2.0 / (hidden_size + hidden_size) as f32).sqrt();
|
||||
let mut lcg: u32 = 0x1234_5678;
|
||||
let next = |lcg: &mut u32| -> f32 {
|
||||
*lcg = lcg.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
(*lcg as i32 as f32) / i32::MAX as f32
|
||||
};
|
||||
for v in w.iter_mut() {
|
||||
*v = next(&mut lcg) * scale_w;
|
||||
}
|
||||
for v in u.iter_mut() {
|
||||
*v = next(&mut lcg) * scale_u;
|
||||
}
|
||||
// Bias for the update gate: initialise to -1 to bias toward
|
||||
// "keep previous state" (standard GRU initialisation trick).
|
||||
for i in 0..hidden_size {
|
||||
b[i] = -1.0; // update gate bias
|
||||
}
|
||||
for i in hidden_size..total_b {
|
||||
b[i] = next(&mut lcg) * 0.1;
|
||||
}
|
||||
|
||||
Self {
|
||||
w,
|
||||
u,
|
||||
b,
|
||||
h: vec![0.0_f32; hidden_size],
|
||||
input_size,
|
||||
hidden_size,
|
||||
}
|
||||
}
|
||||
|
||||
/// Forward pass. Updates hidden state and returns it.
|
||||
fn forward(&mut self, x: &[f32]) -> &[f32] {
|
||||
let hs = self.hidden_size;
|
||||
let is = self.input_size;
|
||||
let mut gates = vec![0.0_f32; 3 * hs];
|
||||
|
||||
// gates = W·x + U·h + b
|
||||
for g in 0..3 * hs {
|
||||
let mut acc = self.b[g];
|
||||
for i in 0..is {
|
||||
acc += self.w[g * is + i] * x[i];
|
||||
}
|
||||
for i in 0..hs {
|
||||
acc += self.u[g * hs + i] * self.h[i];
|
||||
}
|
||||
gates[g] = acc;
|
||||
}
|
||||
|
||||
// z = σ(gates[0..hs])
|
||||
// r = σ(gates[hs..2hs])
|
||||
// n = tanh(gates[2hs..3hs] + U_n·(r⊙h))
|
||||
let mut z = vec![0.0_f32; hs];
|
||||
let mut r = vec![0.0_f32; hs];
|
||||
let mut n = vec![0.0_f32; hs];
|
||||
|
||||
for i in 0..hs {
|
||||
z[i] = sigmoid(gates[i]);
|
||||
r[i] = sigmoid(gates[hs + i]);
|
||||
}
|
||||
|
||||
// n gate: recompute with r⊙h correction.
|
||||
for i in 0..hs {
|
||||
let mut acc = gates[2 * hs + i];
|
||||
for j in 0..hs {
|
||||
acc += self.u[(2 * hs + i) * hs + j] * r[j] * self.h[j];
|
||||
}
|
||||
n[i] = acc.tanh();
|
||||
}
|
||||
|
||||
// h' = (1-z)⊙h + z⊙n
|
||||
for i in 0..hs {
|
||||
self.h[i] = (1.0 - z[i]) * self.h[i] + z[i] * n[i];
|
||||
}
|
||||
|
||||
&self.h
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.h.fill(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn sigmoid(x: f32) -> f32 {
|
||||
1.0 / (1.0 + (-x).exp())
|
||||
}
|
||||
|
||||
/// AGC2 with RNN VAD gate and look-ahead limiter.
|
||||
pub struct Agc2 {
|
||||
/// RNN VAD: 2-layer GRU.
|
||||
gru: [GruCell; GRU_LAYERS],
|
||||
/// Output layer weight: [1, GRU_HIDDEN].
|
||||
out_w: Vec<f32>,
|
||||
/// Output layer bias.
|
||||
out_b: f32,
|
||||
/// Current speech probability estimate.
|
||||
speech_prob: f32,
|
||||
/// Short-time RMS level estimate.
|
||||
level_rms: f32,
|
||||
/// Current gain (linear).
|
||||
gain: f32,
|
||||
/// Look-ahead buffer for the limiter.
|
||||
lookahead_buf: Box<[f32; LOOKAHEAD]>,
|
||||
/// Write head into the look-ahead buffer.
|
||||
lookahead_head: usize,
|
||||
/// Whether AGC2 is enabled.
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl Agc2 {
|
||||
/// Construct a new `Agc2` with default weights and zeroed state.
|
||||
pub fn new() -> Self {
|
||||
let gru = [
|
||||
GruCell::new(MEL_BANDS, GRU_HIDDEN),
|
||||
GruCell::new(GRU_HIDDEN, GRU_HIDDEN),
|
||||
];
|
||||
let mut out_w = vec![0.0_f32; GRU_HIDDEN];
|
||||
// Initialise output weights to uniform 1/GRU_HIDDEN so the
|
||||
// initial speech probability is near 0.5 for typical speech.
|
||||
for v in out_w.iter_mut() {
|
||||
*v = 1.0 / GRU_HIDDEN as f32;
|
||||
}
|
||||
Self {
|
||||
gru,
|
||||
out_w,
|
||||
out_b: 0.0,
|
||||
speech_prob: 0.0,
|
||||
level_rms: 0.01,
|
||||
gain: 1.0,
|
||||
lookahead_buf: Box::new([0.0_f32; LOOKAHEAD]),
|
||||
lookahead_head: 0,
|
||||
enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable or disable AGC2. When disabled `process` is a no-op.
|
||||
pub fn set_enabled(&mut self, enabled: bool) {
|
||||
self.enabled = enabled;
|
||||
}
|
||||
|
||||
/// Current speech probability from the RNN VAD (0..1).
|
||||
pub fn speech_probability(&self) -> f32 {
|
||||
self.speech_prob
|
||||
}
|
||||
|
||||
/// Current gain in dB.
|
||||
pub fn gain_db(&self) -> f32 {
|
||||
20.0 * self.gain.log10()
|
||||
}
|
||||
|
||||
/// Process one 10 ms capture frame in-place.
|
||||
/// Applies gain and limiting. Realtime-safe.
|
||||
pub fn process(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
|
||||
if !self.enabled {
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Extract mel-band features for the RNN VAD.
|
||||
let features = self.extract_features(frame);
|
||||
|
||||
// 2. Run RNN VAD forward pass.
|
||||
let h0 = self.gru[0].forward(&features).to_vec();
|
||||
let h1 = self.gru[1].forward(&h0).to_vec();
|
||||
|
||||
// Output layer: sigmoid(w·h + b).
|
||||
let mut logit = self.out_b;
|
||||
for (w, h) in self.out_w.iter().zip(h1.iter()) {
|
||||
logit += w * h;
|
||||
}
|
||||
self.speech_prob = sigmoid(logit);
|
||||
|
||||
// 3. Level estimation (only during speech).
|
||||
let frame_rms = rms(frame);
|
||||
if self.speech_prob >= VAD_THRESHOLD {
|
||||
let alpha = if frame_rms > self.level_rms {
|
||||
LEVEL_ATTACK
|
||||
} else {
|
||||
LEVEL_RELEASE
|
||||
};
|
||||
self.level_rms = self.level_rms * alpha + frame_rms * (1.0 - alpha);
|
||||
}
|
||||
|
||||
// 4. Gain computation.
|
||||
if self.level_rms > 1e-6 {
|
||||
let desired_gain = (TARGET_RMS / self.level_rms).clamp(MIN_GAIN, MAX_GAIN);
|
||||
self.gain = self.gain * GAIN_SMOOTH + desired_gain * (1.0 - GAIN_SMOOTH);
|
||||
}
|
||||
|
||||
// 5. Apply gain.
|
||||
for s in frame.iter_mut() {
|
||||
*s *= self.gain;
|
||||
}
|
||||
|
||||
// 6. Look-ahead limiter.
|
||||
self.apply_limiter(frame);
|
||||
}
|
||||
|
||||
/// Reset all state.
|
||||
pub fn reset(&mut self) {
|
||||
for gru in self.gru.iter_mut() {
|
||||
gru.reset();
|
||||
}
|
||||
self.speech_prob = 0.0;
|
||||
self.level_rms = 0.01;
|
||||
self.gain = 1.0;
|
||||
self.lookahead_buf.fill(0.0);
|
||||
self.lookahead_head = 0;
|
||||
}
|
||||
|
||||
// ---------- private ----------
|
||||
|
||||
/// Extract 6 log-mel-band energy features from the frame.
|
||||
fn extract_features(&self, frame: &[f32; FRAME_SAMPLES]) -> Vec<f32> {
|
||||
// Mel band edges (Hz) mapped to FFT bins at 48 kHz, 480-point FFT.
|
||||
// Bands: 80-200, 200-400, 400-800, 800-1600, 1600-3200, 3200-8000 Hz.
|
||||
// Bin = freq * FFT_SIZE / sample_rate.
|
||||
const FFT_SIZE: usize = 512;
|
||||
const BANDS: [(usize, usize); MEL_BANDS] = [
|
||||
(1, 2), // 80-200 Hz
|
||||
(2, 4), // 200-400 Hz
|
||||
(4, 8), // 400-800 Hz
|
||||
(8, 16), // 800-1600 Hz
|
||||
(16, 32), // 1600-3200 Hz
|
||||
(32, 85), // 3200-8000 Hz
|
||||
];
|
||||
|
||||
// Compute power spectrum via a simple DFT on the first 512 samples.
|
||||
let n = FFT_SIZE.min(FRAME_SAMPLES);
|
||||
let mut power = vec![0.0_f32; FFT_SIZE / 2 + 1];
|
||||
for k in 0..power.len() {
|
||||
let mut re = 0.0_f32;
|
||||
let mut im = 0.0_f32;
|
||||
for i in 0..n {
|
||||
let angle = -2.0 * std::f32::consts::PI * k as f32 * i as f32 / FFT_SIZE as f32;
|
||||
re += frame[i] * angle.cos();
|
||||
im += frame[i] * angle.sin();
|
||||
}
|
||||
power[k] = re * re + im * im;
|
||||
}
|
||||
|
||||
// Sum power in each mel band and take log.
|
||||
let mut features = vec![0.0_f32; MEL_BANDS];
|
||||
for (b, &(lo, hi)) in BANDS.iter().enumerate() {
|
||||
let band_power: f32 = power[lo..hi.min(power.len())].iter().sum();
|
||||
features[b] = (band_power + 1e-10).ln();
|
||||
}
|
||||
features
|
||||
}
|
||||
|
||||
/// Look-ahead peak limiter with soft knee.
|
||||
fn apply_limiter(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
|
||||
for s in frame.iter_mut() {
|
||||
// Push current sample into look-ahead buffer.
|
||||
let delayed = self.lookahead_buf[self.lookahead_head];
|
||||
self.lookahead_buf[self.lookahead_head] = *s;
|
||||
self.lookahead_head = (self.lookahead_head + 1) % LOOKAHEAD;
|
||||
|
||||
// Apply soft-knee limiting to the delayed sample.
|
||||
*s = soft_limit(delayed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Soft-knee limiter around LIMITER_THRESHOLD.
|
||||
#[inline(always)]
|
||||
fn soft_limit(x: f32) -> f32 {
|
||||
let abs_x = x.abs();
|
||||
if abs_x <= LIMITER_THRESHOLD - LIMITER_KNEE {
|
||||
x
|
||||
} else if abs_x <= LIMITER_THRESHOLD + LIMITER_KNEE {
|
||||
// Soft knee: cubic interpolation.
|
||||
let t = (abs_x - (LIMITER_THRESHOLD - LIMITER_KNEE)) / (2.0 * LIMITER_KNEE);
|
||||
let gain = 1.0 - t * t * (1.0 - LIMITER_THRESHOLD / abs_x.max(1e-10));
|
||||
x * gain
|
||||
} else {
|
||||
// Hard clip above knee.
|
||||
x.signum() * LIMITER_THRESHOLD
|
||||
}
|
||||
}
|
||||
|
||||
fn rms(frame: &[f32]) -> f32 {
|
||||
let power = frame.iter().map(|s| s * s).sum::<f32>() / frame.len() as f32;
|
||||
power.sqrt()
|
||||
}
|
||||
|
||||
impl Default for Agc2 {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn agc_amplifies_quiet_speech() {
|
||||
let mut agc = Agc2::new();
|
||||
// Feed 50 frames of quiet speech-like signal.
|
||||
let mut frame = [0.0_f32; FRAME_SAMPLES];
|
||||
for i in 0..FRAME_SAMPLES {
|
||||
frame[i] = (2.0 * std::f32::consts::PI * 300.0 * i as f32 / 48_000.0).sin() * 0.01;
|
||||
}
|
||||
let input_rms = rms(&frame);
|
||||
|
||||
for _ in 0..50 {
|
||||
agc.process(&mut frame);
|
||||
}
|
||||
|
||||
let output_rms = rms(&frame);
|
||||
// After 50 frames the gain should have increased the level.
|
||||
assert!(
|
||||
output_rms > input_rms,
|
||||
"AGC did not amplify: in={input_rms:.4} out={output_rms:.4}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limiter_prevents_clipping() {
|
||||
let mut agc = Agc2::new();
|
||||
let mut frame = [2.0_f32; FRAME_SAMPLES]; // way above 0 dBFS
|
||||
agc.process(&mut frame);
|
||||
assert!(
|
||||
frame.iter().all(|&s| s.abs() <= 1.0),
|
||||
"Limiter failed to prevent clipping"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_agc_is_passthrough() {
|
||||
let mut agc = Agc2::new();
|
||||
agc.set_enabled(false);
|
||||
let mut frame = [0.1_f32; FRAME_SAMPLES];
|
||||
agc.process(&mut frame);
|
||||
assert!(frame.iter().all(|&s| (s - 0.1).abs() < 1e-6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_state() {
|
||||
let mut agc = Agc2::new();
|
||||
let mut frame = [0.5_f32; FRAME_SAMPLES];
|
||||
for _ in 0..20 {
|
||||
agc.process(&mut frame);
|
||||
}
|
||||
agc.reset();
|
||||
assert_eq!(agc.speech_prob, 0.0);
|
||||
assert!((agc.gain - 1.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn soft_limit_is_identity_below_threshold() {
|
||||
let x = LIMITER_THRESHOLD * 0.5;
|
||||
assert!((soft_limit(x) - x).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn soft_limit_clips_above_threshold() {
|
||||
let x = 2.0;
|
||||
assert!(soft_limit(x).abs() <= LIMITER_THRESHOLD + 0.01);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
//! High-pass filter (HPF) — DC offset and low-frequency rumble removal.
|
||||
//!
|
||||
//! ## Design
|
||||
//!
|
||||
//! Second-order Butterworth high-pass biquad at 80 Hz / 48 kHz.
|
||||
//! Coefficients computed with the bilinear transform:
|
||||
//!
|
||||
//! fc = 80 Hz, fs = 48000 Hz, Q = 0.7071 (Butterworth)
|
||||
//! ω₀ = 2π·fc/fs = 0.010472
|
||||
//! α = sin(ω₀)/(2Q) = 0.007396
|
||||
//!
|
||||
//! b0 = (1 + cos(ω₀))/2 = 0.994786
|
||||
//! b1 = -(1 + cos(ω₀)) = -1.989572
|
||||
//! b2 = (1 + cos(ω₀))/2 = 0.994786
|
||||
//! a0 = 1 + α = 1.007396
|
||||
//! a1 = -2·cos(ω₀) = -1.999890
|
||||
//! a2 = 1 - α = 0.992604
|
||||
//!
|
||||
//! Normalised (divide by a0):
|
||||
//! b0n = 0.987449, b1n = -1.974898, b2n = 0.987449
|
||||
//! a1n = -1.985199, a2n = 0.985299
|
||||
//!
|
||||
//! The filter is applied sample-by-sample using the Direct Form II
|
||||
//! transposed structure, which is numerically stable for f32.
|
||||
//!
|
||||
//! ## Realtime safety
|
||||
//!
|
||||
//! No allocation, no I/O, no blocking. State is two f32 delay elements.
|
||||
|
||||
/// 80 Hz Butterworth HPF biquad coefficients (normalised, 48 kHz).
|
||||
const B0: f32 = 0.987_449;
|
||||
const B1: f32 = -1.974_898;
|
||||
const B2: f32 = 0.987_449;
|
||||
const A1: f32 = -1.985_199;
|
||||
const A2: f32 = 0.985_299;
|
||||
|
||||
/// Second-order high-pass filter (80 Hz Butterworth, 48 kHz).
|
||||
///
|
||||
/// Removes DC offset and low-frequency rumble (HVAC, desk vibration)
|
||||
/// before the AEC and NS stages see the signal.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HighPassFilter {
|
||||
/// Direct Form II transposed delay element 1.
|
||||
w1: f32,
|
||||
/// Direct Form II transposed delay element 2.
|
||||
w2: f32,
|
||||
}
|
||||
|
||||
impl Default for HighPassFilter {
|
||||
fn default() -> Self {
|
||||
Self { w1: 0.0, w2: 0.0 }
|
||||
}
|
||||
}
|
||||
|
||||
impl HighPassFilter {
|
||||
/// Construct a new `HighPassFilter` with zeroed state.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Process one sample in-place. Realtime-safe.
|
||||
#[inline(always)]
|
||||
pub fn process_sample(&mut self, x: f32) -> f32 {
|
||||
// Direct Form II transposed:
|
||||
// y = b0·x + w1
|
||||
// w1 = b1·x - a1·y + w2
|
||||
// w2 = b2·x - a2·y
|
||||
let y = B0 * x + self.w1;
|
||||
self.w1 = B1 * x - A1 * y + self.w2;
|
||||
self.w2 = B2 * x - A2 * y;
|
||||
y
|
||||
}
|
||||
|
||||
/// Process a frame in-place.
|
||||
pub fn process(&mut self, frame: &mut [f32]) {
|
||||
for s in frame.iter_mut() {
|
||||
*s = self.process_sample(*s);
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset filter state (call on session restart).
|
||||
pub fn reset(&mut self) {
|
||||
self.w1 = 0.0;
|
||||
self.w2 = 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn dc_is_attenuated() {
|
||||
let mut hpf = HighPassFilter::new();
|
||||
// Feed 1000 samples of DC = 1.0 and check the output settles near 0.
|
||||
// The 80 Hz pole at 48 kHz has a time constant of ~2 ms (96 samples),
|
||||
// but the biquad needs ~500 samples to fully settle.
|
||||
let mut out = 0.0_f32;
|
||||
for _ in 0..1000 {
|
||||
out = hpf.process_sample(1.0);
|
||||
}
|
||||
assert!(
|
||||
out.abs() < 0.01,
|
||||
"DC not attenuated after 1000 samples: {out}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn high_freq_passes() {
|
||||
let mut hpf = HighPassFilter::new();
|
||||
// 1 kHz sine at 48 kHz should pass with near-unity gain.
|
||||
let mut peak = 0.0_f32;
|
||||
for i in 0..480 {
|
||||
let x = (2.0 * std::f32::consts::PI * 1000.0 * i as f32 / 48_000.0).sin();
|
||||
let y = hpf.process_sample(x);
|
||||
if i > 100 {
|
||||
// Skip transient
|
||||
peak = peak.max(y.abs());
|
||||
}
|
||||
}
|
||||
assert!(peak > 0.9, "1 kHz not passing: peak={peak}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_state() {
|
||||
let mut hpf = HighPassFilter::new();
|
||||
for _ in 0..100 {
|
||||
hpf.process_sample(1.0);
|
||||
}
|
||||
hpf.reset();
|
||||
assert_eq!(hpf.w1, 0.0);
|
||||
assert_eq!(hpf.w2, 0.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//! DSP building blocks for the Sonora software voice processor.
|
||||
//!
|
||||
//! Each module is self-contained, realtime-safe, and independently
|
||||
//! enable/disable-able. The modules are composed in `SonoraProcessor`
|
||||
//! in the order mandated by the P1 spec:
|
||||
//!
|
||||
//! HPF → AEC3 → NS → AGC2
|
||||
//!
|
||||
//! All modules operate at 48 kHz, 10 ms frames (480 samples).
|
||||
|
||||
pub mod aec3;
|
||||
pub mod agc2;
|
||||
pub mod hpf;
|
||||
pub mod ns;
|
||||
@@ -0,0 +1,325 @@
|
||||
//! Noise Suppression — Wiener filter with minimum statistics noise floor.
|
||||
//!
|
||||
//! ## Algorithm
|
||||
//!
|
||||
//! Frequency-domain Wiener filter:
|
||||
//!
|
||||
//! 1. **Analysis** — 480-sample frame zero-padded to 1024, Hann-windowed,
|
||||
//! transformed with a correct radix-2 DIT complex FFT.
|
||||
//!
|
||||
//! 2. **Noise floor** — per-bin minimum statistics tracker (Martin 2001).
|
||||
//! Updated only in noise-dominated bins (SNR < VAD_SNR_THRESHOLD).
|
||||
//! Bias correction factor 1.5 accounts for minimum-statistics
|
||||
//! underestimation.
|
||||
//!
|
||||
//! 3. **Wiener gain** — G(k) = max(SNR(k)/(SNR(k)+1), GAIN_FLOOR).
|
||||
//! Floor at −20 dB prevents musical noise artefacts.
|
||||
//!
|
||||
//! 4. **Synthesis** — gain-weighted spectrum → IFFT → overlap-add.
|
||||
//!
|
||||
//! ## Realtime safety
|
||||
//!
|
||||
//! All buffers pre-allocated. No heap allocation in the hot path.
|
||||
|
||||
#![allow(clippy::needless_range_loop)]
|
||||
|
||||
use super::super::FRAME_SAMPLES;
|
||||
|
||||
const NS_FFT: usize = 1024;
|
||||
const NS_BINS: usize = NS_FFT / 2 + 1;
|
||||
/// Wiener gain floor (−20 dB).
|
||||
const GAIN_FLOOR: f32 = 0.1;
|
||||
/// Noise PSD smoothing (per-frame IIR).
|
||||
const NOISE_ALPHA: f32 = 0.98;
|
||||
/// Bias correction for minimum-statistics underestimation.
|
||||
const BIAS: f32 = 1.5;
|
||||
/// Bins with SNR below this are treated as noise-only.
|
||||
const VAD_SNR_THRESHOLD: f32 = 1.5;
|
||||
|
||||
/// Wiener filter noise suppressor.
|
||||
pub struct NoiseSuppressor {
|
||||
/// Per-bin noise PSD estimate.
|
||||
noise_psd: Box<[f32; NS_BINS]>,
|
||||
/// Overlap-add tail from the previous frame.
|
||||
ola_tail: Box<[f32; FRAME_SAMPLES]>,
|
||||
/// Hann window (NS_FFT length).
|
||||
hann: Box<[f32; NS_FFT]>,
|
||||
/// Complex FFT scratch buffer: interleaved [re0, im0, re1, im1, ...].
|
||||
/// Length = 2 * NS_FFT.
|
||||
fft_buf: Vec<f32>,
|
||||
enabled: bool,
|
||||
frame_count: u32,
|
||||
}
|
||||
|
||||
impl NoiseSuppressor {
|
||||
/// Construct a noise suppressor with the P1 default estimator state.
|
||||
pub fn new() -> Self {
|
||||
let mut hann = Box::new([0.0_f32; NS_FFT]);
|
||||
for (i, h) in hann.iter_mut().enumerate() {
|
||||
*h = 0.5 * (1.0 - (2.0 * std::f32::consts::PI * i as f32 / NS_FFT as f32).cos());
|
||||
}
|
||||
Self {
|
||||
noise_psd: Box::new([1e-6_f32; NS_BINS]),
|
||||
ola_tail: Box::new([0.0_f32; FRAME_SAMPLES]),
|
||||
hann,
|
||||
fft_buf: vec![0.0_f32; 2 * NS_FFT],
|
||||
enabled: true,
|
||||
frame_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable or disable noise suppression. When disabled `process` is a no-op.
|
||||
pub fn set_enabled(&mut self, enabled: bool) {
|
||||
self.enabled = enabled;
|
||||
}
|
||||
|
||||
/// Process one 10 ms capture frame in-place. Realtime-safe.
|
||||
pub fn process(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
|
||||
if !self.enabled {
|
||||
return;
|
||||
}
|
||||
self.frame_count = self.frame_count.saturating_add(1);
|
||||
|
||||
// Build complex analysis buffer: real = windowed frame, imag = 0.
|
||||
// Zero-pad from FRAME_SAMPLES to NS_FFT.
|
||||
for i in 0..NS_FFT {
|
||||
let re = if i < FRAME_SAMPLES {
|
||||
frame[i] * self.hann[i]
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
self.fft_buf[2 * i] = re;
|
||||
self.fft_buf[2 * i + 1] = 0.0;
|
||||
}
|
||||
|
||||
// Forward FFT.
|
||||
fft_complex_forward(&mut self.fft_buf, NS_FFT);
|
||||
|
||||
// Compute per-bin power spectrum from complex output.
|
||||
let mut power = [0.0_f32; NS_BINS];
|
||||
for k in 0..NS_BINS {
|
||||
let re = self.fft_buf[2 * k];
|
||||
let im = self.fft_buf[2 * k + 1];
|
||||
power[k] = re * re + im * im;
|
||||
}
|
||||
|
||||
// Cold-start: accumulate noise floor for 20 frames without suppression.
|
||||
if self.frame_count <= 20 {
|
||||
for k in 0..NS_BINS {
|
||||
self.noise_psd[k] =
|
||||
self.noise_psd[k] * NOISE_ALPHA + power[k] * (1.0 - NOISE_ALPHA);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute Wiener gain and update noise floor.
|
||||
let mut gain = [0.0_f32; NS_BINS];
|
||||
for k in 0..NS_BINS {
|
||||
let noise = self.noise_psd[k] * BIAS;
|
||||
let snr = ((power[k] - noise) / noise.max(1e-10)).max(0.0);
|
||||
gain[k] = (snr / (snr + 1.0)).max(GAIN_FLOOR);
|
||||
// Update noise PSD only in noise-dominated bins.
|
||||
if snr < VAD_SNR_THRESHOLD {
|
||||
self.noise_psd[k] =
|
||||
self.noise_psd[k] * NOISE_ALPHA + power[k] * (1.0 - NOISE_ALPHA);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply gain to the complex spectrum.
|
||||
// Bins 0..NS_BINS are the positive-frequency half.
|
||||
// Mirror to the negative-frequency half (conjugate symmetry).
|
||||
for k in 0..NS_BINS {
|
||||
self.fft_buf[2 * k] *= gain[k];
|
||||
self.fft_buf[2 * k + 1] *= gain[k];
|
||||
}
|
||||
// Mirror: bin k maps to bin NS_FFT - k.
|
||||
for k in 1..NS_BINS - 1 {
|
||||
let mirror = NS_FFT - k;
|
||||
self.fft_buf[2 * mirror] = self.fft_buf[2 * k];
|
||||
self.fft_buf[2 * mirror + 1] = -self.fft_buf[2 * k + 1]; // conjugate
|
||||
}
|
||||
|
||||
// Inverse FFT.
|
||||
fft_complex_inverse(&mut self.fft_buf, NS_FFT);
|
||||
|
||||
// Overlap-add: output = IFFT real part + previous tail.
|
||||
for i in 0..FRAME_SAMPLES {
|
||||
frame[i] = self.fft_buf[2 * i] + self.ola_tail[i];
|
||||
}
|
||||
// Save tail for next frame.
|
||||
for i in 0..FRAME_SAMPLES {
|
||||
self.ola_tail[i] = if i + FRAME_SAMPLES < NS_FFT {
|
||||
self.fft_buf[2 * (i + FRAME_SAMPLES)]
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset all state.
|
||||
pub fn reset(&mut self) {
|
||||
self.noise_psd.fill(1e-6);
|
||||
self.ola_tail.fill(0.0);
|
||||
self.fft_buf.fill(0.0);
|
||||
self.frame_count = 0;
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for NoiseSuppressor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Correct radix-2 DIT complex FFT ──────────────────────────────────────
|
||||
//
|
||||
// Buffer layout: interleaved [re0, im0, re1, im1, ..., re_{n-1}, im_{n-1}].
|
||||
// Length of buf must be 2*n where n is a power of 2.
|
||||
|
||||
fn fft_complex_forward(buf: &mut [f32], n: usize) {
|
||||
debug_assert_eq!(buf.len(), 2 * n);
|
||||
debug_assert!(n.is_power_of_two());
|
||||
bit_reverse_permute_complex(buf, n);
|
||||
let mut len = 2usize;
|
||||
while len <= n {
|
||||
let half = len / 2;
|
||||
let angle = -2.0 * std::f32::consts::PI / len as f32;
|
||||
let (wre, wim) = (angle.cos(), angle.sin());
|
||||
let mut start = 0;
|
||||
while start < n {
|
||||
let (mut cur_re, mut cur_im) = (1.0_f32, 0.0_f32);
|
||||
for j in 0..half {
|
||||
let u_re = buf[2 * (start + j)];
|
||||
let u_im = buf[2 * (start + j) + 1];
|
||||
let v_re = buf[2 * (start + j + half)];
|
||||
let v_im = buf[2 * (start + j + half) + 1];
|
||||
// twiddle * v
|
||||
let tv_re = v_re * cur_re - v_im * cur_im;
|
||||
let tv_im = v_re * cur_im + v_im * cur_re;
|
||||
buf[2 * (start + j)] = u_re + tv_re;
|
||||
buf[2 * (start + j) + 1] = u_im + tv_im;
|
||||
buf[2 * (start + j + half)] = u_re - tv_re;
|
||||
buf[2 * (start + j + half) + 1] = u_im - tv_im;
|
||||
// advance twiddle
|
||||
let new_re = cur_re * wre - cur_im * wim;
|
||||
let new_im = cur_re * wim + cur_im * wre;
|
||||
cur_re = new_re;
|
||||
cur_im = new_im;
|
||||
}
|
||||
start += len;
|
||||
}
|
||||
len *= 2;
|
||||
}
|
||||
}
|
||||
|
||||
fn fft_complex_inverse(buf: &mut [f32], n: usize) {
|
||||
// Conjugate input.
|
||||
for k in 0..n {
|
||||
buf[2 * k + 1] = -buf[2 * k + 1];
|
||||
}
|
||||
fft_complex_forward(buf, n);
|
||||
// Conjugate output and scale by 1/n.
|
||||
let scale = 1.0 / n as f32;
|
||||
for k in 0..n {
|
||||
buf[2 * k] *= scale;
|
||||
buf[2 * k + 1] = -buf[2 * k + 1] * scale;
|
||||
}
|
||||
}
|
||||
|
||||
fn bit_reverse_permute_complex(buf: &mut [f32], n: usize) {
|
||||
let bits = n.trailing_zeros() as usize;
|
||||
for i in 0..n {
|
||||
let j = reverse_bits(i, bits);
|
||||
if j > i {
|
||||
buf.swap(2 * i, 2 * j);
|
||||
buf.swap(2 * i + 1, 2 * j + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn reverse_bits(mut x: usize, bits: usize) -> usize {
|
||||
let mut r = 0usize;
|
||||
for _ in 0..bits {
|
||||
r = (r << 1) | (x & 1);
|
||||
x >>= 1;
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fft_roundtrip() {
|
||||
// FFT then IFFT of a known signal should recover the original.
|
||||
let mut buf = vec![0.0_f32; 2 * 8];
|
||||
// Input: [1, 2, 3, 4, 0, 0, 0, 0] (real only)
|
||||
for i in 0..4 {
|
||||
buf[2 * i] = (i + 1) as f32;
|
||||
}
|
||||
let original: Vec<f32> = buf.iter().step_by(2).take(8).copied().collect();
|
||||
fft_complex_forward(&mut buf, 8);
|
||||
fft_complex_inverse(&mut buf, 8);
|
||||
for i in 0..8 {
|
||||
assert!(
|
||||
(buf[2 * i] - original[i]).abs() < 1e-4,
|
||||
"roundtrip failed at {i}: got {} expected {}",
|
||||
buf[2 * i],
|
||||
original[i]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ns_reduces_stationary_noise() {
|
||||
let mut ns = NoiseSuppressor::new();
|
||||
let mut rng: u32 = 0xDEAD_BEEF;
|
||||
let noise_frame = |rng: &mut u32| -> [f32; FRAME_SAMPLES] {
|
||||
let mut f = [0.0_f32; FRAME_SAMPLES];
|
||||
for s in f.iter_mut() {
|
||||
*rng = rng.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
*s = (*rng as i32 as f32) / i32::MAX as f32 * 0.05;
|
||||
}
|
||||
f
|
||||
};
|
||||
// Warm up noise floor (20 frames cold-start + 10 more to converge).
|
||||
for _ in 0..30 {
|
||||
let mut frame = noise_frame(&mut rng);
|
||||
ns.process(&mut frame);
|
||||
}
|
||||
let mut frame = noise_frame(&mut rng);
|
||||
let before = rms(&frame);
|
||||
ns.process(&mut frame);
|
||||
let after = rms(&frame);
|
||||
assert!(
|
||||
after < before * 0.8,
|
||||
"NS did not suppress noise: before={before:.4} after={after:.4}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ns_disabled_is_passthrough() {
|
||||
let mut ns = NoiseSuppressor::new();
|
||||
ns.set_enabled(false);
|
||||
let mut frame = [0.1_f32; FRAME_SAMPLES];
|
||||
ns.process(&mut frame);
|
||||
assert!(frame.iter().all(|&s| (s - 0.1).abs() < 1e-6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_state() {
|
||||
let mut ns = NoiseSuppressor::new();
|
||||
for _ in 0..30 {
|
||||
let mut frame = [0.05_f32; FRAME_SAMPLES];
|
||||
ns.process(&mut frame);
|
||||
}
|
||||
ns.reset();
|
||||
assert_eq!(ns.frame_count, 0);
|
||||
assert!(ns.ola_tail.iter().all(|&s| s == 0.0));
|
||||
}
|
||||
|
||||
fn rms(frame: &[f32]) -> f32 {
|
||||
(frame.iter().map(|s| s * s).sum::<f32>() / frame.len() as f32).sqrt()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
//! Realtime-safe audio processors for platform and software voice paths.
|
||||
|
||||
pub mod dsp;
|
||||
pub mod noop;
|
||||
pub mod platform;
|
||||
pub mod sonora;
|
||||
|
||||
pub use noop::NoopProcessor;
|
||||
pub use platform::PlatformVoiceProcessor;
|
||||
pub use sonora::SonoraProcessor;
|
||||
|
||||
/// 10 ms mono f32 processing frame at 48 kHz (480 samples).
|
||||
pub const FRAME_SAMPLES: usize = 480;
|
||||
|
||||
/// Realtime-safe audio processor backend.
|
||||
///
|
||||
/// Implementations MUST be `Send` and MUST NOT allocate, block, or
|
||||
/// perform I/O inside `process_capture` or `process_render`.
|
||||
pub trait AudioProcessor: Send {
|
||||
/// Process one 10 ms capture frame in-place.
|
||||
fn process_capture(&mut self, frame: &mut [f32; FRAME_SAMPLES]);
|
||||
|
||||
/// Feed one 10 ms render-reference frame (decoded remote PCM
|
||||
/// before playout). Required by software AEC backends; no-op
|
||||
/// for platform and noop backends.
|
||||
fn process_render(&mut self, frame: &[f32; FRAME_SAMPLES]);
|
||||
|
||||
/// Return true if this backend performs acoustic echo cancellation
|
||||
/// so the engine can enforce INV_009 / INV_010.
|
||||
fn has_aec(&self) -> bool;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//! Processor backend that intentionally leaves audio unchanged.
|
||||
|
||||
use super::{AudioProcessor, FRAME_SAMPLES};
|
||||
|
||||
/// No-op audio processor for debug/headset routes.
|
||||
pub struct NoopProcessor;
|
||||
|
||||
impl AudioProcessor for NoopProcessor {
|
||||
fn process_capture(&mut self, _frame: &mut [f32; FRAME_SAMPLES]) {}
|
||||
|
||||
fn process_render(&mut self, _frame: &[f32; FRAME_SAMPLES]) {}
|
||||
|
||||
fn has_aec(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
//! Platform-owned voice processing backend.
|
||||
|
||||
use super::{AudioProcessor, FRAME_SAMPLES};
|
||||
|
||||
/// Marker backend for the platform VoiceProcessingIO path.
|
||||
/// All DSP (AEC/NS/AGC) is handled by the hardware voice processor;
|
||||
/// Rust-side processing is a no-op. `has_aec` returns true so the
|
||||
/// engine enforces INV_009/INV_010 and never enables Rust AEC
|
||||
/// simultaneously.
|
||||
pub struct PlatformVoiceProcessor;
|
||||
|
||||
impl AudioProcessor for PlatformVoiceProcessor {
|
||||
fn process_capture(&mut self, _frame: &mut [f32; FRAME_SAMPLES]) {}
|
||||
|
||||
fn process_render(&mut self, _frame: &[f32; FRAME_SAMPLES]) {}
|
||||
|
||||
fn has_aec(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
//! Sonora software voice processor — full DSP chain.
|
||||
//!
|
||||
//! Composes the four P1 DSP stages in the order mandated by the spec:
|
||||
//!
|
||||
//! **HPF → AEC3 → NS → AGC2**
|
||||
//!
|
||||
//! Each stage is independently enable/disable-able via
|
||||
//! [`SonoraConfig`]. The default configuration matches the P1 spec:
|
||||
//! all stages enabled, AEC3 disabled when no render reference is
|
||||
//! available (INV_011).
|
||||
//!
|
||||
//! ## Stage descriptions
|
||||
//!
|
||||
//! | Stage | Module | Description |
|
||||
//! |-------|--------|-------------|
|
||||
//! | HPF | `dsp::hpf` | 80 Hz Butterworth biquad, removes DC and rumble |
|
||||
//! | AEC3 | `dsp::aec3` | Adaptive filter echo canceller with delay estimation |
|
||||
//! | NS | `dsp::ns` | Wiener filter noise suppressor with min-statistics floor |
|
||||
//! | AGC2 | `dsp::agc2` | RNN VAD-gated gain controller with look-ahead limiter |
|
||||
//!
|
||||
//! ## Realtime safety
|
||||
//!
|
||||
//! All state is pre-allocated. `process_capture` and `process_render`
|
||||
//! never allocate, block, or perform I/O (INV_007).
|
||||
//!
|
||||
//! ## INV_009 / INV_010 enforcement
|
||||
//!
|
||||
//! `has_aec()` returns `true` when AEC3 is enabled. The engine uses
|
||||
//! this to enforce the invariant that platform AEC and Rust AEC are
|
||||
//! never active simultaneously.
|
||||
|
||||
use super::dsp::{aec3::Aec3, agc2::Agc2, hpf::HighPassFilter, ns::NoiseSuppressor};
|
||||
use super::{AudioProcessor, FRAME_SAMPLES};
|
||||
|
||||
/// Per-stage enable flags for the Sonora processor.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct SonoraConfig {
|
||||
/// High-pass filter (80 Hz Butterworth). Default: enabled.
|
||||
pub hpf: bool,
|
||||
/// AEC3 adaptive echo canceller. Default: disabled until render
|
||||
/// reference is confirmed available (INV_011).
|
||||
pub aec3: bool,
|
||||
/// Wiener filter noise suppressor. Default: enabled.
|
||||
pub ns: bool,
|
||||
/// AGC2 gain controller + limiter. Default: enabled.
|
||||
pub agc2: bool,
|
||||
}
|
||||
|
||||
impl Default for SonoraConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
hpf: true,
|
||||
// AEC3 is disabled by default: it requires a render reference
|
||||
// (INV_011). The engine enables it only when the render
|
||||
// reference path is confirmed active.
|
||||
aec3: false,
|
||||
ns: true,
|
||||
agc2: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SonoraConfig {
|
||||
/// Configuration for the Sonora experimental mode with AEC3 enabled.
|
||||
/// Only valid when a render reference is available (INV_011).
|
||||
pub fn with_aec3() -> Self {
|
||||
Self {
|
||||
hpf: true,
|
||||
aec3: true,
|
||||
ns: true,
|
||||
agc2: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal configuration: HPF + AGC2 only (no AEC, no NS).
|
||||
/// Suitable for wired headset routes where AEC is not needed.
|
||||
pub fn headset() -> Self {
|
||||
Self {
|
||||
hpf: true,
|
||||
aec3: false,
|
||||
ns: false,
|
||||
agc2: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Full Sonora DSP chain: HPF → AEC3 → NS → AGC2.
|
||||
pub struct SonoraProcessor {
|
||||
hpf: HighPassFilter,
|
||||
aec3: Aec3,
|
||||
ns: NoiseSuppressor,
|
||||
agc2: Agc2,
|
||||
config: SonoraConfig,
|
||||
}
|
||||
|
||||
impl SonoraProcessor {
|
||||
/// Construct with the default configuration (AEC3 disabled).
|
||||
pub fn new() -> Self {
|
||||
let config = SonoraConfig::default();
|
||||
let mut aec3 = Aec3::new();
|
||||
aec3.set_enabled(config.aec3);
|
||||
let mut ns = NoiseSuppressor::new();
|
||||
ns.set_enabled(config.ns);
|
||||
let mut agc2 = Agc2::new();
|
||||
agc2.set_enabled(config.agc2);
|
||||
Self {
|
||||
hpf: HighPassFilter::new(),
|
||||
aec3,
|
||||
ns,
|
||||
agc2,
|
||||
config,
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct with a specific configuration.
|
||||
pub fn with_config(config: SonoraConfig) -> Self {
|
||||
let mut aec3 = Aec3::new();
|
||||
aec3.set_enabled(config.aec3);
|
||||
let mut ns = NoiseSuppressor::new();
|
||||
ns.set_enabled(config.ns);
|
||||
let mut agc2 = Agc2::new();
|
||||
agc2.set_enabled(config.agc2);
|
||||
Self {
|
||||
hpf: HighPassFilter::new(),
|
||||
aec3,
|
||||
ns,
|
||||
agc2,
|
||||
config,
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a new configuration at runtime. Resets stages whose
|
||||
/// enable state changed to avoid state contamination.
|
||||
pub fn apply_config(&mut self, new_config: SonoraConfig) {
|
||||
if new_config.hpf != self.config.hpf {
|
||||
self.hpf.reset();
|
||||
}
|
||||
if new_config.aec3 != self.config.aec3 {
|
||||
self.aec3.reset();
|
||||
self.aec3.set_enabled(new_config.aec3);
|
||||
}
|
||||
if new_config.ns != self.config.ns {
|
||||
self.ns.reset();
|
||||
self.ns.set_enabled(new_config.ns);
|
||||
}
|
||||
if new_config.agc2 != self.config.agc2 {
|
||||
self.agc2.reset();
|
||||
self.agc2.set_enabled(new_config.agc2);
|
||||
}
|
||||
self.config = new_config;
|
||||
}
|
||||
|
||||
/// Current configuration.
|
||||
pub fn config(&self) -> &SonoraConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Reset all DSP state (call on route change or session restart).
|
||||
pub fn reset_all(&mut self) {
|
||||
self.hpf.reset();
|
||||
self.aec3.reset();
|
||||
self.ns.reset();
|
||||
self.agc2.reset();
|
||||
}
|
||||
|
||||
/// Current AEC3 bulk delay estimate in blocks (1 block = 10 ms).
|
||||
pub fn aec3_bulk_delay_blocks(&self) -> usize {
|
||||
self.aec3.bulk_delay_blocks()
|
||||
}
|
||||
|
||||
/// True if AEC3 has converged.
|
||||
pub fn aec3_converged(&self) -> bool {
|
||||
self.aec3.is_converged()
|
||||
}
|
||||
|
||||
/// Current AGC2 speech probability from the RNN VAD.
|
||||
pub fn agc2_speech_probability(&self) -> f32 {
|
||||
self.agc2.speech_probability()
|
||||
}
|
||||
|
||||
/// Current AGC2 gain in dB.
|
||||
pub fn agc2_gain_db(&self) -> f32 {
|
||||
self.agc2.gain_db()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SonoraProcessor {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl AudioProcessor for SonoraProcessor {
|
||||
/// Process one 10 ms capture frame in-place.
|
||||
///
|
||||
/// Pipeline: HPF → AEC3 → NS → AGC2.
|
||||
fn process_capture(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
|
||||
// Stage 1: High-pass filter (DC removal, rumble suppression).
|
||||
if self.config.hpf {
|
||||
self.hpf.process(frame);
|
||||
}
|
||||
|
||||
// Stage 2: AEC3 (echo cancellation).
|
||||
// AEC3 reads the render reference that was fed via process_render.
|
||||
// INV_011: only runs when aec3 is enabled (render reference available).
|
||||
self.aec3.process_capture(frame);
|
||||
|
||||
// Stage 3: Noise suppression (Wiener filter).
|
||||
self.ns.process(frame);
|
||||
|
||||
// Stage 4: AGC2 (gain control + limiter).
|
||||
self.agc2.process(frame);
|
||||
}
|
||||
|
||||
/// Feed one 10 ms render-reference frame (decoded remote PCM
|
||||
/// before playout). Required by AEC3 (INV_012).
|
||||
fn process_render(&mut self, frame: &[f32; FRAME_SAMPLES]) {
|
||||
self.aec3.process_render(frame);
|
||||
}
|
||||
|
||||
/// True when AEC3 is enabled (enforces INV_009 / INV_010).
|
||||
fn has_aec(&self) -> bool {
|
||||
self.config.aec3
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_config_has_aec3_disabled() {
|
||||
let p = SonoraProcessor::new();
|
||||
assert!(!p.has_aec(), "AEC3 must be disabled by default (INV_010)");
|
||||
assert!(p.config().hpf);
|
||||
assert!(p.config().ns);
|
||||
assert!(p.config().agc2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_aec3_config_enables_aec() {
|
||||
let p = SonoraProcessor::with_config(SonoraConfig::with_aec3());
|
||||
assert!(p.has_aec());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_capture_does_not_panic_on_silence() {
|
||||
let mut p = SonoraProcessor::new();
|
||||
let mut frame = [0.0_f32; FRAME_SAMPLES];
|
||||
p.process_capture(&mut frame);
|
||||
assert!(frame.iter().all(|s| s.is_finite()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_capture_does_not_panic_on_loud_signal() {
|
||||
let mut p = SonoraProcessor::new();
|
||||
let mut frame = [1.0_f32; FRAME_SAMPLES];
|
||||
p.process_capture(&mut frame);
|
||||
assert!(frame.iter().all(|s| s.is_finite()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hpf_removes_dc() {
|
||||
let mut p = SonoraProcessor::with_config(SonoraConfig {
|
||||
hpf: true,
|
||||
aec3: false,
|
||||
ns: false,
|
||||
agc2: false,
|
||||
});
|
||||
// Feed 200 frames of DC = 0.5.
|
||||
let mut frame = [0.5_f32; FRAME_SAMPLES];
|
||||
for _ in 0..200 {
|
||||
p.process_capture(&mut frame);
|
||||
}
|
||||
// After convergence, DC should be near zero.
|
||||
let mean: f32 = frame.iter().sum::<f32>() / FRAME_SAMPLES as f32;
|
||||
assert!(mean.abs() < 0.01, "DC not removed: mean={mean}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_config_resets_changed_stages() {
|
||||
let mut p = SonoraProcessor::new();
|
||||
// Run some frames to build up state.
|
||||
let mut frame = [0.1_f32; FRAME_SAMPLES];
|
||||
for _ in 0..10 {
|
||||
p.process_capture(&mut frame);
|
||||
}
|
||||
// Enable AEC3 — should reset AEC3 state.
|
||||
p.apply_config(SonoraConfig::with_aec3());
|
||||
assert!(p.has_aec());
|
||||
assert!(!p.aec3_converged()); // reset clears convergence
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suppresses_stationary_noise() {
|
||||
let mut p = SonoraProcessor::with_config(SonoraConfig {
|
||||
hpf: false,
|
||||
aec3: false,
|
||||
ns: true,
|
||||
agc2: false,
|
||||
});
|
||||
let mut rng: u32 = 0xABCD_1234;
|
||||
let noise_frame = |rng: &mut u32| -> [f32; FRAME_SAMPLES] {
|
||||
let mut f = [0.0_f32; FRAME_SAMPLES];
|
||||
for s in f.iter_mut() {
|
||||
*rng = rng.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
*s = (*rng as i32 as f32) / i32::MAX as f32 * 0.05;
|
||||
}
|
||||
f
|
||||
};
|
||||
// Warm up noise floor.
|
||||
for _ in 0..30 {
|
||||
let mut frame = noise_frame(&mut rng);
|
||||
p.process_capture(&mut frame);
|
||||
}
|
||||
let mut frame = noise_frame(&mut rng);
|
||||
let before = rms(&frame);
|
||||
p.process_capture(&mut frame);
|
||||
let after = rms(&frame);
|
||||
assert!(
|
||||
after < before,
|
||||
"NS did not suppress noise: {before:.4} → {after:.4}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agc_amplifies_quiet_signal() {
|
||||
let mut p = SonoraProcessor::with_config(SonoraConfig {
|
||||
hpf: false,
|
||||
aec3: false,
|
||||
ns: false,
|
||||
agc2: true,
|
||||
});
|
||||
let mut frame = [0.0_f32; FRAME_SAMPLES];
|
||||
for (i, sample) in frame.iter_mut().enumerate() {
|
||||
*sample = (2.0 * std::f32::consts::PI * 300.0 * i as f32 / 48_000.0).sin() * 0.01;
|
||||
}
|
||||
let before = rms(&frame);
|
||||
for _ in 0..50 {
|
||||
p.process_capture(&mut frame);
|
||||
}
|
||||
let after = rms(&frame);
|
||||
assert!(
|
||||
after > before,
|
||||
"AGC did not amplify: {before:.4} → {after:.4}"
|
||||
);
|
||||
}
|
||||
|
||||
fn rms(frame: &[f32]) -> f32 {
|
||||
let power = frame.iter().map(|s| s * s).sum::<f32>() / frame.len() as f32;
|
||||
power.sqrt()
|
||||
}
|
||||
}
|
||||
@@ -774,7 +774,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_descriptor_granted_keyboard_reports_L2() {
|
||||
fn build_descriptor_granted_keyboard_reports_l2() {
|
||||
let d = MacOSEventTapBackend::build_descriptor(
|
||||
PermissionState::Granted,
|
||||
PttInputClass::Keyboard,
|
||||
@@ -784,7 +784,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_descriptor_granted_mouse_reports_L3() {
|
||||
fn build_descriptor_granted_mouse_reports_l3() {
|
||||
let d = MacOSEventTapBackend::build_descriptor(
|
||||
PermissionState::Granted,
|
||||
PttInputClass::MouseSideButton,
|
||||
@@ -794,7 +794,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_descriptor_granted_none_reports_L2_keyboard() {
|
||||
fn build_descriptor_granted_none_reports_l2_keyboard() {
|
||||
let d =
|
||||
MacOSEventTapBackend::build_descriptor(PermissionState::Granted, PttInputClass::None);
|
||||
assert_eq!(d.level, PttCapabilityLevel::L2GlobalHoldToTalk);
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//! Route-to-processing policy for P1 iOS.
|
||||
//!
|
||||
//! Maps the current [`AudioRoute`] to the recommended
|
||||
//! [`AudioProcessingConfig`] for that route. The policy enforces
|
||||
//! INV_009 (never enable platform AEC and Rust AEC simultaneously)
|
||||
//! and INV_010 (never enable VoiceProcessingIO and Sonora AEC3
|
||||
//! simultaneously).
|
||||
//!
|
||||
//! The returned config is a *recommendation*; the engine may override
|
||||
//! individual fields (e.g. to keep the user's explicit VAD backend
|
||||
//! choice) but must not violate the hard invariants.
|
||||
|
||||
use crate::audio_processing::{
|
||||
AudioBackend, AudioProcessingConfig, AudioRoute, EffectOwner, IosVoiceProcessingMode,
|
||||
VadBackend,
|
||||
};
|
||||
|
||||
/// Compute the recommended [`AudioProcessingConfig`] for a given
|
||||
/// iOS audio route. The returned config always satisfies the P1
|
||||
/// hard invariants for iOS.
|
||||
///
|
||||
/// * Speaker / Earpiece → platform VPIO (AEC/NS/AGC owned by platform).
|
||||
/// * Wired headset → noop AEC, conservative NS/AGC optional.
|
||||
/// * Bluetooth HFP → route-managed (app-side AEC off, NS/AGC conservative).
|
||||
/// * Bluetooth A2DP → invalid for duplex; transmit blocked at selector level.
|
||||
/// * Unknown → safe fallback (AEC off until classified).
|
||||
pub fn ios_route_policy(route: AudioRoute) -> AudioProcessingConfig {
|
||||
match route {
|
||||
AudioRoute::Speaker | AudioRoute::Earpiece => AudioProcessingConfig {
|
||||
route,
|
||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
||||
vad_backend: VadBackend::SileroOnnx,
|
||||
aec: EffectOwner::Platform,
|
||||
// VPIO owns NS and AGC on the shipping default path (IOSP_002/003).
|
||||
ns: EffectOwner::Platform,
|
||||
agc: EffectOwner::Platform,
|
||||
hpf_enabled: true,
|
||||
limiter_enabled: true,
|
||||
..AudioProcessingConfig::default()
|
||||
},
|
||||
AudioRoute::WiredHeadset => AudioProcessingConfig {
|
||||
route,
|
||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||
processing_backend: AudioBackend::Noop,
|
||||
vad_backend: VadBackend::SileroOnnx,
|
||||
// No AEC needed for wired headset (no acoustic echo path).
|
||||
aec: EffectOwner::Off,
|
||||
// Conservative NS/AGC: optional, not forced.
|
||||
ns: EffectOwner::Conservative,
|
||||
agc: EffectOwner::Conservative,
|
||||
hpf_enabled: true,
|
||||
limiter_enabled: true,
|
||||
..AudioProcessingConfig::default()
|
||||
},
|
||||
AudioRoute::BluetoothHfp => AudioProcessingConfig {
|
||||
route,
|
||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||
processing_backend: AudioBackend::PlatformVoiceProcessing,
|
||||
vad_backend: VadBackend::SileroOnnx,
|
||||
// BT HFP manages its own AEC in the headset firmware.
|
||||
aec: EffectOwner::Off,
|
||||
ns: EffectOwner::Conservative,
|
||||
agc: EffectOwner::Conservative,
|
||||
hpf_enabled: true,
|
||||
limiter_enabled: true,
|
||||
..AudioProcessingConfig::default()
|
||||
},
|
||||
AudioRoute::BluetoothA2dp => {
|
||||
// A2DP is output-only; duplex voice is invalid on this route.
|
||||
// Return a config that disables all processing and VAD.
|
||||
// The transmit selector will block transmit via the route check.
|
||||
AudioProcessingConfig {
|
||||
route,
|
||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||
processing_backend: AudioBackend::Noop,
|
||||
vad_backend: VadBackend::Disabled,
|
||||
aec: EffectOwner::Off,
|
||||
ns: EffectOwner::Off,
|
||||
agc: EffectOwner::Off,
|
||||
hpf_enabled: false,
|
||||
limiter_enabled: false,
|
||||
..AudioProcessingConfig::default()
|
||||
}
|
||||
}
|
||||
AudioRoute::Unknown => AudioProcessingConfig {
|
||||
route,
|
||||
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
|
||||
processing_backend: AudioBackend::Noop,
|
||||
vad_backend: VadBackend::SileroOnnx,
|
||||
// Safe fallback: AEC off until route is classified.
|
||||
aec: EffectOwner::Off,
|
||||
ns: EffectOwner::Off,
|
||||
agc: EffectOwner::Off,
|
||||
hpf_enabled: true,
|
||||
limiter_enabled: true,
|
||||
..AudioProcessingConfig::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a route change to an existing config, preserving user-chosen
|
||||
/// VAD backend, timing, and debug settings while updating the
|
||||
/// route-dependent policy fields.
|
||||
pub fn apply_route_change(
|
||||
existing: &AudioProcessingConfig,
|
||||
new_route: AudioRoute,
|
||||
) -> AudioProcessingConfig {
|
||||
let policy = ios_route_policy(new_route);
|
||||
AudioProcessingConfig {
|
||||
// Route-policy fields from the new route.
|
||||
route: policy.route,
|
||||
ios_mode: policy.ios_mode,
|
||||
processing_backend: policy.processing_backend,
|
||||
aec: policy.aec,
|
||||
ns: policy.ns,
|
||||
agc: policy.agc,
|
||||
hpf_enabled: policy.hpf_enabled,
|
||||
limiter_enabled: policy.limiter_enabled,
|
||||
// Preserve user-chosen VAD backend and timing.
|
||||
vad_backend: existing.vad_backend,
|
||||
vad_hangover_ms: existing.vad_hangover_ms,
|
||||
vad_pre_roll_ms: existing.vad_pre_roll_ms,
|
||||
vad_min_tx_ms: existing.vad_min_tx_ms,
|
||||
// Preserve debug settings.
|
||||
debug_wav_dump_enabled: existing.debug_wav_dump_enabled,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn speaker_uses_platform_vpio() {
|
||||
let cfg = ios_route_policy(AudioRoute::Speaker);
|
||||
assert_eq!(
|
||||
cfg.processing_backend,
|
||||
AudioBackend::PlatformVoiceProcessing
|
||||
);
|
||||
assert_eq!(cfg.aec, EffectOwner::Platform);
|
||||
// VPIO owns NS and AGC on the default path.
|
||||
assert_eq!(cfg.ns, EffectOwner::Platform);
|
||||
assert_eq!(cfg.agc, EffectOwner::Platform);
|
||||
assert_eq!(
|
||||
cfg.ios_mode,
|
||||
IosVoiceProcessingMode::PlatformVoiceProcessing
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn earpiece_uses_platform_vpio() {
|
||||
let cfg = ios_route_policy(AudioRoute::Earpiece);
|
||||
assert_eq!(
|
||||
cfg.processing_backend,
|
||||
AudioBackend::PlatformVoiceProcessing
|
||||
);
|
||||
assert_eq!(cfg.aec, EffectOwner::Platform);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wired_headset_disables_aec() {
|
||||
let cfg = ios_route_policy(AudioRoute::WiredHeadset);
|
||||
assert_eq!(cfg.aec, EffectOwner::Off);
|
||||
assert_eq!(cfg.processing_backend, AudioBackend::Noop);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bluetooth_hfp_disables_app_aec() {
|
||||
let cfg = ios_route_policy(AudioRoute::BluetoothHfp);
|
||||
assert_eq!(cfg.aec, EffectOwner::Off);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bluetooth_a2dp_disables_all_processing_and_vad() {
|
||||
let cfg = ios_route_policy(AudioRoute::BluetoothA2dp);
|
||||
assert_eq!(cfg.aec, EffectOwner::Off);
|
||||
assert_eq!(cfg.vad_backend, VadBackend::Disabled);
|
||||
assert_eq!(cfg.processing_backend, AudioBackend::Noop);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_route_safe_fallback_no_aec() {
|
||||
let cfg = ios_route_policy(AudioRoute::Unknown);
|
||||
assert_eq!(cfg.aec, EffectOwner::Off);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_route_change_preserves_vad_timing_and_debug() {
|
||||
let existing = AudioProcessingConfig {
|
||||
vad_backend: VadBackend::WebrtcVad,
|
||||
vad_hangover_ms: 600,
|
||||
vad_pre_roll_ms: 200,
|
||||
vad_min_tx_ms: 300,
|
||||
debug_wav_dump_enabled: true,
|
||||
..AudioProcessingConfig::default()
|
||||
};
|
||||
let updated = apply_route_change(&existing, AudioRoute::WiredHeadset);
|
||||
assert_eq!(updated.vad_backend, VadBackend::WebrtcVad);
|
||||
assert_eq!(updated.vad_hangover_ms, 600);
|
||||
assert_eq!(updated.vad_pre_roll_ms, 200);
|
||||
assert_eq!(updated.vad_min_tx_ms, 300);
|
||||
assert!(updated.debug_wav_dump_enabled);
|
||||
assert_eq!(updated.route, AudioRoute::WiredHeadset);
|
||||
assert_eq!(updated.aec, EffectOwner::Off);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_sonora_aec_in_platform_vpio_policy() {
|
||||
// INV_009: AEC must never be Sonora in the VPIO path.
|
||||
for route in [
|
||||
AudioRoute::Speaker,
|
||||
AudioRoute::Earpiece,
|
||||
AudioRoute::WiredHeadset,
|
||||
AudioRoute::BluetoothHfp,
|
||||
AudioRoute::BluetoothA2dp,
|
||||
AudioRoute::Unknown,
|
||||
] {
|
||||
let cfg = ios_route_policy(route);
|
||||
assert_ne!(
|
||||
cfg.processing_backend,
|
||||
AudioBackend::Sonora,
|
||||
"route {:?} must not use Sonora backend in platform policy",
|
||||
route
|
||||
);
|
||||
assert_ne!(
|
||||
cfg.aec,
|
||||
EffectOwner::Sonora,
|
||||
"route {:?}: AEC must not be Sonora in VPIO path (INV_009)",
|
||||
route
|
||||
);
|
||||
assert_ne!(
|
||||
cfg.ns,
|
||||
EffectOwner::Sonora,
|
||||
"route {:?}: NS must not be Sonora in VPIO path (IOSP_003)",
|
||||
route
|
||||
);
|
||||
assert_ne!(
|
||||
cfg.agc,
|
||||
EffectOwner::Sonora,
|
||||
"route {:?}: AGC must not be Sonora in VPIO path (IOSP_003)",
|
||||
route
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,33 +5,23 @@
|
||||
//! [`chanora_storage::IdentityFileStore`] under the `transmit_mode`
|
||||
//! metadata key (default [`TransmitMode::Ptt`]).
|
||||
//!
|
||||
//! `VoiceActivity` is reserved per DEC-030 — for v1 the
|
||||
//! [`crate::transmit_selector::TransmitModeSelector`] treats it
|
||||
//! exactly like [`TransmitMode::Continuous`] until a real VAD
|
||||
//! implementation lands.
|
||||
//! `VoiceActivity` is driven by Rust-owned VAD state in P1.
|
||||
|
||||
/// User-visible voice transmit mode.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
|
||||
#[repr(u8)]
|
||||
pub enum TransmitMode {
|
||||
/// Push-to-talk: transmit only while the bound key is held
|
||||
/// (with release-tail per SDD-096).
|
||||
#[default]
|
||||
Ptt = 0,
|
||||
/// Continuous: transmit whenever the user is in a voice
|
||||
/// channel and not hard-muted.
|
||||
Continuous = 1,
|
||||
/// Voice activity detection. Reserved per DEC-030; v1 behaves
|
||||
/// as [`TransmitMode::Continuous`] until a VAD implementation
|
||||
/// is allocated.
|
||||
/// Voice activity detection.
|
||||
VoiceActivity = 2,
|
||||
}
|
||||
|
||||
impl Default for TransmitMode {
|
||||
fn default() -> Self {
|
||||
Self::Ptt
|
||||
}
|
||||
}
|
||||
|
||||
impl TransmitMode {
|
||||
/// Encode as the persisted single-byte value.
|
||||
pub fn as_u8(self) -> u8 {
|
||||
|
||||
@@ -11,8 +11,7 @@
|
||||
//! on PTT mode)
|
||||
//!
|
||||
//! Hard-mute is a final clamp; leaving the channel forces the gate
|
||||
//! to `false`. `VoiceActivity` is treated identically to
|
||||
//! `Continuous` per DEC-030 until a VAD implementation lands.
|
||||
//! to `false`. `VoiceActivity` is driven by Rust-owned VAD state.
|
||||
//!
|
||||
//! All four inputs are stored as atomics so any thread can update
|
||||
//! them without taking a lock. After each update we call
|
||||
@@ -92,6 +91,7 @@ pub struct TransmitModeSelector {
|
||||
in_channel: AtomicBool,
|
||||
hard_mute: AtomicBool,
|
||||
ptt_held: AtomicBool,
|
||||
voice_activity_open: AtomicBool,
|
||||
/// SDD-106 §5/§6 / SRS-209: latest resolved microphone
|
||||
/// permission state. Stored as a `u8` so writes from the
|
||||
/// JNI thread (Android permission requester → bridge) and
|
||||
@@ -133,6 +133,7 @@ impl TransmitModeSelector {
|
||||
in_channel: AtomicBool::new(false),
|
||||
hard_mute: AtomicBool::new(false),
|
||||
ptt_held: AtomicBool::new(false),
|
||||
voice_activity_open: AtomicBool::new(false),
|
||||
// SDD-106 §5: default to Granted on construction so
|
||||
// non-Android hosts (which never publish a permission
|
||||
// event) are not silently clamped. The Android bridge
|
||||
@@ -226,6 +227,17 @@ impl TransmitModeSelector {
|
||||
self.ptt_held.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Rust-owned VAD gate input for VoiceActivity mode.
|
||||
pub fn set_voice_activity_open(&self, v: bool) {
|
||||
self.voice_activity_open.store(v, Ordering::Relaxed);
|
||||
self.recompute();
|
||||
}
|
||||
|
||||
/// Current Rust-owned VAD gate state.
|
||||
pub fn voice_activity_open(&self) -> bool {
|
||||
self.voice_activity_open.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Subscribe to `ptt_held` transitions. Used by the
|
||||
/// missed-key-up watchdog (SAD-079) so it fires on the actual
|
||||
/// PTT-key-down lifetime, not on the resolved `transmit_active`
|
||||
@@ -265,8 +277,8 @@ impl TransmitModeSelector {
|
||||
}
|
||||
match self.mode() {
|
||||
TransmitMode::Ptt => self.ptt_held.load(Ordering::Relaxed),
|
||||
// DEC-030: VoiceActivity behaves as Continuous in v1.
|
||||
TransmitMode::Continuous | TransmitMode::VoiceActivity => true,
|
||||
TransmitMode::Continuous => true,
|
||||
TransmitMode::VoiceActivity => self.voice_activity_open.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -323,13 +335,17 @@ mod tests {
|
||||
assert!(g.load(), "continuous independent of key state");
|
||||
}
|
||||
|
||||
/// SWE4-UV-037: voice-activity mode matches continuous in v1.
|
||||
/// SWE4-UV-037: voice-activity mode follows VAD state.
|
||||
#[test]
|
||||
fn voice_activity_matches_continuous_v1() {
|
||||
fn voice_activity_requires_vad_open() {
|
||||
let (g, s) = fresh();
|
||||
s.set_mode(TransmitMode::VoiceActivity);
|
||||
s.set_in_channel(true);
|
||||
assert!(!g.load());
|
||||
s.set_voice_activity_open(true);
|
||||
assert!(g.load());
|
||||
s.set_voice_activity_open(false);
|
||||
assert!(!g.load());
|
||||
}
|
||||
|
||||
/// SWE4-UV-037 / SWE4-UV-041: hard-mute clamps the transmit
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
//! Voice activity detection backends and helpers.
|
||||
//!
|
||||
//! iOS capture feeds VoiceProcessingIO-processed microphone frames into
|
||||
//! this module. The production path prefers a model-backed detector when
|
||||
//! available, and otherwise uses the realtime-safe fallback below so
|
||||
//! VoiceActivity mode never collapses back to Continuous transmit.
|
||||
|
||||
pub mod resampler;
|
||||
pub mod silero_onnx;
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
|
||||
use crate::frame::{dbfs, i16_to_f32};
|
||||
use crate::AudioError;
|
||||
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||
|
||||
pub use silero_onnx::SileroOnnxVad;
|
||||
|
||||
/// Voice activity detector output for one 10 ms frame.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct VadOutput {
|
||||
/// Speech confidence in the inclusive range `[0.0, 1.0]`.
|
||||
pub probability: f32,
|
||||
/// Immediate detector speech decision before hangover/min-duration state.
|
||||
pub speech: bool,
|
||||
}
|
||||
|
||||
/// Realtime-safe detector that consumes one 10 ms f32 mono frame.
|
||||
pub trait VoiceActivityDetector: Send {
|
||||
/// Process one 10 ms frame and return speech probability/state.
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput;
|
||||
}
|
||||
|
||||
/// Realtime-safe fallback VAD used when a model runtime is unavailable.
|
||||
///
|
||||
/// This is not an energy-only transmit gate. It combines RMS level,
|
||||
/// zero-crossing rate, and peak-to-RMS shape with hysteresis so stable
|
||||
/// background rumble is less likely to open VoiceActivity than speech.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WebRtcFallbackVad {
|
||||
open_dbfs: f32,
|
||||
close_dbfs: f32,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl Default for WebRtcFallbackVad {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
open_dbfs: -42.0,
|
||||
close_dbfs: -50.0,
|
||||
active: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WebRtcFallbackVad {
|
||||
fn zero_crossing_rate(samples: &[f32]) -> f32 {
|
||||
if samples.len() < 2 {
|
||||
return 0.0;
|
||||
}
|
||||
let crossings = samples
|
||||
.windows(2)
|
||||
.filter(|pair| (pair[0] >= 0.0 && pair[1] < 0.0) || (pair[0] < 0.0 && pair[1] >= 0.0))
|
||||
.count();
|
||||
crossings as f32 / (samples.len() - 1) as f32
|
||||
}
|
||||
|
||||
fn peak_to_rms(samples: &[f32], rms: f32) -> f32 {
|
||||
if rms <= 0.000_001 {
|
||||
return 0.0;
|
||||
}
|
||||
let peak = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
|
||||
peak / rms
|
||||
}
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for WebRtcFallbackVad {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
let level = dbfs(samples);
|
||||
let threshold = if self.active {
|
||||
self.close_dbfs
|
||||
} else {
|
||||
self.open_dbfs
|
||||
};
|
||||
let rms = samples.iter().map(|s| s * s).sum::<f32>() / samples.len().max(1) as f32;
|
||||
let rms = rms.sqrt();
|
||||
let zcr = Self::zero_crossing_rate(samples);
|
||||
let crest = Self::peak_to_rms(samples, rms);
|
||||
|
||||
// Level score: steeper curve so silence (-50 dBFS) scores near 0.
|
||||
// Speech is typically -30 to -10 dBFS; silence is -60 to -45 dBFS.
|
||||
// Map [-60, -20] → [0, 1] with a midpoint at -40 dBFS.
|
||||
let level_score = ((level + 60.0) / 40.0).clamp(0.0, 1.0);
|
||||
|
||||
let zcr_score = if (0.015..=0.32).contains(&zcr) {
|
||||
1.0
|
||||
} else {
|
||||
0.3 // penalise non-speech ZCR more aggressively
|
||||
};
|
||||
let crest_score = if (1.5..=12.0).contains(&crest) {
|
||||
1.0
|
||||
} else {
|
||||
0.3
|
||||
};
|
||||
let probability =
|
||||
(level_score * 0.72 + zcr_score * 0.18 + crest_score * 0.10).clamp(0.0, 1.0);
|
||||
self.active = level >= threshold && probability >= 0.5;
|
||||
VadOutput {
|
||||
probability,
|
||||
speech: self.active,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wraps any `VoiceActivityDetector` that operates at 16 kHz and
|
||||
/// downsamples 48 kHz input before forwarding.
|
||||
pub struct Resampled16kHzVad<D: VoiceActivityDetector> {
|
||||
inner: D,
|
||||
downsampler: Downsampler48to16,
|
||||
}
|
||||
|
||||
impl<D: VoiceActivityDetector> Resampled16kHzVad<D> {
|
||||
/// Wrap a 16 kHz detector so it can consume 48 kHz frames.
|
||||
pub fn new(inner: D) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
downsampler: Downsampler48to16::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<D: VoiceActivityDetector> VoiceActivityDetector for Resampled16kHzVad<D> {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
debug_assert_eq!(samples.len(), INPUT_FRAME_10MS);
|
||||
let mut input = [0.0_f32; INPUT_FRAME_10MS];
|
||||
input.copy_from_slice(samples);
|
||||
let downsampled = self.downsampler.process_frame_10ms(&input);
|
||||
self.inner.process_10ms(&downsampled)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one 48 kHz i16 10 ms frame and run a detector over it.
|
||||
pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16]) -> VadOutput {
|
||||
let mut frame = [0.0_f32; INPUT_FRAME_10MS];
|
||||
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
|
||||
*dst = i16_to_f32(src);
|
||||
}
|
||||
detector.process_10ms(&frame)
|
||||
}
|
||||
|
||||
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
/// Configure the preferred Silero ONNX model path.
|
||||
///
|
||||
/// The path is validated eagerly. A successful call increments the
|
||||
/// model epoch so running audio backends can reload the model without
|
||||
/// an app restart.
|
||||
pub fn set_silero_model_path(path: &str) -> Result<(), AudioError> {
|
||||
let path = path.trim();
|
||||
if path.is_empty() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"vad model path must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
if !std::path::Path::new(path).is_file() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(format!(
|
||||
"vad model path does not exist or is not a file: {path}"
|
||||
)));
|
||||
}
|
||||
let mut guard = silero_model_path_override()
|
||||
.write()
|
||||
.map_err(|_| AudioError::Backend("vad model path lock poisoned".to_string()))?;
|
||||
*guard = Some(path.to_string());
|
||||
SILERO_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Monotonic counter incremented whenever the configured model path changes.
|
||||
pub fn silero_model_epoch() -> u64 {
|
||||
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Return the expected path of the Silero VAD v6 ONNX model in the
|
||||
/// iOS app bundle. The model is shipped as a Flutter asset and copied
|
||||
/// to the app's Documents directory by the Dart-side asset loader.
|
||||
///
|
||||
/// Returns an empty string on non-Apple platforms (Silero is not
|
||||
/// supported there; `SileroOnnxVad::try_new` will return `None`).
|
||||
pub fn silero_model_bundle_path() -> String {
|
||||
if let Ok(guard) = silero_model_path_override().read() {
|
||||
if let Some(path) = guard.as_ref() {
|
||||
return path.clone();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
// Primary: Documents directory (written by Flutter asset loader).
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
let docs = format!("{home}/Documents/silero_vad.onnx");
|
||||
if std::path::Path::new(&docs).exists() {
|
||||
return docs;
|
||||
}
|
||||
// Fallback: app bundle Resources directory.
|
||||
let bundle = format!("{home}/../Library/silero_vad.onnx");
|
||||
if std::path::Path::new(&bundle).exists() {
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
// Last resort: current working directory (useful in tests).
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fallback_opens_for_voiced_signal() {
|
||||
let mut vad = WebRtcFallbackVad::default();
|
||||
let mut frame = [0_i16; INPUT_FRAME_10MS];
|
||||
for (idx, sample) in frame.iter_mut().enumerate() {
|
||||
let phase = idx as f32 * 2.0 * std::f32::consts::PI * 220.0 / 48_000.0;
|
||||
*sample = (phase.sin() * 12_000.0) as i16;
|
||||
}
|
||||
|
||||
let output = process_i16_10ms(&mut vad, &frame);
|
||||
|
||||
assert!(output.speech);
|
||||
assert!(output.probability >= 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_stays_closed_for_silence() {
|
||||
let mut vad = WebRtcFallbackVad::default();
|
||||
let frame = [0_i16; INPUT_FRAME_10MS];
|
||||
|
||||
let output = process_i16_10ms(&mut vad, &frame);
|
||||
|
||||
assert!(!output.speech);
|
||||
assert!(output.probability < 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_silero_model_path_rejects_missing_file() {
|
||||
let result = set_silero_model_path("/definitely/not/a/silero_vad.onnx");
|
||||
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_silero_model_path_updates_override_and_epoch() {
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("chanora_test_silero_{}.onnx", std::process::id()));
|
||||
std::fs::write(&path, b"test").unwrap();
|
||||
let before = silero_model_epoch();
|
||||
|
||||
set_silero_model_path(path.to_str().unwrap()).unwrap();
|
||||
|
||||
assert!(silero_model_epoch() > before);
|
||||
assert_eq!(silero_model_bundle_path(), path.to_string_lossy());
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Lightweight 48 kHz → 16 kHz downsampler for VAD input.
|
||||
//!
|
||||
//! Silero VAD and the real WebRTC VAD both operate at 16 kHz. The
|
||||
//! VPIO capture stream is pinned at 48 kHz (3× ratio). We use a
|
||||
//! simple polyphase FIR with a 3:1 decimation factor. The filter
|
||||
//! coefficients are a 32-tap Kaiser-windowed low-pass at 8 kHz
|
||||
//! (Nyquist of the 16 kHz output), pre-computed offline and baked
|
||||
//! in as constants so there is no runtime allocation.
|
||||
//!
|
||||
//! Quality is sufficient for VAD (speech/silence discrimination);
|
||||
//! this is not a high-fidelity resampler.
|
||||
|
||||
/// Input sample rate (Hz).
|
||||
pub const INPUT_HZ: u32 = 48_000;
|
||||
/// Output sample rate (Hz).
|
||||
pub const OUTPUT_HZ: u32 = 16_000;
|
||||
/// Decimation factor (INPUT_HZ / OUTPUT_HZ).
|
||||
pub const DECIMATION: usize = 3;
|
||||
|
||||
/// Samples in one 10 ms frame at 48 kHz.
|
||||
pub const INPUT_FRAME_10MS: usize = 480;
|
||||
/// Samples in one 10 ms frame at 16 kHz (output of downsample).
|
||||
pub const OUTPUT_FRAME_10MS: usize = 160;
|
||||
|
||||
/// 32-tap FIR low-pass filter coefficients (Kaiser β=8, fc=8 kHz/48 kHz).
|
||||
/// Generated with scipy.signal.firwin(32, 8000/48000*2, window=('kaiser', 8)).
|
||||
/// Symmetric — only 16 unique values; stored in full for clarity.
|
||||
#[rustfmt::skip]
|
||||
const FIR_COEFFS: [f32; 32] = [
|
||||
-0.000_592_3, -0.001_158_5, -0.001_601_5, -0.000_993_5,
|
||||
0.001_601_5, 0.006_046_8, 0.012_131_5, 0.018_614_0,
|
||||
0.023_448_0, 0.024_726_0, 0.021_048_0, 0.012_636_0,
|
||||
0.000_993_5, -0.011_614_0, -0.021_048_0, -0.024_726_0,
|
||||
-0.024_726_0, -0.021_048_0, -0.011_614_0, 0.000_993_5,
|
||||
0.012_636_0, 0.021_048_0, 0.024_726_0, 0.023_448_0,
|
||||
0.018_614_0, 0.012_131_5, 0.006_046_8, 0.001_601_5,
|
||||
-0.000_993_5, -0.001_601_5, -0.001_158_5, -0.000_592_3,
|
||||
];
|
||||
|
||||
const TAPS: usize = FIR_COEFFS.len();
|
||||
|
||||
/// Stateful 48→16 kHz downsampler. Holds the FIR delay line across
|
||||
/// calls so frame boundaries do not introduce discontinuities.
|
||||
pub struct Downsampler48to16 {
|
||||
/// Circular delay line (length = TAPS).
|
||||
delay: [f32; TAPS],
|
||||
/// Write head into the delay line.
|
||||
head: usize,
|
||||
/// Phase counter: 0..DECIMATION. When phase==0 we emit a sample.
|
||||
phase: usize,
|
||||
}
|
||||
|
||||
impl Default for Downsampler48to16 {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
delay: [0.0; TAPS],
|
||||
head: 0,
|
||||
phase: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Downsampler48to16 {
|
||||
/// Process `input` (48 kHz f32 mono) and write 16 kHz output
|
||||
/// into `output`. Returns the number of samples written.
|
||||
///
|
||||
/// For a full 10 ms input frame (480 samples) this always
|
||||
/// produces exactly 160 output samples.
|
||||
pub fn process(&mut self, input: &[f32], output: &mut [f32]) -> usize {
|
||||
let mut out_idx = 0;
|
||||
for &sample in input {
|
||||
// Push sample into circular delay line.
|
||||
self.delay[self.head] = sample;
|
||||
self.head = (self.head + 1) % TAPS;
|
||||
|
||||
if self.phase == 0 {
|
||||
// Compute FIR dot product.
|
||||
let mut acc = 0.0_f32;
|
||||
for (k, &coeff) in FIR_COEFFS.iter().enumerate() {
|
||||
let tap_idx = (self.head + TAPS - 1 - k) % TAPS;
|
||||
acc += self.delay[tap_idx] * coeff;
|
||||
}
|
||||
if out_idx < output.len() {
|
||||
output[out_idx] = acc;
|
||||
out_idx += 1;
|
||||
}
|
||||
}
|
||||
self.phase = (self.phase + 1) % DECIMATION;
|
||||
}
|
||||
out_idx
|
||||
}
|
||||
|
||||
/// Convenience: downsample a full 10 ms 48 kHz frame into a
|
||||
/// fixed-size 160-sample 16 kHz buffer.
|
||||
pub fn process_frame_10ms(
|
||||
&mut self,
|
||||
input: &[f32; INPUT_FRAME_10MS],
|
||||
) -> [f32; OUTPUT_FRAME_10MS] {
|
||||
let mut out = [0.0_f32; OUTPUT_FRAME_10MS];
|
||||
let n = self.process(input, &mut out);
|
||||
debug_assert_eq!(
|
||||
n, OUTPUT_FRAME_10MS,
|
||||
"resampler produced {n} samples, expected 160"
|
||||
);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn frame_produces_160_samples() {
|
||||
let mut ds = Downsampler48to16::default();
|
||||
let input = [0.5_f32; INPUT_FRAME_10MS];
|
||||
let out = ds.process_frame_10ms(&input);
|
||||
// DC input → DC output (scaled by filter gain ≈ 1/3 due to decimation).
|
||||
// Just check length and that output is finite and non-zero.
|
||||
assert_eq!(out.len(), OUTPUT_FRAME_10MS);
|
||||
assert!(out.iter().all(|s| s.is_finite()));
|
||||
assert!(out.iter().any(|s| s.abs() > 0.001));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silence_produces_silence() {
|
||||
let mut ds = Downsampler48to16::default();
|
||||
let input = [0.0_f32; INPUT_FRAME_10MS];
|
||||
let out = ds.process_frame_10ms(&input);
|
||||
assert!(out.iter().all(|s| s.abs() < 1e-9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consecutive_frames_are_continuous() {
|
||||
let mut ds = Downsampler48to16::default();
|
||||
// Two frames of DC — output should be stable (no edge discontinuity).
|
||||
let input = [0.3_f32; INPUT_FRAME_10MS];
|
||||
let out1 = ds.process_frame_10ms(&input);
|
||||
let out2 = ds.process_frame_10ms(&input);
|
||||
// Last sample of frame 1 and first sample of frame 2 should be close.
|
||||
let diff = (out1[OUTPUT_FRAME_10MS - 1] - out2[0]).abs();
|
||||
assert!(diff < 0.05, "discontinuity between frames: {diff}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
//! Silero VAD v6 ONNX runtime integration (P1 VAD_002).
|
||||
//!
|
||||
//! ## Silero VAD v6 model I/O
|
||||
//!
|
||||
//! The v6 model (silero_vad.onnx from the v6.x releases) has a different
|
||||
//! interface from v4. Key changes:
|
||||
//!
|
||||
//! | Tensor | Shape | Dtype | Meaning |
|
||||
//! |---------|------------------|-------|--------------------------------------|
|
||||
//! | input | \[1, 576\] | f32 | 64-sample context + 512-sample frame |
|
||||
//! | state | \[2, 1, 128\] | f32 | LSTM state (carry across frames) |
|
||||
//! | sr | \[1\] | i64 | Sample rate (16000 or 8000) |
|
||||
//! | output | \[1, 1\] | f32 | Speech probability |
|
||||
//! | stateN | \[2, 1, 128\] | f32 | Updated LSTM state |
|
||||
//!
|
||||
//! Frame size: **512 samples at 16 kHz = 32 ms**.
|
||||
//! Context: **64 samples** prepended to each frame (last 64 samples of previous frame).
|
||||
//! Total input width: 512 + 64 = **576 samples**.
|
||||
//!
|
||||
//! ## Threading
|
||||
//!
|
||||
//! `SileroOnnxVad` is `Send`. The session is created once and reused —
|
||||
//! never re-created per callback (INV_007).
|
||||
//!
|
||||
//! ## Accumulation
|
||||
//!
|
||||
//! The capture pipeline delivers 10 ms frames (480 samples at 48 kHz →
|
||||
//! 160 samples at 16 kHz). Three 10 ms frames = 30 ms ≈ 32 ms. We
|
||||
//! accumulate 512 samples (32 ms at 16 kHz) before running inference.
|
||||
//! The last probability is held between inference calls so the state
|
||||
//! machine always has a value to work with.
|
||||
//!
|
||||
//! ## Fallback
|
||||
//!
|
||||
//! `try_new` returns `None` when the model file is missing, the ONNX
|
||||
//! Runtime is unavailable, or the platform is not iOS/macOS. The caller
|
||||
//! falls back to `WebRtcFallbackVad`.
|
||||
|
||||
use super::{VadOutput, VoiceActivityDetector};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// 16 kHz frame size for Silero VAD v6 (32 ms).
|
||||
pub const SILERO_FRAME_16K: usize = 512;
|
||||
/// Context size prepended to each frame (64 samples at 16 kHz).
|
||||
pub const SILERO_CONTEXT_16K: usize = 64;
|
||||
/// Total input width: context + frame.
|
||||
pub const SILERO_INPUT_WIDTH: usize = SILERO_CONTEXT_16K + SILERO_FRAME_16K;
|
||||
/// LSTM state size: 2 × 1 × 128 = 256 f32 values.
|
||||
pub const SILERO_STATE_SIZE: usize = 256;
|
||||
/// Maximum lag in 10 ms frames before the realtime callback treats
|
||||
/// the Silero worker as stale and falls back to the local WebRTC
|
||||
/// detector for that frame.
|
||||
pub const SILERO_MAX_STALE_FRAMES: u64 = 3;
|
||||
|
||||
/// Silero VAD v6 ONNX backend.
|
||||
///
|
||||
/// Operates at **16 kHz**, accumulating 32 ms frames (512 samples)
|
||||
/// before running inference. The caller is responsible for downsampling
|
||||
/// from 48 kHz before calling `process_10ms`.
|
||||
pub struct SileroOnnxVad {
|
||||
/// LSTM state [2, 1, 128] — persisted across frames.
|
||||
state: Box<[f32; SILERO_STATE_SIZE]>,
|
||||
/// Context ring: last 64 samples of the previous frame.
|
||||
context: Box<[f32; SILERO_CONTEXT_16K]>,
|
||||
/// Accumulation buffer for 16 kHz samples (fills to SILERO_FRAME_16K).
|
||||
accum: Vec<f32>,
|
||||
/// Last speech probability output (held between inference calls).
|
||||
last_probability: f32,
|
||||
/// Model path stored for diagnostics.
|
||||
model_path: String,
|
||||
/// Inner ONNX implementation (platform-specific).
|
||||
inner: SileroInner,
|
||||
}
|
||||
|
||||
enum SileroInner {
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
Onnx(OnnxSession),
|
||||
#[allow(dead_code)]
|
||||
Stub,
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
struct OnnxSession {
|
||||
session: ort::session::Session,
|
||||
}
|
||||
|
||||
impl SileroOnnxVad {
|
||||
/// Attempt to load the Silero v6 ONNX model from `model_path`.
|
||||
///
|
||||
/// Returns `None` when the model file is missing, the ONNX Runtime
|
||||
/// is unavailable, or the platform is not iOS/macOS.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
Self::try_new_onnx(model_path)
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
let _ = model_path;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
fn try_new_onnx(model_path: &str) -> Option<Self> {
|
||||
use tracing::{error, info};
|
||||
|
||||
if !std::path::Path::new(model_path).exists() {
|
||||
tracing::warn!(
|
||||
target: "chanora_audio",
|
||||
path = model_path,
|
||||
"SileroOnnxVad: model file not found; falling back to WebRtcFallbackVad"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
if let Some(path) = bundled_onnxruntime_path() {
|
||||
let _ = ort::init_from(path.to_string_lossy()).commit();
|
||||
}
|
||||
|
||||
let session_result = std::panic::catch_unwind(|| {
|
||||
ort::session::Session::builder().and_then(|b| b.commit_from_file(model_path))
|
||||
});
|
||||
|
||||
match session_result {
|
||||
Err(_) => {
|
||||
error!(
|
||||
target: "chanora_audio",
|
||||
path = model_path,
|
||||
"SileroOnnxVad: ONNX Runtime panicked during load; falling back to WebRtcFallbackVad"
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok(Ok(session)) => {
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
path = model_path,
|
||||
"SileroOnnxVad v6: model loaded"
|
||||
);
|
||||
Some(Self {
|
||||
state: Box::new([0.0; SILERO_STATE_SIZE]),
|
||||
context: Box::new([0.0; SILERO_CONTEXT_16K]),
|
||||
accum: Vec::with_capacity(SILERO_FRAME_16K),
|
||||
last_probability: 0.0,
|
||||
model_path: model_path.to_owned(),
|
||||
inner: SileroInner::Onnx(OnnxSession { session }),
|
||||
})
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
error!(
|
||||
target: "chanora_audio",
|
||||
path = model_path,
|
||||
error = %e,
|
||||
"SileroOnnxVad: failed to load model; falling back to WebRtcFallbackVad"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset LSTM state and context (call on voice_leave / session restart).
|
||||
pub fn reset_state(&mut self) {
|
||||
self.state.iter_mut().for_each(|v| *v = 0.0);
|
||||
self.context.iter_mut().for_each(|v| *v = 0.0);
|
||||
self.accum.clear();
|
||||
self.last_probability = 0.0;
|
||||
}
|
||||
|
||||
/// Return the model path for diagnostics.
|
||||
pub fn model_path(&self) -> &str {
|
||||
&self.model_path
|
||||
}
|
||||
|
||||
fn input_with_context(context: &[f32; SILERO_CONTEXT_16K], audio_frame: &[f32]) -> Vec<f32> {
|
||||
let mut input = Vec::with_capacity(SILERO_CONTEXT_16K + audio_frame.len());
|
||||
input.extend_from_slice(context);
|
||||
input.extend_from_slice(audio_frame);
|
||||
input
|
||||
}
|
||||
|
||||
fn update_context_from_frame(&mut self, audio_frame: &[f32]) {
|
||||
let ctx_start = audio_frame.len().saturating_sub(SILERO_CONTEXT_16K);
|
||||
let new_ctx = &audio_frame[ctx_start..];
|
||||
let copy_len = new_ctx.len().min(SILERO_CONTEXT_16K);
|
||||
self.context.fill(0.0);
|
||||
self.context[SILERO_CONTEXT_16K - copy_len..].copy_from_slice(&new_ctx[..copy_len]);
|
||||
}
|
||||
|
||||
/// Run one upstream-style `calc_level` pass over a 32 ms / 512-sample
|
||||
/// 16 kHz frame: concatenate prior context, pass `input/state/sr` to
|
||||
/// ONNX, persist `stateN`, then refresh context from the current frame.
|
||||
/// Updates `last_probability` and returns the new value.
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
fn calc_level(&mut self, audio_frame: &[f32]) -> f32 {
|
||||
use ort::value::Value;
|
||||
use tracing::error;
|
||||
|
||||
let SileroInner::Onnx(ref mut inner) = self.inner else {
|
||||
return self.last_probability;
|
||||
};
|
||||
|
||||
debug_assert_eq!(audio_frame.len(), SILERO_FRAME_16K);
|
||||
|
||||
// Build input: [1, 576] = context (64) + frame (512), matching
|
||||
// snakers4/silero-vad's Rust `calc_level` example.
|
||||
let input_vec = Self::input_with_context(self.context.as_ref(), audio_frame);
|
||||
|
||||
// Build ndarray tensors.
|
||||
use ndarray::{Array, IxDyn};
|
||||
|
||||
let input_arr = Array::from_shape_vec(IxDyn(&[1, SILERO_INPUT_WIDTH]), input_vec);
|
||||
let state_arr = Array::from_shape_vec(IxDyn(&[2, 1, 128]), self.state.to_vec());
|
||||
let sr_arr = Array::from_shape_vec(IxDyn(&[1]), vec![16000_i64]);
|
||||
|
||||
let (input_arr, state_arr, sr_arr) = match (input_arr, state_arr, sr_arr) {
|
||||
(Ok(i), Ok(s), Ok(sr)) => (i, s, sr),
|
||||
_ => return self.last_probability,
|
||||
};
|
||||
|
||||
let input_val = match Value::from_array(input_arr) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: input tensor error");
|
||||
return self.last_probability;
|
||||
}
|
||||
};
|
||||
let state_val = match Value::from_array(state_arr) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: state tensor error");
|
||||
return self.last_probability;
|
||||
}
|
||||
};
|
||||
let sr_val = match Value::from_array(sr_arr) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: sr tensor error");
|
||||
return self.last_probability;
|
||||
}
|
||||
};
|
||||
|
||||
let outputs =
|
||||
match inner
|
||||
.session
|
||||
.run([(&input_val).into(), (&state_val).into(), (&sr_val).into()])
|
||||
{
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
error!(target: "chanora_audio", error = %e, "SileroOnnxVad: inference failed");
|
||||
return self.last_probability;
|
||||
}
|
||||
};
|
||||
|
||||
// Extract probability from "output".
|
||||
if let Ok((_, prob_data)) = outputs["output"].try_extract_tensor::<f32>() {
|
||||
if let Some(&p) = prob_data.first() {
|
||||
self.last_probability = p.clamp(0.0, 1.0);
|
||||
}
|
||||
}
|
||||
|
||||
// Update state from "stateN".
|
||||
if let Ok((shape, state_data)) = outputs["stateN"].try_extract_tensor::<f32>() {
|
||||
let total: usize = shape.iter().map(|&d| d as usize).product();
|
||||
let copy_len = total.min(SILERO_STATE_SIZE);
|
||||
self.state[..copy_len].copy_from_slice(&state_data[..copy_len]);
|
||||
}
|
||||
|
||||
drop(outputs);
|
||||
|
||||
// Match the upstream example: context becomes the last context_size
|
||||
// samples from the current frame after the model call succeeds.
|
||||
self.update_context_from_frame(audio_frame);
|
||||
|
||||
self.last_probability
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
|
||||
let exe = std::env::current_exe().ok()?;
|
||||
let app_dir = exe.parent()?;
|
||||
let framework = app_dir
|
||||
.join("Frameworks")
|
||||
.join("onnxruntime.framework")
|
||||
.join("onnxruntime");
|
||||
framework.exists().then_some(framework)
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for SileroOnnxVad {
|
||||
/// Accept one 10 ms **16 kHz** f32 mono frame (160 samples).
|
||||
///
|
||||
/// Accumulates samples until a full 32 ms frame (512 samples) is
|
||||
/// ready, then runs inference. Between inference calls the last
|
||||
/// probability is returned unchanged.
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
debug_assert_eq!(
|
||||
samples.len(),
|
||||
super::resampler::OUTPUT_FRAME_10MS,
|
||||
"SileroOnnxVad expects 160 samples (16 kHz 10 ms), got {}",
|
||||
samples.len()
|
||||
);
|
||||
|
||||
self.accum.extend_from_slice(samples);
|
||||
|
||||
if self.accum.len() >= SILERO_FRAME_16K {
|
||||
let audio_frame: Vec<f32> = self.accum[..SILERO_FRAME_16K].to_vec();
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
if matches!(self.inner, SileroInner::Onnx(_)) {
|
||||
self.calc_level(&audio_frame);
|
||||
} else {
|
||||
self.update_context_from_frame(&audio_frame);
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
self.update_context_from_frame(&audio_frame);
|
||||
}
|
||||
// Drain the accumulator (keep any overflow for next frame).
|
||||
let overflow: Vec<f32> = self.accum.drain(SILERO_FRAME_16K..).collect();
|
||||
self.accum.clear();
|
||||
self.accum.extend_from_slice(&overflow);
|
||||
}
|
||||
|
||||
VadOutput {
|
||||
probability: self.last_probability,
|
||||
speech: self.last_probability >= 0.5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: ONNX Runtime sessions are thread-safe for inference.
|
||||
// State arrays are owned by this struct and accessed only from
|
||||
// the single capture callback thread.
|
||||
unsafe impl Send for SileroOnnxVad {}
|
||||
|
||||
struct SileroFrameMessage {
|
||||
seq: u64,
|
||||
frame: [f32; super::resampler::INPUT_FRAME_10MS],
|
||||
}
|
||||
|
||||
/// Background Silero worker. The realtime callback only enqueues
|
||||
/// 10 ms frames and reads the latest probability atomically.
|
||||
pub struct SileroOnnxVadWorker {
|
||||
tx: Option<std::sync::mpsc::SyncSender<SileroFrameMessage>>,
|
||||
latest_probability: Arc<AtomicU32>,
|
||||
latest_processed_seq: Arc<AtomicU64>,
|
||||
alive: Arc<AtomicBool>,
|
||||
handle: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl SileroOnnxVadWorker {
|
||||
/// Start a background Silero worker if the model loads.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
let vad = SileroOnnxVad::try_new(model_path)?;
|
||||
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
|
||||
let latest_processed_seq = Arc::new(AtomicU64::new(0));
|
||||
let alive = Arc::new(AtomicBool::new(true));
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(8);
|
||||
let latest_probability_for_thread = latest_probability.clone();
|
||||
let latest_processed_seq_for_thread = latest_processed_seq.clone();
|
||||
let alive_for_thread = alive.clone();
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("chanora-silero-vad".to_string())
|
||||
.spawn(move || {
|
||||
let mut vad = super::Resampled16kHzVad::new(vad);
|
||||
while alive_for_thread.load(Ordering::Relaxed) {
|
||||
let message = match rx.recv() {
|
||||
Ok(message) => message,
|
||||
Err(_) => break,
|
||||
};
|
||||
let output = vad.process_10ms(&message.frame);
|
||||
latest_probability_for_thread.store(
|
||||
output.probability.clamp(0.0, 1.0).to_bits(),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
latest_processed_seq_for_thread.store(message.seq, Ordering::Relaxed);
|
||||
}
|
||||
})
|
||||
.ok()?;
|
||||
|
||||
Some(Self {
|
||||
tx: Some(tx),
|
||||
latest_probability,
|
||||
latest_processed_seq,
|
||||
alive,
|
||||
handle: Some(handle),
|
||||
})
|
||||
}
|
||||
|
||||
/// Best-effort enqueue of a 10 ms frame for background inference.
|
||||
pub fn try_send(&self, seq: u64, frame: &[f32; super::resampler::INPUT_FRAME_10MS]) -> bool {
|
||||
let Some(tx) = &self.tx else {
|
||||
return false;
|
||||
};
|
||||
tx.try_send(SileroFrameMessage { seq, frame: *frame })
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// Latest probability published by the background worker.
|
||||
pub fn latest_probability(&self) -> f32 {
|
||||
f32::from_bits(self.latest_probability.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Number of 10 ms frames the worker is behind the capture thread.
|
||||
pub fn lag_frames(&self, capture_seq: u64) -> u64 {
|
||||
capture_seq.saturating_sub(self.latest_processed_seq.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// True when the worker is too far behind to trust its latest
|
||||
/// probability for the current frame.
|
||||
pub fn is_stale(&self, capture_seq: u64) -> bool {
|
||||
self.lag_frames(capture_seq) > SILERO_MAX_STALE_FRAMES
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SileroOnnxVadWorker {
|
||||
fn drop(&mut self) {
|
||||
self.alive.store(false, Ordering::Relaxed);
|
||||
let _ = self.tx.take();
|
||||
let _ = self.handle.take();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_stub_vad() -> SileroOnnxVad {
|
||||
SileroOnnxVad {
|
||||
state: Box::new([0.0; SILERO_STATE_SIZE]),
|
||||
context: Box::new([0.0; SILERO_CONTEXT_16K]),
|
||||
accum: Vec::new(),
|
||||
last_probability: 0.0,
|
||||
model_path: String::new(),
|
||||
inner: SileroInner::Stub,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_new_returns_none_without_model_file() {
|
||||
let result = SileroOnnxVad::try_new("/nonexistent/silero_vad.onnx");
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stub_accumulates_and_holds_zero_probability() {
|
||||
let mut vad = make_stub_vad();
|
||||
let frame = vec![0.0_f32; super::super::resampler::OUTPUT_FRAME_10MS];
|
||||
// Feed 3 frames (30 ms < 32 ms) — no inference yet.
|
||||
for _ in 0..3 {
|
||||
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame);
|
||||
assert_eq!(out.probability, 0.0);
|
||||
}
|
||||
// Feed 1 more frame (40 ms > 32 ms) — accumulator drains.
|
||||
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame);
|
||||
assert_eq!(out.probability, 0.0); // stub stays at 0
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_state_clears_all() {
|
||||
let mut vad = make_stub_vad();
|
||||
vad.state[0] = 1.0;
|
||||
vad.context[0] = 1.0;
|
||||
vad.last_probability = 0.9;
|
||||
vad.accum.push(0.5);
|
||||
vad.reset_state();
|
||||
assert_eq!(vad.state[0], 0.0);
|
||||
assert_eq!(vad.context[0], 0.0);
|
||||
assert_eq!(vad.last_probability, 0.0);
|
||||
assert!(vad.accum.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accumulates_correct_number_of_samples() {
|
||||
let mut vad = make_stub_vad();
|
||||
let frame = vec![0.1_f32; super::super::resampler::OUTPUT_FRAME_10MS]; // 160 samples
|
||||
// 3 × 160 = 480 < 512 — not yet full.
|
||||
for _ in 0..3 {
|
||||
VoiceActivityDetector::process_10ms(&mut vad, &frame);
|
||||
}
|
||||
assert_eq!(vad.accum.len(), 480);
|
||||
// 4th frame: 640 > 512 — inference fires, 128 samples remain.
|
||||
VoiceActivityDetector::process_10ms(&mut vad, &frame);
|
||||
assert_eq!(vad.accum.len(), 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_concatenates_context_before_frame_like_upstream_example() {
|
||||
let mut context = [0.0_f32; SILERO_CONTEXT_16K];
|
||||
context[0] = -1.0;
|
||||
context[SILERO_CONTEXT_16K - 1] = 1.0;
|
||||
let frame = vec![0.25_f32; SILERO_FRAME_16K];
|
||||
|
||||
let input = SileroOnnxVad::input_with_context(&context, &frame);
|
||||
|
||||
assert_eq!(input.len(), SILERO_INPUT_WIDTH);
|
||||
assert_eq!(input[0], -1.0);
|
||||
assert_eq!(input[SILERO_CONTEXT_16K - 1], 1.0);
|
||||
assert_eq!(input[SILERO_CONTEXT_16K], 0.25);
|
||||
assert_eq!(input[SILERO_INPUT_WIDTH - 1], 0.25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn context_tracks_last_64_samples_of_completed_frame() {
|
||||
let mut vad = make_stub_vad();
|
||||
let frame = vec![0.0_f32; super::super::resampler::OUTPUT_FRAME_10MS];
|
||||
for idx in 0..4 {
|
||||
let mut chunk = frame.clone();
|
||||
let chunk_len = chunk.len();
|
||||
for (sample_idx, sample) in chunk.iter_mut().enumerate() {
|
||||
*sample = (idx * chunk_len + sample_idx) as f32;
|
||||
}
|
||||
VoiceActivityDetector::process_10ms(&mut vad, &chunk);
|
||||
}
|
||||
|
||||
let completed_frame: Vec<f32> = (0..SILERO_FRAME_16K).map(|v| v as f32).collect();
|
||||
assert_eq!(
|
||||
vad.context.as_ref(),
|
||||
&completed_frame[SILERO_FRAME_16K - SILERO_CONTEXT_16K..]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
//! Voice-activity state machine for P1 transmit gating.
|
||||
//!
|
||||
//! Implements the full P1 VAD gate policy:
|
||||
//! * backend/model speech decisions — no custom probability thresholds.
|
||||
//! * `open_after_ms` — speech must be detected for this long before
|
||||
//! the gate opens (prevents false opens on transients). Default 40 ms.
|
||||
//! * `hangover_ms` — gate stays open for this long after speech drops
|
||||
//! out of the backend decision (prevents choppy transmit close). Default 500 ms.
|
||||
//! * `min_tx_ms` — minimum transmit duration after gate opens. Default 200 ms.
|
||||
//!
|
||||
//! Pre-roll (first-syllable preservation) is handled in the capture
|
||||
//! pipeline, not here. The state machine only decides whether the gate
|
||||
//! is open or closed.
|
||||
|
||||
/// Shared VAD timing constants and gate state machine.
|
||||
///
|
||||
/// Exposing these values here keeps the audio config and platform
|
||||
/// capture paths aligned without repeating the same magic numbers in
|
||||
/// multiple modules.
|
||||
/// Default confirmation window before the gate opens, in milliseconds.
|
||||
pub const VAD_OPEN_AFTER_MS: u32 = 40;
|
||||
/// Default hangover duration in milliseconds.
|
||||
pub const VAD_HANGOVER_MS: u32 = 500;
|
||||
/// Default minimum transmit duration in milliseconds.
|
||||
pub const VAD_MIN_TX_MS: u32 = 200;
|
||||
|
||||
/// Hangover/open-after/minimum-transmit state machine.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VoiceActivityStateMachine {
|
||||
/// Frames of continuous speech required before gate opens.
|
||||
open_after_frames: u32,
|
||||
hangover_frames: u32,
|
||||
min_tx_frames: u32,
|
||||
active: bool,
|
||||
hangover_remaining: u32,
|
||||
min_tx_remaining: u32,
|
||||
/// Frames of continuous speech seen since last open attempt.
|
||||
open_confirm_frames: u32,
|
||||
/// Frames spent open without a strong speech score. This keeps
|
||||
/// stale or borderline VAD output from holding the mic open forever.
|
||||
weak_hold_frames: u32,
|
||||
}
|
||||
|
||||
impl VoiceActivityStateMachine {
|
||||
/// Create a state machine. Frame duration is 10 ms.
|
||||
pub fn new(open_after_ms: u32, hangover_ms: u32, min_tx_ms: u32) -> Self {
|
||||
Self {
|
||||
open_after_frames: open_after_ms / 10,
|
||||
hangover_frames: hangover_ms / 10,
|
||||
min_tx_frames: min_tx_ms / 10,
|
||||
active: false,
|
||||
hangover_remaining: 0,
|
||||
min_tx_remaining: 0,
|
||||
open_confirm_frames: 0,
|
||||
weak_hold_frames: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Update timers without forcing a close. Used by
|
||||
/// live settings changes while audio is already running.
|
||||
pub fn configure(&mut self, open_after_ms: u32, hangover_ms: u32, min_tx_ms: u32) {
|
||||
self.open_after_frames = open_after_ms / 10;
|
||||
self.hangover_frames = hangover_ms / 10;
|
||||
self.min_tx_frames = min_tx_ms / 10;
|
||||
self.hangover_remaining = self.hangover_remaining.min(self.hangover_frames);
|
||||
self.min_tx_remaining = self.min_tx_remaining.min(self.min_tx_frames);
|
||||
}
|
||||
|
||||
/// Advance by one 10 ms backend speech decision and return whether
|
||||
/// transmit should be open for VoiceActivity mode.
|
||||
pub fn update(&mut self, speech: bool) -> bool {
|
||||
if self.active {
|
||||
if self.min_tx_remaining > 0 {
|
||||
self.min_tx_remaining -= 1;
|
||||
}
|
||||
if speech {
|
||||
self.weak_hold_frames = 0;
|
||||
} else {
|
||||
self.weak_hold_frames = self.weak_hold_frames.saturating_add(1);
|
||||
}
|
||||
if self.weak_hold_frames >= self.weak_hold_limit_frames() && self.min_tx_remaining == 0
|
||||
{
|
||||
self.close();
|
||||
return false;
|
||||
}
|
||||
if speech {
|
||||
// Speech still present — reset hangover.
|
||||
self.hangover_remaining = self.hangover_frames;
|
||||
} else if self.hangover_remaining > 0 {
|
||||
self.hangover_remaining -= 1;
|
||||
} else if self.min_tx_remaining == 0 {
|
||||
// Hangover expired and min-tx elapsed — close gate.
|
||||
self.close();
|
||||
}
|
||||
} else {
|
||||
// Gate is closed. Accumulate confirmation frames.
|
||||
if speech {
|
||||
self.open_confirm_frames += 1;
|
||||
if self.open_confirm_frames >= self.open_after_frames.max(1) {
|
||||
// Confirmed speech — open gate.
|
||||
self.active = true;
|
||||
self.hangover_remaining = self.hangover_frames;
|
||||
self.min_tx_remaining = self.min_tx_frames;
|
||||
self.open_confirm_frames = 0;
|
||||
self.weak_hold_frames = 0;
|
||||
}
|
||||
} else {
|
||||
// Speech is no longer detected — reset confirmation.
|
||||
self.open_confirm_frames = 0;
|
||||
}
|
||||
}
|
||||
self.active
|
||||
}
|
||||
|
||||
/// Current active state.
|
||||
pub fn active(&self) -> bool {
|
||||
self.active
|
||||
}
|
||||
|
||||
fn weak_hold_limit_frames(&self) -> u32 {
|
||||
(self.hangover_frames + self.min_tx_frames + self.open_after_frames).clamp(30, 100)
|
||||
}
|
||||
|
||||
fn close(&mut self) {
|
||||
self.active = false;
|
||||
self.hangover_remaining = 0;
|
||||
self.min_tx_remaining = 0;
|
||||
self.open_confirm_frames = 0;
|
||||
self.weak_hold_frames = 0;
|
||||
}
|
||||
|
||||
/// Reset all state (call on session restart / voice_leave).
|
||||
pub fn reset(&mut self) {
|
||||
self.close();
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for VoiceActivityStateMachine {
|
||||
fn default() -> Self {
|
||||
Self::new(VAD_OPEN_AFTER_MS, VAD_HANGOVER_MS, VAD_MIN_TX_MS)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hangover_keeps_gate_open_after_close() {
|
||||
// open_after_ms=0 so gate opens immediately on first frame.
|
||||
let mut sm = VoiceActivityStateMachine::new(0, 30, 0);
|
||||
assert!(sm.update(true));
|
||||
assert!(sm.update(false));
|
||||
assert!(sm.update(false));
|
||||
assert!(sm.update(false));
|
||||
assert!(!sm.update(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_after_requires_confirmation_frames() {
|
||||
// open_after_ms=20 → 2 frames required.
|
||||
let mut sm = VoiceActivityStateMachine::new(20, 0, 0);
|
||||
// First frame: not yet open.
|
||||
assert!(!sm.update(true));
|
||||
// Second frame: now open.
|
||||
assert!(sm.update(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_after_resets_on_silence() {
|
||||
// open_after_ms=20 → 2 frames required.
|
||||
let mut sm = VoiceActivityStateMachine::new(20, 0, 0);
|
||||
assert!(!sm.update(true)); // 1 frame
|
||||
assert!(!sm.update(false)); // silence resets counter
|
||||
assert!(!sm.update(true)); // 1 frame again
|
||||
assert!(sm.update(true)); // 2nd frame → open
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_tx_keeps_gate_open_briefly() {
|
||||
// open_after_ms=0, hangover=0, min_tx=20ms (2 frames).
|
||||
// After opening: min_tx_remaining decrements each frame.
|
||||
// Gate closes on the frame where it reaches 0.
|
||||
let mut sm = VoiceActivityStateMachine::new(0, 0, 20);
|
||||
assert!(sm.update(true)); // opens; min_tx_remaining=2
|
||||
assert!(sm.update(false)); // min_tx_remaining=1; still open
|
||||
assert!(!sm.update(false)); // min_tx_remaining=0; gate closes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_all_state() {
|
||||
let mut sm = VoiceActivityStateMachine::new(0, 100, 0);
|
||||
assert!(sm.update(true)); // open
|
||||
sm.reset();
|
||||
assert!(!sm.active());
|
||||
// After reset, gate should not be open even with hangover pending.
|
||||
assert!(!sm.update(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_uses_p1_spec_values() {
|
||||
let sm = VoiceActivityStateMachine::default();
|
||||
assert_eq!(sm.open_after_frames, VAD_OPEN_AFTER_MS / 10);
|
||||
assert_eq!(sm.hangover_frames, VAD_HANGOVER_MS / 10);
|
||||
assert_eq!(sm.min_tx_frames, VAD_MIN_TX_MS / 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_config_update_shortens_existing_hangover() {
|
||||
let mut sm = VoiceActivityStateMachine::new(0, 1000, 0);
|
||||
assert!(sm.update(true));
|
||||
assert!(sm.update(false));
|
||||
sm.configure(0, 100, 0);
|
||||
for _ in 0..10 {
|
||||
assert!(sm.update(false));
|
||||
}
|
||||
assert!(!sm.update(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_closed_decisions_cannot_hold_gate_forever() {
|
||||
let mut sm = VoiceActivityStateMachine::new(0, 500, 0);
|
||||
assert!(sm.update(true));
|
||||
for _ in 0..49 {
|
||||
assert!(sm.update(false));
|
||||
}
|
||||
assert!(!sm.update(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speech_decision_resets_weak_hold_limit() {
|
||||
let mut sm = VoiceActivityStateMachine::new(0, 500, 0);
|
||||
assert!(sm.update(true));
|
||||
for _ in 0..40 {
|
||||
assert!(sm.update(false));
|
||||
}
|
||||
assert!(sm.update(true));
|
||||
for _ in 0..40 {
|
||||
assert!(sm.update(false));
|
||||
}
|
||||
assert!(sm.active());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
/// Diagnostics returned by render downmix helpers.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub(crate) struct RenderDownmixStats {
|
||||
/// Peak absolute sample magnitude after i16 conversion.
|
||||
pub peak_i16: i16,
|
||||
/// Samples clipped while applying output gain.
|
||||
pub clipped_samples: u64,
|
||||
}
|
||||
|
||||
/// Downmix interleaved stereo f32 samples into mono i16 samples.
|
||||
///
|
||||
/// The helper is allocation-free and safe for realtime render callbacks.
|
||||
/// If the stereo source is shorter than expected, the remainder of `out`
|
||||
/// is filled with silence.
|
||||
pub(crate) fn downmix_stereo_f32_to_mono_i16(
|
||||
stereo: &[f32],
|
||||
out: &mut [i16],
|
||||
gain: f32,
|
||||
muted: bool,
|
||||
) -> RenderDownmixStats {
|
||||
if muted {
|
||||
out.fill(0);
|
||||
return RenderDownmixStats::default();
|
||||
}
|
||||
|
||||
let available_frames = stereo.len() / 2;
|
||||
if available_frames < out.len() {
|
||||
out.fill(0);
|
||||
}
|
||||
|
||||
let mut peak = 0_u16;
|
||||
let mut clipped_samples = 0_u64;
|
||||
for (dst, lr) in out.iter_mut().zip(stereo.chunks_exact(2)) {
|
||||
let mono = (lr[0] + lr[1]) * 0.5 * gain;
|
||||
let clamped = mono.clamp(-1.0, 1.0);
|
||||
if (mono - clamped).abs() > f32::EPSILON {
|
||||
clipped_samples = clipped_samples.saturating_add(1);
|
||||
}
|
||||
let sample = (clamped * i16::MAX as f32) as i16;
|
||||
*dst = sample;
|
||||
peak = peak.max(sample.unsigned_abs());
|
||||
}
|
||||
|
||||
RenderDownmixStats {
|
||||
peak_i16: peak.min(i16::MAX as u16) as i16,
|
||||
clipped_samples,
|
||||
}
|
||||
}
|
||||
|
||||
/// Downmix interleaved stereo f32 samples into mono f32 samples.
|
||||
///
|
||||
/// Used for software-AEC render references and debug WAV taps.
|
||||
#[cfg(any(target_os = "ios", test))]
|
||||
pub(crate) fn downmix_stereo_f32_to_mono_f32(stereo: &[f32], out: &mut [f32]) {
|
||||
let available_frames = stereo.len() / 2;
|
||||
if available_frames < out.len() {
|
||||
out.fill(0.0);
|
||||
}
|
||||
|
||||
for (dst, lr) in out.iter_mut().zip(stereo.chunks_exact(2)) {
|
||||
*dst = (lr[0] + lr[1]) * 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn downmix_i16_applies_gain_and_reports_clipping() {
|
||||
let stereo = [1.0_f32, 1.0, 0.25, -0.25, -2.0, -2.0];
|
||||
let mut out = [0_i16; 3];
|
||||
|
||||
let stats = downmix_stereo_f32_to_mono_i16(&stereo, &mut out, 2.0, false);
|
||||
|
||||
assert_eq!(out[0], i16::MAX);
|
||||
assert_eq!(out[1], 0);
|
||||
assert_eq!(out[2], -i16::MAX);
|
||||
assert_eq!(stats.peak_i16, i16::MAX);
|
||||
assert_eq!(stats.clipped_samples, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downmix_i16_mutes_output() {
|
||||
let stereo = [1.0_f32, 1.0, -1.0, -1.0];
|
||||
let mut out = [123_i16; 2];
|
||||
|
||||
let stats = downmix_stereo_f32_to_mono_i16(&stereo, &mut out, 1.0, true);
|
||||
|
||||
assert_eq!(out, [0, 0]);
|
||||
assert_eq!(stats, RenderDownmixStats::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downmix_f32_fills_missing_tail_with_silence() {
|
||||
let stereo = [1.0_f32, -1.0];
|
||||
let mut out = [9.0_f32; 2];
|
||||
|
||||
downmix_stereo_f32_to_mono_f32(&stereo, &mut out);
|
||||
|
||||
assert_eq!(out, [0.0, 0.0]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user