feat: Android Oboe voice backend — WebRTC APM, VAD, HW/SW toggle, BBCode welcome, link trust, foreground task

Audio engine (Rust):
- Android Oboe: WebRTC APM (AEC/NS/AGC/HPF) + TEN/Silero ONNX VAD
- Hardware effects (JNI) with software fallback per-effect
- Render reference buffer for AEC between output/capture callbacks
- Voice activity gate: suppress transmission when speaker muted (all platforms)
- Audio focus (SDD-109) + Bluetooth SCO (SDD-110) via JNI
- ONNX Runtime 1.26 via ort 2.0.0-rc.12 (down from rc.10, ndarray 0.17)
- VAD worker channel capacity 8→32, initial seq u64::MAX (warm-up fix)
- TEN VAD default backend (was Silero)
- Platform→WebrtcApm resolution after hardware binding
- oboe-rs edisonjwa fork with get_raw_session_id()

Android Kotlin:
- AndroidAudioFocusController + AndroidBluetoothScoController
- AndroidAudioLifecycleController (route changes to Flutter)
- ProGuard rules for new controllers

Flutter UI:
- VoiceSettings: Android HW/SW toggle (Platform auto / WebRTC APM)
- VoiceStatusChip: mute warning border + Speaker muted label
- BBCode welcome message parser (BbCodeText, case-insensitive)
- Welcome message foldable (expanded by default)
- Link trust dialog (domain wildcards, SharedPreferences)
- HapticFeedback on voice sheet opener
- Server name in AppBar, version v0.1.0
- Default channel (id=1) visible, serverquery clients hidden
- flutter_foreground_task integration

Config:
- ort load-dynamic on all non-iOS (Android/Linux/Windows)
- ONNX Runtime AAR 1.26.0
- ndarray moved to common deps (was Apple-only)
This commit is contained in:
Edison Jwa
2026-05-22 09:29:57 +09:00
parent 6af4ecab0f
commit bf284018e6
37 changed files with 3676 additions and 703 deletions
+17 -8
View File
@@ -13,6 +13,12 @@ publish.workspace = true
chanora_protocol = { path = "../chanora_protocol" }
thiserror.workspace = true
tracing.workspace = true
sonora = "0.1"
webrtc-vad = "0.4"
# ndarray is required by ort's tensor construction API and by
# Silero / TEN VAD ONNX inference across all platforms.
ndarray = "0.17"
# Opus encoder. tsclientlib already pulls this; we depend explicitly so
# this crate can compile against it without going through tsclientlib.
@@ -47,19 +53,15 @@ coreaudio-rs = "0.14"
# Grand Central Dispatch bindings — used to run AudioUnit initialize/start
# on the main queue to avoid the VPIO RPC timeout on iOS simulator.
dispatch2 = "0.3"
# ndarray is required by ort's tensor construction API.
ndarray = "0.16"
[target.'cfg(target_os = "ios")'.dependencies]
# ONNX Runtime Rust binding for Silero VAD v6 (P1 VAD_002). The official
# iOS CocoaPod ships ONNX Runtime as a static framework, so iOS links it
# into chanora_bridge at build time instead of loading a dylib at runtime.
ort = { version = "2.0.0-rc.10", default-features = false, features = ["std", "ndarray"] }
ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray"] }
[target.'cfg(target_os = "macos")'.dependencies]
# macOS keeps dynamic loading so developer machines can provide ORT via
# ORT_DYLIB_PATH without forcing a bundled runtime into desktop builds.
ort = { version = "2.0.0-rc.10", default-features = false, features = ["load-dynamic", "ndarray"] }
[target.'cfg(not(target_os = "ios"))'.dependencies]
ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "api-24"] }
[target.'cfg(target_os = "android")'.dependencies]
# Android cross-builds should not pull OpenSSL. Use rustls here while keeping
@@ -75,7 +77,14 @@ ndk-context = "0.1"
# shipped with `oboe-sys` 0.6 covers armv7 / aarch64 / x86 / x86_64.
# Default features keep the precompiled library + pregenerated bindings
# so we avoid the clang-sys / libclang requirement on the build host.
oboe = "0.6"
#
# Using local fork edisonjwa/oboe-rs (v0.6.2) with:
# - catch_unwind safety in callbacks
# - unwrap_or_default in enum getters (no more SessionId panic)
# - get_raw_session_id() for JNI hardware effect binding
# - deduplicated macro impls
# - PowerSavingOffloaded PerformanceMode variant
oboe = { path = "../../../oboe-rs" }
[target.'cfg(target_os = "windows")'.dependencies]
# Real Windows global PTT (SDD-083 / SDD-084): RegisterRawInputDevices
+631 -45
View File
@@ -66,72 +66,194 @@ use oboe::{
PerformanceMode, SessionId, SharingMode, Usage,
};
use crate::processor::AudioProcessor;
// `BackendEvent` / `BackendEventRx` / `BackendEventTx` moved to
// `mobile_voice_backend` so the trait can expose `take_event_rx`
// (SDD-111 item 1) cross-platform.
// --- Render-reference buffer for AEC (SDD-111 / SDD-120) ---------
//
// The output (render) callback writes the audio that will be played
// into this ring buffer. The capture callback reads the latest render
// frame and feeds it to WebRTC APM's `process_render` so AEC can
// subtract the speaker output from the microphone input.
//
// 4 slots × 10 ms × 48 kHz mono f32. One slot is always being written
// by the render callback; the capture callback reads the slot that was
// most recently completed.
const RENDER_REF_SLOTS: usize = 4;
const RENDER_REF_SAMPLES: usize = crate::frame::FRAME_10MS_SAMPLES;
struct RenderReferenceBuffer {
buf: Box<[[f32; RENDER_REF_SAMPLES]; RENDER_REF_SLOTS]>,
write_idx: std::sync::atomic::AtomicUsize,
}
impl RenderReferenceBuffer {
fn new() -> Arc<Self> {
Arc::new(Self {
buf: Box::new([[0.0_f32; RENDER_REF_SAMPLES]; RENDER_REF_SLOTS]),
write_idx: std::sync::atomic::AtomicUsize::new(0),
})
}
fn write(&self, frame: &[f32; RENDER_REF_SAMPLES]) {
let idx = self.write_idx.load(Ordering::Relaxed);
unsafe {
let slot =
&self.buf[idx] as *const [f32; RENDER_REF_SAMPLES] as *mut [f32; RENDER_REF_SAMPLES];
(*slot).copy_from_slice(frame);
}
self.write_idx
.store((idx + 1) % RENDER_REF_SLOTS, Ordering::Relaxed);
}
fn read_latest(&self) -> [f32; RENDER_REF_SAMPLES] {
let wi = self.write_idx.load(Ordering::Relaxed);
let ri = (wi + RENDER_REF_SLOTS - 1) % RENDER_REF_SLOTS;
self.buf[ri]
}
}
unsafe impl Send for RenderReferenceBuffer {}
unsafe impl Sync for RenderReferenceBuffer {}
// --- Capture state for Oboe input callback (SDD-111 / SDD-120) ----
//
// Mirrors the iOS `IosCaptureState` and the cpal-side `CaptureState`.
// Oboe delivers 48 kHz mono i16 PCM; we apply mic gain, accumulate to
// FRAME_20MS_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
// and try-send the resulting packet on `voice_out_tx`.
// Enhanced with WebRTC APM (AEC/NS/AGC) and VAD (voice activity
// detection). Oboe delivers 48 kHz mono i16 PCM in variable-size
// chunks. We accumulate into 10 ms frames, then:
//
// 1. i16 → f32 conversion
// 2. Read render reference (for AEC)
// 3. WebRtcApmProcessor::process_render + process_capture
// 4. VAD → VoiceActivityStateMachine → TransmitModeSelector
// 5. f32 → i16 conversion + mic gain
// 6. Accumulate to 20 ms → Opus encode → send
struct AndroidCaptureState {
encoder: OpusEncoder,
/// Accumulator for 48 kHz mono PCM. 2x capacity to absorb
/// cpal-style buffer-size jitter without reallocating.
pcm_accum: Vec<i16>,
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: usize,
fallback_warned_backend: Option<crate::VadBackend>,
}
impl AndroidCaptureState {
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("android")?;
// Android always uses software WebRTC APM for AEC/NS/AGC/HPF.
// The config's EffectOwner fields are resolved by open() AFTER
// hardware-effect binding; the processor is constructed here
// with all modules enabled regardless, so the resolved config
// (Platform vs WebrtcApm) only affects diagnostics, not behaviour.
let webrtc_apm_config = audio_processing_config
.lock()
.map(|cfg| {
let mut c = crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg);
c.aec = true;
c.ns = true;
c.agc = true;
c
})
.unwrap_or(crate::processor::webrtc_apm::WebRtcApmConfig {
aec: true,
ns: true,
agc: true,
hpf: true,
..Default::default()
});
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
output_muted,
frames_sent,
mic_gain,
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
webrtc_apm_config,
)?,
audio_processing_config,
audio_processing_stats,
render_reference,
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
fallback_warned_backend: None,
})
}
/// Consume i16 mono frames from Oboe, accumulate to FRAME_20MS_SAMPLES,
/// encode + send when PTT is held. Oboe delivers at the device's
/// native sample rate (always 48 kHz for modern Android per SRS-210),
/// so no resampling is needed.
fn ingest(&mut self, samples: &[i16]) {
/// Consume i16 mono frames from Oboe. Accumulate to 10 ms chunks,
/// process each through WebRTC APM + VAD, then encode 20 ms frames.
fn ingest_i16(&mut self, samples: &[i16]) {
let mut offset = 0;
while offset < samples.len() {
let remaining =
crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let take = remaining.min(samples.len() - offset);
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
.copy_from_slice(&samples[offset..offset + take]);
self.pending_10ms_len += take;
offset += take;
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
let frame = self.pending_10ms;
self.process_10ms_capture_frame(&frame);
self.pending_10ms_len = 0;
}
}
if !self.transmit_active.load(Ordering::Relaxed) {
self.pcm_accum.clear();
return;
}
// Mic-gain application.
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
self.pcm_accum.extend_from_slice(samples);
} else {
let gain = self.mic_gain;
self.pcm_accum.extend(samples.iter().map(|&s| {
let scaled = (s as f32) * gain;
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
}));
}
// Drain complete 20 ms frames.
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
frame.copy_from_slice(
&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES],
);
self.pcm_accum
.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
crate::opus_voice::send_voip_frame(
@@ -154,11 +276,186 @@ impl AndroidCaptureState {
);
}
Err(e) => {
warn!(target: "chanora_audio", error = %e, "android Oboe opus encode failed");
warn!(
target: "chanora_audio",
error = %e,
"android Oboe opus encode failed"
);
}
}
}
}
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend != Some(failed_backend) {
self.fallback_warned_backend = Some(failed_backend);
warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"android: VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
}
fn process_10ms_capture_frame(
&mut self,
samples: &[i16; crate::frame::FRAME_10MS_SAMPLES],
) {
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
*dst = crate::frame::i16_to_f32(src);
}
let input_dbfs = crate::frame::dbfs(&frame);
let render_ref = self.render_reference.read_latest();
self.webrtc_apm_processor.process_render(&render_ref);
self.webrtc_apm_processor.process_capture(&mut frame);
let (vad_hangover, vad_backend) = self
.audio_processing_config
.try_lock()
.map(|cfg| (cfg.vad_hangover_ms, cfg.vad_backend))
.unwrap_or((
crate::voice_activity::VAD_HANGOVER_MS,
crate::VadBackend::WebrtcVad,
));
self.vad_state.configure(
crate::voice_activity::VAD_OPEN_AFTER_MS,
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
// VAD backend switching (mirrors iOS Raw path).
let silero_epoch = crate::vad::silero_model_epoch();
let silero_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
let path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
self.ten_vad_worker = None;
if self.silero_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"android: Silero VAD model not found at {path}; falling back to WebRTC VAD"
);
}
}
crate::VadBackend::TenVad => {
let path = crate::vad::ten_model_bundle_path();
self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path);
self.silero_vad_worker = None;
if self.ten_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"android: TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD"
);
}
}
_ => {
self.silero_vad_worker = None;
self.ten_vad_worker = None;
}
}
self.vad_state.reset();
}
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let active = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(sel) = &self.voice_activity_selector {
sel.set_voice_activity_open(active && !output_muted);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
active && !output_muted,
self.transmit_active.load(Ordering::Relaxed),
);
if !self.transmit_active.load(Ordering::Relaxed) || output_muted {
return;
}
let gain = self.mic_gain;
if (gain - 1.0).abs() < f32::EPSILON {
self.pcm_accum
.extend(frame.iter().copied().map(crate::frame::f32_to_i16));
} else {
self.pcm_accum.extend(frame.iter().copied().map(|s| {
let scaled = (crate::frame::f32_to_i16(s) as f32) * gain;
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
}));
}
}
}
struct InputCallback {
@@ -176,7 +473,7 @@ impl AudioInputCallback for InputCallback {
) -> DataCallbackResult {
let _ = catch_unwind(AssertUnwindSafe(|| {
if let Ok(mut s) = self.state.lock() {
s.ingest(frames);
s.ingest_i16(frames);
}
}));
DataCallbackResult::Continue
@@ -203,6 +500,7 @@ struct OutputCallback {
output_muted: Arc<AtomicBool>,
event_tx: BackendEventTx,
scratch: Arc<Mutex<Vec<f32>>>,
render_reference: Arc<RenderReferenceBuffer>,
}
impl AudioOutputCallback for OutputCallback {
@@ -223,16 +521,17 @@ impl AudioOutputCallback for OutputCallback {
*s = 0.0;
}
}
// Non-blocking pull from AudioHandler (same pattern as iOS VPIO).
match self.handler.try_lock() {
Ok(mut h) => {
let _ = h.fill_buffer(&mut scratch[..needed]);
}
Err(std::sync::TryLockError::WouldBlock) => {
// scratch already zeroed above.
}
Err(std::sync::TryLockError::WouldBlock) => {}
Err(std::sync::TryLockError::Poisoned(e)) => {
warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {}", e);
warn!(
target: "chanora_audio",
"AudioHandler mutex poisoned: {}",
e
);
}
}
let gain = f32::from_bits(self.output_gain.load(Ordering::Relaxed));
@@ -243,6 +542,19 @@ impl AudioOutputCallback for OutputCallback {
gain,
muted,
);
// Write the first 10 ms of render audio into the reference
// buffer for the capture-side AEC.
let mono_n = needed / 2;
let render_n = mono_n.min(crate::frame::FRAME_10MS_SAMPLES);
let mut ref_frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (i, chunk) in scratch[..render_n * 2].chunks_exact(2).enumerate() {
if i >= render_n {
break;
}
ref_frame[i] = (chunk[0] + chunk[1]) * 0.5;
}
self.render_reference.write(&ref_frame);
}));
DataCallbackResult::Continue
}
@@ -273,7 +585,7 @@ pub struct VoiceAudioParams {
pub frames_sent: Arc<AtomicU32>,
/// Pre-encode amplitude scale (1.0 = unity).
pub mic_gain: f32,
/// AudioHandler that inbound解码+混合 feeds into; the Oboe output
/// AudioHandler that inbound decode+mix feeds into; the Oboe output
/// callback pulls mixed stereo f32 from it.
pub handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
/// Master output gain (f32 bits stored in AtomicU32 for lock-free
@@ -281,6 +593,14 @@ pub struct VoiceAudioParams {
pub output_gain: Arc<AtomicU32>,
/// True = output silence regardless of incoming voice frames.
pub output_muted: Arc<AtomicBool>,
/// Optional TransmitModeSelector for VoiceActivity transmit mode.
/// The capture callback calls set_voice_activity_open on this when
/// VAD detects speech. None means VoiceActivity mode is disabled.
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
/// Shared audio-processing config (WebRTC APM flags, VAD backend).
pub audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
/// Shared audio-processing statistics for diagnostics.
pub audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
}
// --- The backend itself ------------------------------------------
@@ -334,22 +654,30 @@ impl AndroidVoiceUnit {
) -> Result<Self, BackendError> {
let (event_tx, event_rx) = mpsc::unbounded_channel();
// SDD-120: build the capture state that the Oboe input callback
// will own via Arc<Mutex>. Same Opus VoIP tuning as iOS and
// desktop (32 kbps, complexity 10, inband FEC, 5 % PLC).
// Shared render-reference buffer (for AEC). The output
// callback writes; the capture callback reads.
let render_ref_buf = RenderReferenceBuffer::new();
let render_ref_for_capture = render_ref_buf.clone();
// Clone the APM config Arc before params is partially moved
// into the capture state constructor below.
let apm_config_clone = params.audio_processing_config.clone();
let capture_state = Arc::new(Mutex::new(
AndroidCaptureState::new(
params.voice_out_tx,
params.transmit_active,
params.output_muted.clone(),
params.frames_sent,
params.mic_gain,
params.voice_activity_selector,
params.audio_processing_config,
params.audio_processing_stats,
render_ref_for_capture,
)
.map_err(|e| BackendError::OpenFailed(format!("capture state init: {e}")))?,
));
// Scratch buffer for the output callback (realtime-safe
// pre-allocation). 8192 floats covers the largest practical
// burst size at 48 kHz with headroom.
let scratch = Arc::new(Mutex::new(Vec::with_capacity(8192)));
// --- Open input stream (SDD-112) ---------------------------
@@ -423,13 +751,19 @@ impl AndroidVoiceUnit {
.as_mut()
.map(|s| s.get_frames_per_burst())
.unwrap_or(0);
let session_id = None;
warn!(
target: "chanora_audio",
"android: oboe-rs get_session_id is skipped because oboe 0.6.1 \
panics on some Android allocated-session values; hardware \
effects are disabled for this stream"
);
// The oboe-rs fork (edisonjwa/oboe-rs 0.6.2) fixes the
// get_session_id() panic with unwrap_or_default(), but the
// SessionId enum still only models builder parameters (None =
// -1, Allocate = 0). The actual system audio session ID (>0)
// written by AAudio to mSessionId after stream open cannot be
// expressed in the current enum; get_raw_session_id() is a
// follow-up addition to the fork.
//
// For now: hardware effects require the system session ID.
// WebRTC APM software processing handles AEC/NS/AGC/HPF.
let session_id: Option<i32> = input_stream
.as_ref()
.and_then(|s| s.get_raw_session_id());
// --- Open output stream (SDD-112) --------------------------
let output_builder = AudioStreamBuilder::default()
@@ -450,12 +784,14 @@ impl AndroidVoiceUnit {
.set_usage(Usage::VoiceCommunication)
.set_content_type(oboe::ContentType::Speech);
let render_ref_for_output = render_ref_buf.clone();
let output_cb = OutputCallback {
handler: params.handler.clone(),
output_gain: params.output_gain.clone(),
output_muted: params.output_muted.clone(),
event_tx: event_tx.clone(),
scratch: scratch.clone(),
render_reference: render_ref_for_output,
};
let output_builder = output_builder.set_callback(output_cb);
@@ -474,6 +810,7 @@ impl AndroidVoiceUnit {
params.output_gain.clone(),
params.output_muted.clone(),
scratch.clone(),
render_ref_buf,
)?
}
};
@@ -511,6 +848,52 @@ impl AndroidVoiceUnit {
HardwareEffectHandles::default()
};
// --- SDD-113 config resolution: hardware-available? -------
// Android gives the user a choice between hardware (JNI) and
// software (WebRTC APM) effects. The AudioProcessingConfig's
// EffectOwner fields encode that choice:
// Platform → prefer hardware; software fallback if missing
// WebrtcApm → always software WebRTC APM
// Off → disable effect entirely
//
// Here we resolve Platform → WebrtcApm for each effect whose
// hardware binding failed (or wasn't attempted). This is read
// by the capture callback's WebRtcApmProcessor.
{
use crate::audio_processing::EffectOwner;
let mut apm_cfg = apm_config_clone.lock().unwrap();
let hw_aec = hw_effects.aec.is_some();
let hw_ns = hw_effects.ns.is_some();
let hw_agc = hw_effects.agc.is_some();
if apm_cfg.aec == EffectOwner::Platform && !hw_aec {
apm_cfg.aec = EffectOwner::WebrtcApm;
}
if apm_cfg.ns == EffectOwner::Platform && !hw_ns {
apm_cfg.ns = EffectOwner::WebrtcApm;
}
if apm_cfg.agc == EffectOwner::Platform && !hw_agc {
apm_cfg.agc = EffectOwner::WebrtcApm;
}
if apm_cfg.processing_backend
== crate::audio_processing::AudioBackend::PlatformVoiceProcessing
&& (!hw_aec || !hw_ns || !hw_agc)
{
apm_cfg.processing_backend = crate::audio_processing::AudioBackend::WebrtcApm;
}
info!(
target: "chanora_audio",
aec = ?apm_cfg.aec,
ns = ?apm_cfg.ns,
agc = ?apm_cfg.agc,
hpf = apm_cfg.hpf_enabled,
hw_aec,
hw_ns,
hw_agc,
session_id,
"android: audio processing config resolved (hardware effects: aec={hw_aec} ns={hw_ns} agc={hw_agc})"
);
}
// --- SDD-112 item 10 / SDD-113 item 7 / SDD-116 item 3 ---
// Publish the diagnostics snapshot. Per-effect engagement is
// derived from (a) the JNI handle (`Hardware`) or (b) the
@@ -647,6 +1030,7 @@ impl AndroidVoiceUnit {
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
scratch: Arc<Mutex<Vec<f32>>>,
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<AudioStreamAsync<OboeOutput, OutputCallback>, BackendError> {
let cb = OutputCallback {
handler,
@@ -654,6 +1038,7 @@ impl AndroidVoiceUnit {
output_muted,
event_tx: event_tx.clone(),
scratch,
render_reference,
};
let builder = AudioStreamBuilder::default()
.set_direction::<OboeOutput>()
@@ -806,6 +1191,7 @@ fn perf_from_oboe(p: PerformanceMode) -> AchievedPerformanceMode {
match p {
PerformanceMode::LowLatency => AchievedPerformanceMode::LowLatency,
PerformanceMode::PowerSaving => AchievedPerformanceMode::PowerSaving,
PerformanceMode::PowerSavingOffloaded => AchievedPerformanceMode::PowerSaving,
PerformanceMode::None => AchievedPerformanceMode::None,
}
}
@@ -1041,6 +1427,51 @@ fn release_hardware_effects_inner(handles: &mut HardwareEffectHandles) {
}
}
// --- Process-global BackendEvent sender for JNI callbacks --------
//
// Kotlin-side listeners (audio focus, Bluetooth SCO, device route
// changes) need to publish events into the Rust engine's event
// channel. Since the engine's `BackendEventTx` is created at voice
// start, we store it here as a process-global so the JNI callbacks
// can reach it without holding a direct Rust reference.
//
// Cleared on voice stop; the Kotlin listeners are idempotent when
// no sender is registered (they log and continue).
static GLOBAL_BACKEND_EVENT_TX: std::sync::OnceLock<
std::sync::Mutex<Option<tokio::sync::mpsc::UnboundedSender<BackendEvent>>>,
> = std::sync::OnceLock::new();
fn global_event_tx_slot(
) -> &'static std::sync::Mutex<Option<tokio::sync::mpsc::UnboundedSender<BackendEvent>>> {
GLOBAL_BACKEND_EVENT_TX.get_or_init(|| std::sync::Mutex::new(None))
}
pub(crate) fn register_global_event_sender(tx: BackendEventTx) {
if let Ok(mut g) = global_event_tx_slot().lock() {
*g = Some(tx);
}
}
pub(crate) fn clear_global_event_sender() {
if let Ok(mut g) = global_event_tx_slot().lock() {
*g = None;
}
}
fn try_send_backend_event(event: BackendEvent) {
if let Ok(g) = global_event_tx_slot().lock() {
if let Some(tx) = g.as_ref() {
if tx.send(event).is_err() {
warn!(
target: "chanora_audio",
"android: global BackendEvent channel closed; event dropped"
);
}
}
}
}
// --- SDD-115 foreground-service JNI helpers ----------------------
//
// The Kotlin class `AndroidVoiceForegroundService` (Wave 2B-2)
@@ -1174,3 +1605,158 @@ fn load_app_class<'local>(
}
}
}
// --- SDD-109 / SDD-110 JNI callbacks: focus + SCO events ----------
//
// Kotlin-side OnAudioFocusChangeListener and BroadcastReceiver for
// ACTION_SCO_AUDIO_STATE_UPDATED call these Rust entry points via
// JNI. Each function marshals the platform event into a BackendEvent
// and posts it through the global event sender registered by the
// engine at voice start.
//
// SDD-115 callback safety: every entry point is wrapped in
// catch_unwind so a panic in the Rust engine can never unwind
// into the JVM.
/// SDD-109: audio focus change published by Kotlin's
/// `AndroidAudioFocusController`. `state` is the `focusChange`
/// value from `OnAudioFocusChangeListener`.
///
/// Symbol naming: JNI function declared in
/// `app.chanora.chanora_flutter.AndroidAudioFocusController`.
#[no_mangle]
pub extern "system" fn Java_app_chanora_chanora_1flutter_AndroidAudioFocusController_publishFocusChange<
'local,
>(
_env: jni::JNIEnv<'local>,
_class: jni::objects::JClass<'local>,
state: jni::sys::jint,
) {
let _ = catch_unwind(AssertUnwindSafe(|| {
// AUDIOFOCUS_LOSS = -1, LOSS_TRANSIENT = -2, LOSS_TRANSIENT_CAN_DUCK = -3,
// GAIN = 1 (AudioManager.AUDIOFOCUS_REQUEST_GRANTED is also 1, but we only
// call this from the listener callback so the values are well-known).
let event = match state {
-1 => BackendEvent::FocusLost,
-2 => BackendEvent::FocusTransient,
-3 => BackendEvent::FocusTransientCanDuck,
1 | 2 | 3 | 4 => BackendEvent::FocusGain,
_ => {
warn!(
target: "chanora_audio",
state,
"android: unknown audio focus change value; treating as FocusLost"
);
BackendEvent::FocusLost
}
};
try_send_backend_event(event);
}));
}
/// SDD-110: Bluetooth SCO state change published by Kotlin's
/// `AndroidBluetoothScoController`. `state` is the `STATE`
/// value from `ACTION_SCO_AUDIO_STATE_UPDATED`:
/// - `ACTION_SCO_AUDIO_STATE_UPDATED` is always fired with `EXTRA_SCO_AUDIO_STATE`
/// - `SCO_STATE_CONNECTING = 0`, `SCO_STATE_CONNECTED = 1`, `SCO_STATE_DISCONNECTED = 2`
///
/// Symbol naming: JNI function declared in
/// `app.chanora.chanora_flutter.AndroidBluetoothScoController`.
#[no_mangle]
pub extern "system" fn Java_app_chanora_chanora_1flutter_AndroidBluetoothScoController_publishScoStateChange<
'local,
>(
_env: jni::JNIEnv<'local>,
_class: jni::objects::JClass<'local>,
state: jni::sys::jint,
) {
let _ = catch_unwind(AssertUnwindSafe(|| {
try_send_backend_event(BackendEvent::BluetoothScoStateChanged(state));
}));
}
/// SDD-115 integration: start the Android audio focus listener.
/// Called by the engine after the voice unit is started. Uses JNI
/// to invoke `AndroidAudioFocusController.start(Context)`.
pub fn chanora_android_request_audio_focus() -> bool {
call_static_void_context(
"app/chanora/chanora_flutter/AndroidAudioFocusController",
"start",
)
}
/// SDD-115 integration: stop the Android audio focus listener.
/// Called by the engine on voice stop.
pub fn chanora_android_abandon_audio_focus() -> bool {
call_static_void_context(
"app/chanora/chanora_flutter/AndroidAudioFocusController",
"stop",
)
}
/// SDD-115 integration: start Bluetooth SCO.
/// Called by the engine after the voice unit is started.
pub fn chanora_android_start_bluetooth_sco() -> bool {
call_static_void_context(
"app/chanora/chanora_flutter/AndroidBluetoothScoController",
"start",
)
}
/// SDD-115 integration: stop Bluetooth SCO.
/// Called by the engine on voice stop.
pub fn chanora_android_stop_bluetooth_sco() -> bool {
call_static_void_context(
"app/chanora/chanora_flutter/AndroidBluetoothScoController",
"stop",
)
}
fn call_static_void_context(fqcn: &str, method: &str) -> bool {
use jni::objects::{JObject, JValue};
let ctx = ndk_context::android_context();
if ctx.vm().is_null() || ctx.context().is_null() {
warn!(
target: "chanora_audio",
class = fqcn,
method,
"android: ndk_context not initialised; call skipped"
);
return false;
}
let jvm = match unsafe { jni::JavaVM::from_raw(ctx.vm() as *mut _) } {
Ok(v) => v,
Err(e) => {
warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: JavaVM::from_raw failed");
return false;
}
};
let mut env = match jvm.attach_current_thread() {
Ok(e) => e,
Err(e) => {
warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: attach_current_thread failed");
return false;
}
};
let context_obj = unsafe { JObject::from_raw(ctx.context() as jni::sys::jobject) };
let class = match load_app_class(&mut env, &context_obj, fqcn) {
Some(c) => c,
None => return false,
};
match env.call_static_method(
&class,
method,
"(Landroid/content/Context;)V",
&[JValue::Object(&context_obj)],
) {
Ok(_) => {
info!(target: "chanora_audio", class = fqcn, method, "android: dispatched");
true
}
Err(e) => {
let _ = env.exception_clear();
warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: static call failed");
false
}
}
}
+24 -20
View File
@@ -58,7 +58,7 @@ impl AudioRoute {
pub enum IosVoiceProcessingMode {
/// Shipping default: Apple VoiceProcessingIO owns AEC/NS/AGC.
PlatformVoiceProcessing,
/// Experimental Sonora capture-processing path.
/// Experimental raw capture-processing path.
SonoraExperimental,
}
@@ -69,7 +69,7 @@ pub enum AudioBackend {
PlatformVoiceProcessing,
/// Rust-native Sonora backend.
Sonora,
/// Future WebRTC APM backend.
/// WebRTC Audio Processing Module backend.
WebrtcApm,
/// No processing.
Noop,
@@ -168,9 +168,9 @@ impl Default for AudioProcessingConfig {
route: AudioRoute::Speaker,
ios_mode: IosVoiceProcessingMode::PlatformVoiceProcessing,
processing_backend: AudioBackend::PlatformVoiceProcessing,
vad_backend: VadBackend::SileroOnnx,
vad_backend: VadBackend::TenVad,
aec: EffectOwner::Platform,
// iOS VPIO owns NS/AGC on the default shipping path. Rust/Sonora
// iOS VPIO owns NS/AGC on the default shipping path. Software
// effects are opt-in through the experimental raw route only.
ns: EffectOwner::Platform,
agc: EffectOwner::Platform,
@@ -194,19 +194,23 @@ impl AudioProcessingConfig {
}
if self.ios_mode == IosVoiceProcessingMode::PlatformVoiceProcessing
&& (self.processing_backend == AudioBackend::Sonora
|| self.processing_backend == AudioBackend::WebrtcApm
|| self.aec == EffectOwner::Sonora
|| self.aec == EffectOwner::WebrtcApm
|| self.ns == EffectOwner::Sonora
|| self.agc == EffectOwner::Sonora)
|| self.ns == EffectOwner::WebrtcApm
|| self.agc == EffectOwner::Sonora
|| self.agc == EffectOwner::WebrtcApm)
{
return Err(AudioError::InvalidAudioProcessingConfig(
"Sonora cannot be enabled with iOS VoiceProcessingIO".to_string(),
"software audio processing cannot be enabled with iOS VoiceProcessingIO"
.to_string(),
));
}
if self.ios_mode == IosVoiceProcessingMode::SonoraExperimental {
if self.processing_backend != AudioBackend::Sonora {
if self.processing_backend != AudioBackend::WebrtcApm {
return Err(AudioError::InvalidAudioProcessingConfig(
"ios Sonora experimental mode requires the Sonora processing backend"
.to_string(),
"ios raw processing mode requires the WebRTC APM backend".to_string(),
));
}
}
@@ -246,9 +250,9 @@ mod tests {
}
#[test]
fn platform_voice_processing_rejects_sonora_effects() {
fn platform_voice_processing_rejects_software_effects() {
let config = AudioProcessingConfig {
ns: EffectOwner::Sonora,
ns: EffectOwner::WebrtcApm,
..AudioProcessingConfig::default()
};
@@ -261,13 +265,13 @@ mod tests {
}
#[test]
fn sonora_experimental_allows_full_sonora_chain() {
fn raw_processing_allows_full_webrtc_apm_chain() {
let config = AudioProcessingConfig {
ios_mode: IosVoiceProcessingMode::SonoraExperimental,
processing_backend: AudioBackend::Sonora,
aec: EffectOwner::Sonora,
ns: EffectOwner::Sonora,
agc: EffectOwner::Sonora,
processing_backend: AudioBackend::WebrtcApm,
aec: EffectOwner::WebrtcApm,
ns: EffectOwner::WebrtcApm,
agc: EffectOwner::WebrtcApm,
..AudioProcessingConfig::default()
};
@@ -275,13 +279,13 @@ mod tests {
}
#[test]
fn sonora_experimental_rejects_non_sonora_backend() {
fn raw_processing_rejects_non_webrtc_apm_backend() {
let config = AudioProcessingConfig {
ios_mode: IosVoiceProcessingMode::SonoraExperimental,
processing_backend: AudioBackend::PlatformVoiceProcessing,
aec: EffectOwner::Sonora,
ns: EffectOwner::Sonora,
agc: EffectOwner::Sonora,
aec: EffectOwner::WebrtcApm,
ns: EffectOwner::WebrtcApm,
agc: EffectOwner::WebrtcApm,
..AudioProcessingConfig::default()
};
+76 -10
View File
@@ -310,14 +310,14 @@ fn open_ios_voice_backend(
audio_processing_stats.clone(),
) {
Ok(unit) => {
info!(target: "chanora_audio", "ios: RemoteIO/Sonora experimental backend selected");
info!(target: "chanora_audio", "ios: RemoteIO/WebRTC APM backend selected");
return Ok(IosVoiceBackend::Raw(unit));
}
Err(e) => {
warn!(
target: "chanora_audio",
error = %e,
"ios: RemoteIO/Sonora backend failed; falling back to VoiceProcessingIO"
"ios: RemoteIO/WebRTC APM backend failed; falling back to VoiceProcessingIO"
);
}
}
@@ -724,6 +724,9 @@ impl AudioEngine {
handler: audio_handler.clone(),
output_gain: output_gain.clone(),
output_muted: output_muted.clone(),
voice_activity_selector: cfg.voice_activity_selector.clone(),
audio_processing_config: audio_processing_config.clone(),
audio_processing_stats: audio_processing_stats.clone(),
};
let mut android_voice_unit =
crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params).map_err(|e| {
@@ -739,16 +742,75 @@ impl AudioEngine {
if let Some(mut event_rx) = android_voice_unit.take_event_rx() {
tokio::spawn(async move {
while let Some(event) = event_rx.recv().await {
if let BackendEvent::Disconnected = event {
warn!(
target: "chanora_audio",
"android: backend disconnected event received; stream reconnect requires session restart"
);
match event {
BackendEvent::Disconnected => {
warn!(
target: "chanora_audio",
"android: backend disconnected event received; stream reconnect requires session restart"
);
}
BackendEvent::FocusLost => {
warn!(
target: "chanora_audio",
"android: audio focus lost permanently (SDD-109); engine should leave session"
);
}
BackendEvent::FocusTransient => {
info!(
target: "chanora_audio",
"android: transient audio focus loss (SDD-109); pausing capture"
);
}
BackendEvent::FocusTransientCanDuck => {
info!(
target: "chanora_audio",
"android: transient audio focus loss with ducking (SDD-109); continuing"
);
}
BackendEvent::FocusGain => {
info!(
target: "chanora_audio",
"android: audio focus regained (SDD-109); resuming capture"
);
}
BackendEvent::BluetoothScoStateChanged(s) => {
info!(
target: "chanora_audio",
sco_state = s,
"android: Bluetooth SCO state changed (SDD-110)"
);
}
}
}
});
}
crate::android_voice_unit::register_global_event_sender(android_voice_unit.event_sender());
if crate::android_voice_unit::chanora_android_request_audio_focus() {
info!(
target: "chanora_audio",
"android: audio focus requested (SDD-109)"
);
} else {
warn!(
target: "chanora_audio",
"android: audio focus request failed; engine operates without focus (SDD-109)"
);
}
if crate::android_voice_unit::chanora_android_start_bluetooth_sco() {
info!(
target: "chanora_audio",
"android: Bluetooth SCO started (SDD-110)"
);
} else {
warn!(
target: "chanora_audio",
"android: Bluetooth SCO start failed; BT HFP may not route correctly (SDD-110)"
);
}
let capture_active = true;
let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
@@ -949,13 +1011,17 @@ impl AudioEngine {
let _ = self._ios_voice_backend.lock().unwrap().take();
}
// SDD-115 reverse-order teardown on Android:
// 1) close the voice unit (releases SDD-113 hardware
// 1) stop Bluetooth SCO + abandon audio focus;
// 2) close the voice unit (releases SDD-113 hardware
// effects then stops + closes the Oboe streams);
// 2) restore the prior audio mode (SDD-108 §1 on 1 → 0
// 3) restore the prior audio mode (SDD-108 §1 on 1 → 0
// transition);
// 3) stop the foreground service.
// 4) stop the foreground service.
#[cfg(target_os = "android")]
{
crate::android_voice_unit::chanora_android_stop_bluetooth_sco();
crate::android_voice_unit::chanora_android_abandon_audio_focus();
crate::android_voice_unit::clear_global_event_sender();
if let Some(mut unit) = self._android_voice_unit.lock().unwrap().take() {
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
if let Err(e) = unit.close() {
+132 -27
View File
@@ -1,15 +1,15 @@
//! Optional raw iOS RemoteIO path for the Sonora experimental mode.
//! Optional raw iOS RemoteIO path for the WebRTC APM experimental mode.
//!
//! Provides an alternative to `ios_voice_unit.rs` for the
//! `SonoraExperimental` processing mode. Instead of
//! `kAudioUnitSubType_VoiceProcessingIO` (which owns AEC/NS/AGC), it
//! opens `kAudioUnitSubType_RemoteIO` with voice processing explicitly
//! disabled so Rust's Sonora DSP chain can own the full signal path.
//! disabled so WebRTC APM can own the full signal path.
//!
//! ## Hard invariants enforced here
//!
//! * INV_009: Rust AEC only active when platform AEC is disabled.
//! * INV_010: VoiceProcessingIO and Sonora AEC3 are mutually exclusive.
//! * INV_010: VoiceProcessingIO and WebRTC APM AEC are mutually exclusive.
//! * INV_011: Software AEC backend receives both capture and render-reference.
//! * INV_012: Render reference is copied from decoded/mixed remote PCM
//! before playout.
@@ -112,19 +112,25 @@ mod inner {
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
/// Processing config — retained for route-change reloads; not read in the hot path.
#[allow(dead_code)]
/// Processing config — retained for route-change reloads.
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
sonora_processor: crate::processor::SonoraProcessor,
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: usize,
fallback_warned_backend: Option<crate::VadBackend>,
wav_recorder: Option<Arc<crate::debug_wav::WavDebugRecorder>>,
}
@@ -132,6 +138,7 @@ mod inner {
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
@@ -140,32 +147,48 @@ mod inner {
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("ios raw")?;
let webrtc_apm_config = audio_processing_config
.lock()
.map(|cfg| crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg))
.unwrap_or_default();
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
output_muted,
frames_sent,
mic_gain,
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
sonora_processor: crate::processor::SonoraProcessor::with_config(
crate::processor::sonora::SonoraConfig::with_aec3(),
),
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
webrtc_apm_config,
)?,
audio_processing_stats,
render_reference,
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
fallback_warned_backend: None,
wav_recorder: None,
})
}
fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) {
if let Ok(mut cfg) = self.audio_processing_config.try_lock() {
let _ = cfg.disable_failed_vad_backend(failed_backend);
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend != Some(failed_backend) {
self.fallback_warned_backend = Some(failed_backend);
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
}
@@ -237,12 +260,12 @@ mod inner {
rec.push_raw_mic(&frame);
}
// INV_012: feed render reference to Sonora AEC3 before capture.
// Feed render reference to WebRTC APM before capture so AEC can adapt.
let render_ref = self.render_reference.read_latest();
self.sonora_processor.process_render(&render_ref);
self.sonora_processor.process_capture(&mut frame);
self.webrtc_apm_processor.process_render(&render_ref);
self.webrtc_apm_processor.process_capture(&mut frame);
// WAV tap: processed mic (after Sonora).
// WAV tap: processed mic (after WebRTC APM).
if let Some(ref rec) = self.wav_recorder {
rec.push_processed_mic(&frame);
}
@@ -260,37 +283,118 @@ mod inner {
vad_hangover,
crate::voice_activity::VAD_MIN_TX_MS,
);
// Switch VAD backend when config changes.
let silero_epoch = crate::vad::silero_model_epoch();
let silero_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
let path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
self.ten_vad_worker = None;
if self.silero_vad_worker.is_none() {
tracing::warn!(
target: "chanora_audio",
"Silero VAD model not found at {path}; falling back to WebRTC VAD"
);
}
}
crate::VadBackend::TenVad => {
let path = crate::vad::ten_model_bundle_path();
self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path);
self.silero_vad_worker = None;
if self.ten_vad_worker.is_none() {
tracing::warn!(
target: "chanora_audio",
"TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD"
);
}
}
_ => {
self.silero_vad_worker = None;
self.ten_vad_worker = None;
}
}
self.vad_state.reset();
}
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
let capture_seq = self.capture_frame_seq;
let mut used_fallback_vad = false;
let vad = if vad_backend == crate::VadBackend::Disabled {
crate::vad::VadOutput {
probability: 1.0,
speech: true,
}
} else {
used_fallback_vad = matches!(
vad_backend,
crate::VadBackend::SileroOnnx | crate::VadBackend::TenVad
);
if used_fallback_vad {
self.disable_failed_vad_backend(vad_backend);
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let active = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(sel) = &self.voice_activity_selector {
sel.set_voice_activity_open(active);
sel.set_voice_activity_open(active && !output_muted);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
active,
active && !output_muted,
self.transmit_active.load(Ordering::Relaxed),
);
if !self.transmit_active.load(Ordering::Relaxed) {
if !self.transmit_active.load(Ordering::Relaxed) || output_muted {
return;
}
@@ -336,7 +440,7 @@ mod inner {
let cfg = audio_processing_config.lock().unwrap();
if cfg.ios_mode == crate::IosVoiceProcessingMode::PlatformVoiceProcessing {
return Err(AudioError::InvalidAudioProcessingConfig(
"IosRawUnit requires SonoraExperimental mode".to_string(),
"IosRawUnit requires raw WebRTC APM mode".to_string(),
));
}
}
@@ -369,6 +473,7 @@ mod inner {
let mut capture_state = RawCaptureState::new(
voice_out_tx,
transmit_active,
output_muted.clone(),
frames_sent,
mic_gain,
voice_activity_selector,
+59 -24
View File
@@ -139,6 +139,7 @@ struct IosCaptureState {
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
@@ -146,10 +147,12 @@ struct IosCaptureState {
/// Background Silero worker — enqueues frames off the realtime
/// callback and publishes the latest probability atomically.
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad: Option<crate::vad::TenOnnxVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
fallback_warned_backend: Option<crate::VadBackend>,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
sonora_processor: crate::processor::SonoraProcessor,
@@ -173,6 +176,7 @@ impl IosCaptureState {
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
@@ -188,13 +192,16 @@ impl IosCaptureState {
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
output_muted,
frames_sent,
mic_gain,
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
sonora_processor: crate::processor::SonoraProcessor::new(),
@@ -210,12 +217,16 @@ impl IosCaptureState {
})
}
fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) {
if let Ok(mut cfg) = self.audio_processing_config.try_lock() {
if cfg.disable_failed_vad_backend(failed_backend) {
self.current_vad_backend = crate::VadBackend::WebrtcVad;
}
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend == Some(failed_backend) {
return;
}
self.fallback_warned_backend = Some(failed_backend);
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
/// Consume the i16 mono buffer delivered by VPIO, accumulate
@@ -367,6 +378,7 @@ impl IosCaptureState {
if vad_backend != self.current_vad_backend || silero_model_changed {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
self.silero_model_epoch = silero_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
@@ -383,22 +395,28 @@ impl IosCaptureState {
target: "chanora_audio",
"Silero VAD model not found at {model_path}; falling back to WebRTC VAD"
);
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
}
self.audio_processing_stats
.set_vad_fallback_active(self.silero_vad_worker.is_none());
}
crate::VadBackend::TenVad => {
self.silero_vad_worker = None;
warn!(
target: "chanora_audio",
"TEN VAD selected but native TEN runtime is not bundled; falling back to WebRTC VAD"
);
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
self.audio_processing_stats.set_vad_fallback_active(true);
let model_path = crate::vad::ten_model_bundle_path();
self.ten_vad = crate::vad::TenOnnxVadWorker::try_new(&model_path);
if self.ten_vad.is_none() {
warn!(
target: "chanora_audio",
"TEN VAD ONNX model not available at {model_path}; falling back to WebRTC VAD"
);
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
}
self.audio_processing_stats
.set_vad_fallback_active(self.ten_vad.is_none());
}
_ => {
self.silero_vad_worker = None;
self.ten_vad = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
}
@@ -419,14 +437,14 @@ impl IosCaptureState {
);
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
// Apply the enabled stages through the SonoraProcessor.
// We reconfigure it on-the-fly to match the current settings.
// VPIO owns AEC. Keep the legacy non-AEC conditioning path here until
// the raw WebRTC APM path is explicitly selected.
if run_ns || run_agc || run_hpf {
use crate::processor::sonora::SonoraConfig;
use crate::processor::AudioProcessor;
let new_cfg = SonoraConfig {
hpf: run_hpf,
aec3: false, // NEVER in VPIO path (INV_009)
aec3: false,
ns: run_ns,
agc2: run_agc,
};
@@ -448,7 +466,8 @@ impl IosCaptureState {
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
if worker.try_send(capture_seq, &frame) && !worker.is_stale(capture_seq) {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
@@ -456,33 +475,48 @@ impl IosCaptureState {
}
} else {
used_fallback_vad = true;
self.silero_vad_worker = None;
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
used_fallback_vad = true;
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
if let Some(worker) = self.ten_vad.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let gate_open = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(selector) = &self.voice_activity_selector {
selector.set_voice_activity_open(gate_open);
selector.set_voice_activity_open(gate_open && !output_muted);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
gate_open,
gate_open && !output_muted,
transmit_active,
);
@@ -700,6 +734,7 @@ impl IosVoiceUnit {
let mut capture_state = IosCaptureState::new(
voice_out_tx,
transmit_active,
output_muted.clone(),
frames_sent,
mic_gain,
voice_activity_selector,
@@ -4,10 +4,12 @@ pub mod dsp;
pub mod noop;
pub mod platform;
pub mod sonora;
pub mod webrtc_apm;
pub use noop::NoopProcessor;
pub use platform::PlatformVoiceProcessor;
pub use sonora::SonoraProcessor;
pub use webrtc_apm::WebRtcApmProcessor;
/// 10 ms mono f32 processing frame at 48 kHz (480 samples).
pub const FRAME_SAMPLES: usize = 480;
@@ -0,0 +1,193 @@
//! WebRTC Audio Processing Module backend.
//!
//! This backend delegates capture-side voice processing to WebRTC APM
//! instead of Chanora's experimental Rust DSP chain. Frames are 10 ms,
//! mono, 48 kHz f32, matching the rest of the voice pipeline.
use sonora::config::{
AdaptiveDigital, EchoCanceller, GainController2, HighPassFilter, NoiseSuppression,
NoiseSuppressionLevel, Pipeline,
};
use sonora::{AudioProcessing, Config, StreamConfig};
use super::{AudioProcessor, FRAME_SAMPLES};
/// Per-module WebRTC APM enable flags.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WebRtcApmConfig {
/// Acoustic echo cancellation.
pub aec: bool,
/// Automatic gain control.
pub agc: bool,
/// High-pass filter.
pub hpf: bool,
/// Noise suppression.
pub ns: bool,
/// WebRTC VAD is available for transmit gating.
pub vad: bool,
}
impl WebRtcApmConfig {
/// Resolve WebRTC APM flags from the shared audio-processing config.
pub fn from_audio_config(config: &crate::AudioProcessingConfig) -> Self {
Self {
aec: config.aec == crate::EffectOwner::WebrtcApm,
agc: matches!(
config.agc,
crate::EffectOwner::WebrtcApm | crate::EffectOwner::Conservative
),
hpf: config.hpf_enabled,
ns: matches!(
config.ns,
crate::EffectOwner::WebrtcApm | crate::EffectOwner::Conservative
),
vad: config.vad_backend == crate::VadBackend::WebrtcVad,
}
}
fn to_webrtc_config(self) -> Config {
Config {
pipeline: Pipeline {
maximum_internal_processing_rate: sonora::config::MaxProcessingRate::Rate48kHz,
..Default::default()
},
high_pass_filter: self.hpf.then_some(HighPassFilter::default()),
echo_canceller: self.aec.then_some(EchoCanceller::default()),
noise_suppression: self.ns.then_some(NoiseSuppression {
level: NoiseSuppressionLevel::Moderate,
analyze_linear_aec_output_when_available: false,
}),
gain_controller2: self.agc.then_some(GainController2 {
input_volume_controller: false,
adaptive_digital: Some(AdaptiveDigital::default()),
fixed_digital: Default::default(),
}),
..Default::default()
}
}
/// Stable runtime-log summary.
pub fn summary(self) -> String {
format!(
"aec={} agc={} hpf={} ns={} vad={}",
self.aec, self.agc, self.hpf, self.ns, self.vad
)
}
}
impl Default for WebRtcApmConfig {
fn default() -> Self {
Self {
aec: true,
agc: true,
hpf: true,
ns: true,
vad: true,
}
}
}
/// WebRTC APM processor with preallocated render scratch.
pub struct WebRtcApmProcessor {
processor: AudioProcessing,
config: WebRtcApmConfig,
capture_scratch: [f32; FRAME_SAMPLES],
render_scratch: [f32; FRAME_SAMPLES],
}
impl WebRtcApmProcessor {
/// Construct with a specific module configuration.
pub fn with_config(config: WebRtcApmConfig) -> Result<Self, crate::AudioError> {
let stream_config = StreamConfig::new(crate::frame::SAMPLE_RATE_HZ, 1);
let processor = AudioProcessing::builder()
.config(config.to_webrtc_config())
.capture_config(stream_config)
.render_config(stream_config)
.echo_detector(config.aec)
.build();
tracing::info!(
target: "chanora_audio",
modules = %config.summary(),
"WebRTC APM processor active"
);
Ok(Self {
processor,
config,
capture_scratch: [0.0; FRAME_SAMPLES],
render_scratch: [0.0; FRAME_SAMPLES],
})
}
/// Current module configuration.
pub fn config(&self) -> WebRtcApmConfig {
self.config
}
/// Apply updated module flags outside the realtime callback.
pub fn apply_config(&mut self, config: WebRtcApmConfig) {
if config == self.config {
return;
}
self.processor.apply_config(config.to_webrtc_config());
self.config = config;
tracing::info!(
target: "chanora_audio",
modules = %config.summary(),
"WebRTC APM processor reconfigured"
);
}
}
impl AudioProcessor for WebRtcApmProcessor {
fn process_capture(&mut self, frame: &mut [f32; FRAME_SAMPLES]) {
let channels = [&frame[..]];
let mut out = [&mut self.capture_scratch[..]];
if let Err(error) = self.processor.process_capture_f32(&channels, &mut out) {
tracing::trace!(target: "chanora_audio", %error, "WebRTC APM capture frame skipped");
} else {
frame.copy_from_slice(&self.capture_scratch);
}
}
fn process_render(&mut self, frame: &[f32; FRAME_SAMPLES]) {
let channels = [&frame[..]];
let mut out = [&mut self.render_scratch[..]];
if let Err(error) = self.processor.process_render_f32(&channels, &mut out) {
tracing::trace!(target: "chanora_audio", %error, "WebRTC APM render frame skipped");
}
}
fn has_aec(&self) -> bool {
self.config.aec
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_config_enables_all_modules() {
let cfg = WebRtcApmConfig::default();
assert!(cfg.aec && cfg.agc && cfg.hpf && cfg.ns && cfg.vad);
}
#[test]
fn process_silence_is_stable() {
let mut processor = WebRtcApmProcessor::with_config(WebRtcApmConfig {
aec: false,
agc: true,
hpf: true,
ns: true,
vad: true,
})
.expect("webrtc apm init");
let render = [0.0_f32; FRAME_SAMPLES];
let mut capture = [0.0_f32; FRAME_SAMPLES];
processor.process_render(&render);
processor.process_capture(&mut capture);
assert!(capture.iter().all(|s| s.is_finite()));
}
}
+124 -75
View File
@@ -7,15 +7,17 @@
pub mod resampler;
pub mod silero_onnx;
pub mod ten_onnx;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{OnceLock, RwLock};
use crate::frame::{dbfs, i16_to_f32};
use crate::frame::{f32_to_i16, i16_to_f32};
use crate::AudioError;
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
pub use silero_onnx::SileroOnnxVad;
pub use ten_onnx::{TenOnnxVad, TenOnnxVadWorker};
/// Voice activity detector output for one 10 ms frame.
#[derive(Debug, Clone, Copy)]
@@ -32,83 +34,38 @@ pub trait VoiceActivityDetector: Send {
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput;
}
/// Realtime-safe fallback VAD used when a model runtime is unavailable.
///
/// This is not an energy-only transmit gate. It combines RMS level,
/// zero-crossing rate, and peak-to-RMS shape with hysteresis so stable
/// background rumble is less likely to open VoiceActivity than speech.
#[derive(Debug, Clone)]
/// Realtime-safe WebRTC VAD used when a model runtime is unavailable.
pub struct WebRtcFallbackVad {
open_dbfs: f32,
close_dbfs: f32,
active: bool,
vad: webrtc_vad::Vad,
frame_i16: [i16; INPUT_FRAME_10MS],
}
// `webrtc_vad::Vad` owns an FFI pointer and is only touched from the
// capture thread after construction. Moving the wrapper between threads is
// safe; sharing it concurrently is not required and not implemented.
unsafe impl Send for WebRtcFallbackVad {}
impl Default for WebRtcFallbackVad {
fn default() -> Self {
Self {
open_dbfs: -42.0,
close_dbfs: -50.0,
active: false,
vad: webrtc_vad::Vad::new_with_rate_and_mode(
webrtc_vad::SampleRate::Rate48kHz,
webrtc_vad::VadMode::Aggressive,
),
frame_i16: [0; INPUT_FRAME_10MS],
}
}
}
impl WebRtcFallbackVad {
fn zero_crossing_rate(samples: &[f32]) -> f32 {
if samples.len() < 2 {
return 0.0;
}
let crossings = samples
.windows(2)
.filter(|pair| (pair[0] >= 0.0 && pair[1] < 0.0) || (pair[0] < 0.0 && pair[1] >= 0.0))
.count();
crossings as f32 / (samples.len() - 1) as f32
}
fn peak_to_rms(samples: &[f32], rms: f32) -> f32 {
if rms <= 0.000_001 {
return 0.0;
}
let peak = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
peak / rms
}
}
impl VoiceActivityDetector for WebRtcFallbackVad {
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
let level = dbfs(samples);
let threshold = if self.active {
self.close_dbfs
} else {
self.open_dbfs
};
let rms = samples.iter().map(|s| s * s).sum::<f32>() / samples.len().max(1) as f32;
let rms = rms.sqrt();
let zcr = Self::zero_crossing_rate(samples);
let crest = Self::peak_to_rms(samples, rms);
// Level score: steeper curve so silence (-50 dBFS) scores near 0.
// Speech is typically -30 to -10 dBFS; silence is -60 to -45 dBFS.
// Map [-60, -20] → [0, 1] with a midpoint at -40 dBFS.
let level_score = ((level + 60.0) / 40.0).clamp(0.0, 1.0);
let zcr_score = if (0.015..=0.32).contains(&zcr) {
1.0
} else {
0.3 // penalise non-speech ZCR more aggressively
};
let crest_score = if (1.5..=12.0).contains(&crest) {
1.0
} else {
0.3
};
let probability =
(level_score * 0.72 + zcr_score * 0.18 + crest_score * 0.10).clamp(0.0, 1.0);
self.active = level >= threshold && probability >= 0.5;
for (dst, src) in self.frame_i16.iter_mut().zip(samples.iter().copied()) {
*dst = f32_to_i16(src);
}
let speech = self.vad.is_voice_segment(&self.frame_i16).unwrap_or(false);
VadOutput {
probability,
speech: self.active,
probability: if speech { 1.0 } else { 0.0 },
speech,
}
}
}
@@ -151,11 +108,17 @@ pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
static TEN_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
static TEN_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
}
fn ten_model_path_override() -> &'static RwLock<Option<String>> {
TEN_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
}
/// Configure the preferred Silero ONNX model path.
///
/// The path is validated eagerly. A successful call increments the
@@ -186,12 +149,39 @@ pub fn silero_model_epoch() -> u64 {
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
}
/// Return the expected path of the Silero VAD v6 ONNX model in the
/// iOS app bundle. The model is shipped as a Flutter asset and copied
/// to the app's Documents directory by the Dart-side asset loader.
/// Configure the preferred TEN VAD ONNX model path.
pub fn set_ten_model_path(path: &str) -> Result<(), AudioError> {
let path = path.trim();
if path.is_empty() {
return Err(AudioError::InvalidAudioProcessingConfig(
"ten vad model path must not be empty".to_string(),
));
}
if !std::path::Path::new(path).is_file() {
return Err(AudioError::InvalidAudioProcessingConfig(format!(
"ten vad model path does not exist or is not a file: {path}"
)));
}
let mut guard = ten_model_path_override()
.write()
.map_err(|_| AudioError::Backend("ten vad model path lock poisoned".to_string()))?;
*guard = Some(path.to_string());
TEN_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
Ok(())
}
/// Monotonic counter incremented whenever the configured TEN model path changes.
pub fn ten_model_epoch() -> u64 {
TEN_MODEL_EPOCH.load(Ordering::Relaxed)
}
/// Return the expected path of the Silero VAD v6 ONNX model.
/// The model is shipped as a Flutter asset and copied to the app's
/// data directory by the Dart-side asset loader.
///
/// Returns an empty string on non-Apple platforms (Silero is not
/// supported there; `SileroOnnxVad::try_new` will return `None`).
/// On iOS/Android the model lives in the app's Documents/files
/// directory. On desktop, the caller should set the path explicitly
/// via `set_silero_model_path`.
pub fn silero_model_bundle_path() -> String {
if let Ok(guard) = silero_model_path_override().read() {
if let Some(path) = guard.as_ref() {
@@ -199,26 +189,85 @@ pub fn silero_model_bundle_path() -> String {
}
}
// iOS: Documents directory (written by Flutter asset loader).
// macOS: same Documents pattern.
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
// Primary: Documents directory (written by Flutter asset loader).
if let Ok(home) = std::env::var("HOME") {
let docs = format!("{home}/Documents/silero_vad.onnx");
if std::path::Path::new(&docs).exists() {
return docs;
}
// Fallback: app bundle Resources directory.
let bundle = format!("{home}/../Library/silero_vad.onnx");
if std::path::Path::new(&bundle).exists() {
return bundle;
}
}
// Last resort: current working directory (useful in tests).
"silero_vad.onnx".to_string()
}
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
// Android: the model is in the app's files directory, same
// Documents path pattern used by Flutter's path_provider.
#[cfg(target_os = "android")]
{
String::new()
// On Android, Flutter's getApplicationDocumentsDirectory
// resolves to /data/data/<package>/app_flutter.
// The Silero model path is set explicitly via
// set_silero_model_path from Dart before voice starts,
// so this fallback is rarely needed.
"silero_vad.onnx".to_string()
}
// Desktop (Windows, Linux): rely on the override set by Dart.
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
{
if let Ok(cwd) = std::env::current_dir() {
let local = cwd.join("silero_vad.onnx");
if local.exists() {
return local.to_string_lossy().to_string();
}
}
"silero_vad.onnx".to_string()
}
}
/// Return the expected path of the TEN VAD ONNX model copied by Flutter.
pub fn ten_model_bundle_path() -> String {
if let Ok(guard) = ten_model_path_override().read() {
if let Some(path) = guard.as_ref() {
return path.clone();
}
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
if let Ok(home) = std::env::var("HOME") {
let docs = format!("{home}/Documents/ten_vad.onnx");
if std::path::Path::new(&docs).exists() {
return docs;
}
let bundle = format!("{home}/../Library/ten_vad.onnx");
if std::path::Path::new(&bundle).exists() {
return bundle;
}
}
"ten_vad.onnx".to_string()
}
#[cfg(target_os = "android")]
{
"ten_vad.onnx".to_string()
}
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
{
if let Ok(cwd) = std::env::current_dir() {
let local = cwd.join("ten_vad.onnx");
if local.exists() {
return local.to_string_lossy().to_string();
}
}
"ten_vad.onnx".to_string()
}
}
+13 -36
View File
@@ -75,13 +75,9 @@ pub struct SileroOnnxVad {
}
enum SileroInner {
#[cfg(any(target_os = "ios", target_os = "macos"))]
Onnx(OnnxSession),
#[allow(dead_code)]
Stub,
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
struct OnnxSession {
session: ort::session::Session,
}
@@ -90,20 +86,11 @@ impl SileroOnnxVad {
/// Attempt to load the Silero v6 ONNX model from `model_path`.
///
/// Returns `None` when the model file is missing, the ONNX Runtime
/// is unavailable, or the platform is not iOS/macOS.
/// is unavailable, or the platform does not support ONNX.
pub fn try_new(model_path: &str) -> Option<Self> {
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
Self::try_new_onnx(model_path)
}
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
let _ = model_path;
None
}
Self::try_new_onnx(model_path)
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
fn try_new_onnx(model_path: &str) -> Option<Self> {
use tracing::{error, info};
@@ -116,13 +103,8 @@ impl SileroOnnxVad {
return None;
}
#[cfg(target_os = "macos")]
if let Some(path) = bundled_onnxruntime_path() {
let _ = ort::init_from(path.to_string_lossy()).commit();
}
let session_result = std::panic::catch_unwind(|| {
ort::session::Session::builder().and_then(|b| b.commit_from_file(model_path))
ort::session::Session::builder().and_then(|mut b| b.commit_from_file(model_path))
});
match session_result {
@@ -193,7 +175,6 @@ impl SileroOnnxVad {
/// 16 kHz frame: concatenate prior context, pass `input/state/sr` to
/// ONNX, persist `stateN`, then refresh context from the current frame.
/// Updates `last_probability` and returns the new value.
#[cfg(any(target_os = "ios", target_os = "macos"))]
fn calc_level(&mut self, audio_frame: &[f32]) -> f32 {
use ort::value::Value;
use tracing::error;
@@ -279,7 +260,7 @@ impl SileroOnnxVad {
}
#[cfg(target_os = "macos")]
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
pub(crate) fn bundled_onnxruntime_path_for_vad() -> Option<std::path::PathBuf> {
let exe = std::env::current_exe().ok()?;
let app_dir = exe.parent()?;
let framework = app_dir
@@ -307,16 +288,9 @@ impl VoiceActivityDetector for SileroOnnxVad {
if self.accum.len() >= SILERO_FRAME_16K {
let audio_frame: Vec<f32> = self.accum[..SILERO_FRAME_16K].to_vec();
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
if matches!(self.inner, SileroInner::Onnx(_)) {
self.calc_level(&audio_frame);
} else {
self.update_context_from_frame(&audio_frame);
}
}
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
if matches!(self.inner, SileroInner::Onnx(_)) {
self.calc_level(&audio_frame);
} else {
self.update_context_from_frame(&audio_frame);
}
// Drain the accumulator (keep any overflow for next frame).
@@ -357,9 +331,9 @@ impl SileroOnnxVadWorker {
pub fn try_new(model_path: &str) -> Option<Self> {
let vad = SileroOnnxVad::try_new(model_path)?;
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(0));
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(8);
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(32);
let latest_probability_for_thread = latest_probability.clone();
let latest_processed_seq_for_thread = latest_processed_seq.clone();
let alive_for_thread = alive.clone();
@@ -414,7 +388,10 @@ impl SileroOnnxVadWorker {
/// True when the worker is too far behind to trust its latest
/// probability for the current frame.
pub fn is_stale(&self, capture_seq: u64) -> bool {
self.lag_frames(capture_seq) > SILERO_MAX_STALE_FRAMES
let latest = self
.latest_processed_seq
.load(std::sync::atomic::Ordering::Relaxed);
latest == u64::MAX || capture_seq.saturating_sub(latest) > SILERO_MAX_STALE_FRAMES
}
}
+432
View File
@@ -0,0 +1,432 @@
//! TEN VAD ONNX backend.
//!
//! TEN's ONNX graph does not accept raw PCM. It expects the same feature
//! stack produced by TEN's `AUP_Aed_aivad_proc`: three context frames of
//! 40 log-mel powers plus one pitch feature, followed by four recurrent
//! state tensors. This module ports that preprocessing path to Rust and
//! keeps ONNX Runtime off the realtime callback where possible.
use crate::frame::f32_to_i16;
use super::resampler::{Downsampler48to16, INPUT_FRAME_10MS};
use super::{VadOutput, VoiceActivityDetector};
const SAMPLE_RATE_16K: f32 = 16_000.0;
const HOP_16K: usize = 256;
const WINDOW_16K: usize = 768;
const FFT_SIZE: usize = 1024;
const N_BINS: usize = FFT_SIZE / 2 + 1;
const MEL_BANDS: usize = 40;
const FEATURE_LEN: usize = 41;
const CONTEXT: usize = 3;
const HIDDEN: usize = 64;
const POWER_NORMALIZER: f32 = 32768.0 * 32768.0;
const EPS: f32 = 1.0e-20;
const FEATURE_MEANS: [f32; FEATURE_LEN] = [
-8.198236, -6.2657166, -5.4838185, -4.7586913, -4.417089, -4.142893, -3.9128504, -3.845928,
-3.6570904, -3.7234187, -3.8761342, -3.843891, -3.6904051, -3.7560658, -3.6986961, -3.650463,
-3.7004688, -3.5673213, -3.4989002, -3.477807, -3.458816, -3.4449239, -3.4013286, -3.3062613,
-3.2785568, -3.2332509, -3.198616, -3.2045264, -3.2087986, -3.257838, -3.3813767, -3.5340214,
-3.640868, -3.7268589, -3.773731, -3.8046672, -3.832901, -3.8711205, -3.990593, -4.4802895,
92.3569,
];
const FEATURE_STDS: [f32; FEATURE_LEN] = [
5.166064, 4.9772096, 4.698896, 4.6306214, 4.634348, 4.641156, 4.6406765, 4.666367, 4.6505346,
4.640021, 4.6374, 4.620099, 4.5963163, 4.562655, 4.5543604, 4.5669107, 4.56249, 4.5624127,
4.5852995, 4.6001797, 4.592846, 4.5859227, 4.5834966, 4.626093, 4.626958, 4.6262894, 4.637006,
4.683016, 4.726814, 4.7342896, 4.753227, 4.849723, 4.869435, 4.884483, 4.921327, 4.9592123,
4.996619, 5.0448236, 5.072217, 5.0964394, 115.21369,
];
/// TEN VAD using ONNX Runtime and Rust-ported TEN feature preprocessing.
pub struct TenOnnxVad {
session: ort::session::Session,
downsampler: Downsampler48to16,
hop_accum: Vec<f32>,
sample_fifo: Vec<f32>,
feature_stack: [[f32; FEATURE_LEN]; CONTEXT],
states: [[f32; HIDDEN]; 4],
mel_filters: Vec<[f32; N_BINS]>,
last_probability: f32,
last_speech: bool,
}
unsafe impl Send for TenOnnxVad {}
impl TenOnnxVad {
/// Load TEN VAD ONNX model.
pub fn try_new(model_path: &str) -> Option<Self> {
if !std::path::Path::new(model_path).exists() {
tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model not found");
return None;
}
let session = match std::panic::catch_unwind(|| {
ort::session::Session::builder().and_then(|mut b| b.commit_from_file(model_path))
}) {
Ok(Ok(session)) => session,
Ok(Err(error)) => {
tracing::warn!(target: "chanora_audio", %error, path = model_path, "TEN VAD ONNX model load failed");
return None;
}
Err(_) => {
tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX Runtime panicked during load");
return None;
}
};
tracing::info!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model loaded");
Some(Self {
session,
downsampler: Downsampler48to16::default(),
hop_accum: Vec::with_capacity(HOP_16K + super::resampler::OUTPUT_FRAME_10MS),
sample_fifo: Vec::with_capacity(WINDOW_16K + HOP_16K),
feature_stack: [[0.0; FEATURE_LEN]; CONTEXT],
states: [[0.0; HIDDEN]; 4],
mel_filters: build_mel_filters(),
last_probability: 0.0,
last_speech: false,
})
}
fn process_hop(&mut self, hop: &[f32]) {
self.sample_fifo.extend_from_slice(hop);
let frame = if self.sample_fifo.len() >= WINDOW_16K {
let start = self.sample_fifo.len() - WINDOW_16K;
self.sample_fifo[start..].to_vec()
} else {
let mut padded = vec![0.0; WINDOW_16K - self.sample_fifo.len()];
padded.extend_from_slice(&self.sample_fifo);
padded
};
if self.sample_fifo.len() > WINDOW_16K {
let excess = self.sample_fifo.len() - WINDOW_16K;
self.sample_fifo.drain(..excess);
}
let feature = compute_feature(&self.mel_filters, &frame);
self.feature_stack.copy_within(1..CONTEXT, 0);
self.feature_stack[CONTEXT - 1] = feature;
self.run_onnx();
}
fn run_onnx(&mut self) {
use ndarray::{Array, IxDyn};
use ort::value::Value;
let input: Vec<f32> = self.feature_stack.iter().flatten().copied().collect();
let input_arr = match Array::from_shape_vec(IxDyn(&[1, CONTEXT, FEATURE_LEN]), input) {
Ok(v) => v,
Err(_) => return,
};
let state_arrs = [0, 1, 2, 3]
.map(|idx| Array::from_shape_vec(IxDyn(&[1, HIDDEN]), self.states[idx].to_vec()));
let input_val = match Value::from_array(input_arr) {
Ok(v) => v,
Err(error) => {
tracing::warn!(target: "chanora_audio", %error, "TEN VAD input tensor error");
return;
}
};
let state_vals = match state_arrs {
[Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d],
_ => return,
};
let state_vals = match state_vals.map(Value::from_array) {
[Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d],
_ => return,
};
let outputs = match self.session.run([
(&input_val).into(),
(&state_vals[0]).into(),
(&state_vals[1]).into(),
(&state_vals[2]).into(),
(&state_vals[3]).into(),
]) {
Ok(outputs) => outputs,
Err(error) => {
tracing::warn!(target: "chanora_audio", %error, "TEN VAD ONNX inference failed");
return;
}
};
if let Ok((_, prob)) = outputs["output_1"].try_extract_tensor::<f32>() {
if let Some(&p) = prob.first() {
self.last_probability = p.clamp(0.0, 1.0);
self.last_speech = self.last_probability >= 0.5;
}
}
for (idx, name) in ["output_2", "output_3", "output_6", "output_7"]
.iter()
.enumerate()
{
if let Ok((_, state)) = outputs[*name].try_extract_tensor::<f32>() {
let copy_len = state.len().min(HIDDEN);
self.states[idx][..copy_len].copy_from_slice(&state[..copy_len]);
}
}
}
}
#[cfg(target_os = "macos")]
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
let exe = std::env::current_exe().ok()?;
let app_dir = exe.parent()?;
let framework = app_dir
.join("Frameworks")
.join("onnxruntime.framework")
.join("onnxruntime");
framework.exists().then_some(framework)
}
fn compute_feature(mel_filters: &[[f32; N_BINS]], frame: &[f32]) -> [f32; FEATURE_LEN] {
let power = power_spectrum(frame);
let mut feature = [0.0; FEATURE_LEN];
for band in 0..MEL_BANDS {
let energy = mel_filters[band]
.iter()
.zip(power.iter())
.map(|(w, p)| w * p)
.sum::<f32>()
/ POWER_NORMALIZER;
let log_energy = (energy + EPS).ln();
feature[band] = (log_energy - FEATURE_MEANS[band]) / (FEATURE_STDS[band] + EPS);
}
let pitch_hz = estimate_pitch_hz(frame);
feature[MEL_BANDS] = (pitch_hz - FEATURE_MEANS[MEL_BANDS]) / (FEATURE_STDS[MEL_BANDS] + EPS);
feature
}
impl VoiceActivityDetector for TenOnnxVad {
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
debug_assert_eq!(samples.len(), INPUT_FRAME_10MS);
let mut input = [0.0_f32; INPUT_FRAME_10MS];
input.copy_from_slice(samples);
let downsampled = self.downsampler.process_frame_10ms(&input);
self.hop_accum.extend_from_slice(&downsampled);
while self.hop_accum.len() >= HOP_16K {
let hop: Vec<f32> = self.hop_accum[..HOP_16K].to_vec();
self.hop_accum.drain(..HOP_16K);
self.process_hop(&hop);
}
VadOutput {
probability: self.last_probability,
speech: self.last_speech,
}
}
}
fn hz_to_mel(hz: f32) -> f32 {
2595.0 * (1.0 + hz / 700.0).log10()
}
fn mel_to_hz(mel: f32) -> f32 {
700.0 * (10.0_f32.powf(mel / 2595.0) - 1.0)
}
fn build_mel_filters() -> Vec<[f32; N_BINS]> {
let low_mel = hz_to_mel(0.0);
let high_mel = hz_to_mel(8000.0);
let mut bins = [0_usize; MEL_BANDS + 2];
for idx in 0..bins.len() {
let mel = idx as f32 * (high_mel - low_mel) / (MEL_BANDS as f32 + 1.0) + low_mel;
let hz = mel_to_hz(mel);
let mut bin = ((FFT_SIZE as f32 + 1.0) * hz / SAMPLE_RATE_16K).floor() as usize;
bin = bin.min(N_BINS - 1);
if idx > 0 && bin == bins[idx - 1] {
bin = (bin + 1).min(N_BINS - 1);
}
bins[idx] = bin;
}
let mut filters = vec![[0.0_f32; N_BINS]; MEL_BANDS];
for band in 0..MEL_BANDS {
let left = bins[band];
let center = bins[band + 1].max(left + 1);
let right = bins[band + 2].max(center + 1).min(N_BINS - 1);
for i in left..center.min(N_BINS) {
filters[band][i] = (i - left) as f32 / (center - left) as f32;
}
for i in center..=right {
filters[band][i] = (right - i) as f32 / (right - center).max(1) as f32;
}
}
filters
}
fn power_spectrum(frame: &[f32]) -> [f32; N_BINS] {
let mut windowed = [0.0_f32; FFT_SIZE];
for (idx, sample) in frame.iter().take(WINDOW_16K).enumerate() {
let hann = 0.5 - 0.5 * (2.0 * std::f32::consts::PI * idx as f32 / WINDOW_16K as f32).cos();
windowed[idx] = f32_to_i16(*sample) as f32 * hann;
}
let mut out = [0.0_f32; N_BINS];
for (k, dst) in out.iter_mut().enumerate() {
let mut re = 0.0_f32;
let mut im = 0.0_f32;
for (n, &x) in windowed.iter().enumerate() {
let phase = -2.0 * std::f32::consts::PI * k as f32 * n as f32 / FFT_SIZE as f32;
re += x * phase.cos();
im += x * phase.sin();
}
*dst = re * re + im * im;
}
out
}
fn estimate_pitch_hz(frame: &[f32]) -> f32 {
let min_lag = (SAMPLE_RATE_16K / 400.0) as usize;
let max_lag = (SAMPLE_RATE_16K / 60.0) as usize;
let mut best_lag = 0_usize;
let mut best_corr = 0.0_f32;
for lag in min_lag..=max_lag.min(frame.len().saturating_sub(1)) {
let mut corr = 0.0_f32;
let mut energy = 0.0_f32;
for i in lag..frame.len() {
corr += frame[i] * frame[i - lag];
energy += frame[i - lag] * frame[i - lag];
}
let norm = if energy > 1.0e-8 {
corr / energy.sqrt()
} else {
0.0
};
if norm > best_corr {
best_corr = norm;
best_lag = lag;
}
}
if best_lag == 0 || best_corr < 0.01 {
0.0
} else {
SAMPLE_RATE_16K / best_lag as f32
}
}
// ---------------------------------------------------------------------------
// Background worker — same pattern as SileroOnnxVadWorker so the realtime
// callback never blocks on STFT / pitch / ONNX inference.
// ---------------------------------------------------------------------------
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64};
use std::sync::Arc;
use std::thread::JoinHandle;
/// Maximum number of 10 ms frames the worker may lag before the callback
/// treats its output as stale and uses WebRTC fallback instead.
const TEN_MAX_STALE_FRAMES: u64 = 8;
struct TenFrameMessage {
seq: u64,
frame: [f32; INPUT_FRAME_10MS],
}
/// Background TEN VAD worker. The realtime callback only enqueues 10 ms
/// frames and reads the latest probability atomically.
pub struct TenOnnxVadWorker {
tx: Option<std::sync::mpsc::SyncSender<TenFrameMessage>>,
latest_probability: Arc<AtomicU32>,
latest_processed_seq: Arc<AtomicU64>,
alive: Arc<AtomicBool>,
handle: Option<JoinHandle<()>>,
}
impl TenOnnxVadWorker {
/// Start a background TEN worker if the model loads.
pub fn try_new(model_path: &str) -> Option<Self> {
let vad = TenOnnxVad::try_new(model_path)?;
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<TenFrameMessage>(32);
let prob_arc = latest_probability.clone();
let seq_arc = latest_processed_seq.clone();
let alive_arc = alive.clone();
let handle = std::thread::Builder::new()
.name("chanora-ten-vad".to_string())
.spawn(move || {
let mut vad = vad;
while alive_arc.load(std::sync::atomic::Ordering::Relaxed) {
let msg = match rx.recv() {
Ok(m) => m,
Err(_) => break,
};
let mut frame_f32 = [0.0_f32; INPUT_FRAME_10MS];
frame_f32.copy_from_slice(&msg.frame);
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame_f32);
prob_arc.store(
out.probability.clamp(0.0, 1.0).to_bits(),
std::sync::atomic::Ordering::Relaxed,
);
seq_arc.store(msg.seq, std::sync::atomic::Ordering::Relaxed);
}
})
.ok()?;
Some(Self {
tx: Some(tx),
latest_probability,
latest_processed_seq,
alive,
handle: Some(handle),
})
}
/// Best-effort enqueue of a 10 ms frame for background inference.
pub fn try_send(&self, seq: u64, frame: &[f32; INPUT_FRAME_10MS]) -> bool {
let Some(tx) = &self.tx else {
return false;
};
tx.try_send(TenFrameMessage { seq, frame: *frame }).is_ok()
}
/// Latest probability published by the background worker.
pub fn latest_probability(&self) -> f32 {
f32::from_bits(
self.latest_probability
.load(std::sync::atomic::Ordering::Relaxed),
)
}
/// True when the worker is too far behind to trust its output.
pub fn is_stale(&self, capture_seq: u64) -> bool {
let latest = self
.latest_processed_seq
.load(std::sync::atomic::Ordering::Relaxed);
latest == u64::MAX || capture_seq.saturating_sub(latest) > TEN_MAX_STALE_FRAMES
}
}
impl Drop for TenOnnxVadWorker {
fn drop(&mut self) {
self.alive
.store(false, std::sync::atomic::Ordering::Relaxed);
drop(self.tx.take());
if let Some(h) = self.handle.take() {
let _ = h.join();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mel_filter_bank_has_expected_shape() {
let filters = build_mel_filters();
assert_eq!(filters.len(), MEL_BANDS);
assert!(filters.iter().all(|f| f.iter().any(|&v| v > 0.0)));
}
#[test]
fn preprocessing_produces_finite_features() {
let filters = build_mel_filters();
let frame = vec![0.0_f32; WINDOW_16K];
let feature = compute_feature(&filters, &frame);
assert!(feature.iter().all(|v| v.is_finite()));
}
}
+19 -4
View File
@@ -1725,6 +1725,21 @@ pub async fn set_vad_model_path(path: String) -> Result<(), BridgeError> {
Ok(())
}
/// Configure the TEN VAD ONNX model path.
pub async fn set_ten_vad_model_path(path: String) -> Result<(), BridgeError> {
if path.trim().is_empty() {
return Err(BridgeError::InvalidCommand(
"ten vad model path must not be empty".to_string(),
));
}
runtime()
.spawn(async move { chanora_audio::vad::set_ten_model_path(&path).map_err(|e| e) })
.await
.map_err(|e| task_join_error("set_ten_vad_model_path", e))?
.map_err(|e| BridgeError::Unmapped(format!("set_ten_vad_model_path: {e}")))?;
Ok(())
}
/// Enable or disable audio debug WAV dumping.
pub async fn enable_audio_debug_wav_dump(enabled: bool) -> Result<(), BridgeError> {
runtime()
@@ -1746,20 +1761,20 @@ pub async fn set_ios_voice_processing_mode(
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => {
BridgeAudioBackend::PlatformVoiceProcessing
}
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeAudioBackend::Sonora,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeAudioBackend::WebrtcApm,
},
vad_backend: BridgeVadBackend::SileroOnnx,
aec: match mode {
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::Sonora,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm,
},
ns: match mode {
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::Sonora,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm,
},
agc: match mode {
BridgeIosVoiceProcessingMode::PlatformVoiceProcessing => BridgeEffectOwner::Platform,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::Sonora,
BridgeIosVoiceProcessingMode::SonoraExperimental => BridgeEffectOwner::WebrtcApm,
},
hpf_enabled: true,
limiter_enabled: true,
+140 -30
View File
@@ -38,7 +38,7 @@ flutter_rust_bridge::frb_generated_boilerplate!(
default_rust_auto_opaque = RustAutoOpaqueMoi,
);
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.12.0";
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -1835973251;
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -436507436;
// Section: executor
@@ -398,6 +398,41 @@ fn wire__crate__api__export_diagnostics_impl(
},
)
}
fn wire__crate__api__get_audio_processing_config_impl(
port_: flutter_rust_bridge::for_generated::MessagePort,
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
rust_vec_len_: i32,
data_len_: i32,
) {
FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
flutter_rust_bridge::for_generated::TaskInfo {
debug_name: "get_audio_processing_config",
port: Some(port_),
mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal,
},
move || {
let message = unsafe {
flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(
ptr_,
rust_vec_len_,
data_len_,
)
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
deserializer.end();
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::get_audio_processing_config().await?;
Ok(output_ok)
})()
.await,
)
}
},
)
}
fn wire__crate__api__get_ptt_binding_impl(
port_: flutter_rust_bridge::for_generated::MessagePort,
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
@@ -598,6 +633,38 @@ fn wire__crate__api__handle_media_services_reset_impl(
},
)
}
fn wire__crate__api__handle_media_services_reset_with_route_impl(
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
rust_vec_len_: i32,
data_len_: i32,
) -> flutter_rust_bridge::for_generated::WireSyncRust2DartSse {
FLUTTER_RUST_BRIDGE_HANDLER.wrap_sync::<flutter_rust_bridge::for_generated::SseCodec, _>(
flutter_rust_bridge::for_generated::TaskInfo {
debug_name: "handle_media_services_reset_with_route",
port: None,
mode: flutter_rust_bridge::for_generated::FfiCallMode::Sync,
},
move || {
let message = unsafe {
flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(
ptr_,
rust_vec_len_,
data_len_,
)
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
let api_route_class = <String>::sse_decode(&mut deserializer);
deserializer.end();
transform_result_sse::<_, ()>((move || {
let output_ok = Result::<_, ()>::Ok({
crate::api::handle_media_services_reset_with_route(api_route_class);
})?;
Ok(output_ok)
})())
},
)
}
fn wire__crate__api__handle_route_change_impl(
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
rust_vec_len_: i32,
@@ -1198,6 +1265,42 @@ fn wire__crate__api__set_release_tail_ms_impl(
},
)
}
fn wire__crate__api__set_ten_vad_model_path_impl(
port_: flutter_rust_bridge::for_generated::MessagePort,
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
rust_vec_len_: i32,
data_len_: i32,
) {
FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
flutter_rust_bridge::for_generated::TaskInfo {
debug_name: "set_ten_vad_model_path",
port: Some(port_),
mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal,
},
move || {
let message = unsafe {
flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(
ptr_,
rust_vec_len_,
data_len_,
)
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
let api_path = <String>::sse_decode(&mut deserializer);
deserializer.end();
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::set_ten_vad_model_path(api_path).await?;
Ok(output_ok)
})()
.await,
)
}
},
)
}
fn wire__crate__api__set_transmit_mode_impl(
port_: flutter_rust_bridge::for_generated::MessagePort,
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
@@ -2090,31 +2193,33 @@ fn pde_ffi_dispatcher_primary_impl(
7 => wire__crate__api__disconnect_impl(port, ptr, rust_vec_len, data_len),
8 => wire__crate__api__enable_audio_debug_wav_dump_impl(port, ptr, rust_vec_len, data_len),
9 => wire__crate__api__events_stream_impl(port, ptr, rust_vec_len, data_len),
11 => wire__crate__api__get_ptt_binding_impl(port, ptr, rust_vec_len, data_len),
12 => wire__crate__api__get_release_tail_ms_impl(port, ptr, rust_vec_len, data_len),
13 => wire__crate__api__get_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
18 => wire__crate__api__init_storage_impl(port, ptr, rust_vec_len, data_len),
19 => wire__crate__api__is_connected_impl(port, ptr, rust_vec_len, data_len),
20 => wire__crate__api__list_bookmarks_impl(port, ptr, rust_vec_len, data_len),
22 => wire__crate__api__move_to_channel_impl(port, ptr, rust_vec_len, data_len),
23 => wire__crate__api__ptt_descriptor_impl(port, ptr, rust_vec_len, data_len),
24 => wire__crate__api__set_audio_processing_config_impl(port, ptr, rust_vec_len, data_len),
25 => wire__crate__api__set_hard_mute_impl(port, ptr, rust_vec_len, data_len),
26 => wire__crate__api__set_input_muted_impl(port, ptr, rust_vec_len, data_len),
27 => {
11 => wire__crate__api__get_audio_processing_config_impl(port, ptr, rust_vec_len, data_len),
12 => wire__crate__api__get_ptt_binding_impl(port, ptr, rust_vec_len, data_len),
13 => wire__crate__api__get_release_tail_ms_impl(port, ptr, rust_vec_len, data_len),
14 => wire__crate__api__get_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
20 => wire__crate__api__init_storage_impl(port, ptr, rust_vec_len, data_len),
21 => wire__crate__api__is_connected_impl(port, ptr, rust_vec_len, data_len),
22 => wire__crate__api__list_bookmarks_impl(port, ptr, rust_vec_len, data_len),
24 => wire__crate__api__move_to_channel_impl(port, ptr, rust_vec_len, data_len),
25 => wire__crate__api__ptt_descriptor_impl(port, ptr, rust_vec_len, data_len),
26 => wire__crate__api__set_audio_processing_config_impl(port, ptr, rust_vec_len, data_len),
27 => wire__crate__api__set_hard_mute_impl(port, ptr, rust_vec_len, data_len),
28 => wire__crate__api__set_input_muted_impl(port, ptr, rust_vec_len, data_len),
29 => {
wire__crate__api__set_ios_voice_processing_mode_impl(port, ptr, rust_vec_len, data_len)
}
29 => wire__crate__api__set_output_gain_impl(port, ptr, rust_vec_len, data_len),
30 => wire__crate__api__set_output_muted_impl(port, ptr, rust_vec_len, data_len),
31 => wire__crate__api__set_ptt_impl(port, ptr, rust_vec_len, data_len),
32 => wire__crate__api__set_ptt_binding_impl(port, ptr, rust_vec_len, data_len),
33 => wire__crate__api__set_release_tail_ms_impl(port, ptr, rust_vec_len, data_len),
34 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
35 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
36 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len),
37 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len),
38 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len),
39 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len),
31 => wire__crate__api__set_output_gain_impl(port, ptr, rust_vec_len, data_len),
32 => wire__crate__api__set_output_muted_impl(port, ptr, rust_vec_len, data_len),
33 => wire__crate__api__set_ptt_impl(port, ptr, rust_vec_len, data_len),
34 => wire__crate__api__set_ptt_binding_impl(port, ptr, rust_vec_len, data_len),
35 => wire__crate__api__set_release_tail_ms_impl(port, ptr, rust_vec_len, data_len),
36 => wire__crate__api__set_ten_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
37 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
38 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
39 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len),
40 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len),
41 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len),
42 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len),
_ => unreachable!(),
}
}
@@ -2128,12 +2233,17 @@ fn pde_ffi_dispatcher_sync_impl(
// Codec=Pde (Serialization + dispatch), see doc to use other codecs
match func_id {
10 => wire__crate__api__export_diagnostics_impl(ptr, rust_vec_len, data_len),
14 => wire__crate__api__handle_interruption_began_impl(ptr, rust_vec_len, data_len),
15 => wire__crate__api__handle_interruption_ended_impl(ptr, rust_vec_len, data_len),
16 => wire__crate__api__handle_media_services_reset_impl(ptr, rust_vec_len, data_len),
17 => wire__crate__api__handle_route_change_impl(ptr, rust_vec_len, data_len),
21 => wire__crate__api__log_file_path_str_impl(ptr, rust_vec_len, data_len),
28 => wire__crate__api__set_network_state_impl(ptr, rust_vec_len, data_len),
15 => wire__crate__api__handle_interruption_began_impl(ptr, rust_vec_len, data_len),
16 => wire__crate__api__handle_interruption_ended_impl(ptr, rust_vec_len, data_len),
17 => wire__crate__api__handle_media_services_reset_impl(ptr, rust_vec_len, data_len),
18 => wire__crate__api__handle_media_services_reset_with_route_impl(
ptr,
rust_vec_len,
data_len,
),
19 => wire__crate__api__handle_route_change_impl(ptr, rust_vec_len, data_len),
23 => wire__crate__api__log_file_path_str_impl(ptr, rust_vec_len, data_len),
30 => wire__crate__api__set_network_state_impl(ptr, rust_vec_len, data_len),
_ => unreachable!(),
}
}