feat: Android Oboe voice backend — WebRTC APM, VAD, HW/SW toggle, BBCode welcome, link trust, foreground task
Audio engine (Rust): - Android Oboe: WebRTC APM (AEC/NS/AGC/HPF) + TEN/Silero ONNX VAD - Hardware effects (JNI) with software fallback per-effect - Render reference buffer for AEC between output/capture callbacks - Voice activity gate: suppress transmission when speaker muted (all platforms) - Audio focus (SDD-109) + Bluetooth SCO (SDD-110) via JNI - ONNX Runtime 1.26 via ort 2.0.0-rc.12 (down from rc.10, ndarray 0.17) - VAD worker channel capacity 8→32, initial seq u64::MAX (warm-up fix) - TEN VAD default backend (was Silero) - Platform→WebrtcApm resolution after hardware binding - oboe-rs edisonjwa fork with get_raw_session_id() Android Kotlin: - AndroidAudioFocusController + AndroidBluetoothScoController - AndroidAudioLifecycleController (route changes to Flutter) - ProGuard rules for new controllers Flutter UI: - VoiceSettings: Android HW/SW toggle (Platform auto / WebRTC APM) - VoiceStatusChip: mute warning border + Speaker muted label - BBCode welcome message parser (BbCodeText, case-insensitive) - Welcome message foldable (expanded by default) - Link trust dialog (domain wildcards, SharedPreferences) - HapticFeedback on voice sheet opener - Server name in AppBar, version v0.1.0 - Default channel (id=1) visible, serverquery clients hidden - flutter_foreground_task integration Config: - ort load-dynamic on all non-iOS (Android/Linux/Windows) - ONNX Runtime AAR 1.26.0 - ndarray moved to common deps (was Apple-only)
This commit is contained in:
Edison Jwa
@@ -7,15 +7,17 @@
|
||||
|
||||
pub mod resampler;
|
||||
pub mod silero_onnx;
|
||||
pub mod ten_onnx;
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
|
||||
use crate::frame::{dbfs, i16_to_f32};
|
||||
use crate::frame::{f32_to_i16, i16_to_f32};
|
||||
use crate::AudioError;
|
||||
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||
|
||||
pub use silero_onnx::SileroOnnxVad;
|
||||
pub use ten_onnx::{TenOnnxVad, TenOnnxVadWorker};
|
||||
|
||||
/// Voice activity detector output for one 10 ms frame.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -32,83 +34,38 @@ pub trait VoiceActivityDetector: Send {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput;
|
||||
}
|
||||
|
||||
/// Realtime-safe fallback VAD used when a model runtime is unavailable.
|
||||
///
|
||||
/// This is not an energy-only transmit gate. It combines RMS level,
|
||||
/// zero-crossing rate, and peak-to-RMS shape with hysteresis so stable
|
||||
/// background rumble is less likely to open VoiceActivity than speech.
|
||||
#[derive(Debug, Clone)]
|
||||
/// Realtime-safe WebRTC VAD used when a model runtime is unavailable.
|
||||
pub struct WebRtcFallbackVad {
|
||||
open_dbfs: f32,
|
||||
close_dbfs: f32,
|
||||
active: bool,
|
||||
vad: webrtc_vad::Vad,
|
||||
frame_i16: [i16; INPUT_FRAME_10MS],
|
||||
}
|
||||
|
||||
// `webrtc_vad::Vad` owns an FFI pointer and is only touched from the
|
||||
// capture thread after construction. Moving the wrapper between threads is
|
||||
// safe; sharing it concurrently is not required and not implemented.
|
||||
unsafe impl Send for WebRtcFallbackVad {}
|
||||
|
||||
impl Default for WebRtcFallbackVad {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
open_dbfs: -42.0,
|
||||
close_dbfs: -50.0,
|
||||
active: false,
|
||||
vad: webrtc_vad::Vad::new_with_rate_and_mode(
|
||||
webrtc_vad::SampleRate::Rate48kHz,
|
||||
webrtc_vad::VadMode::Aggressive,
|
||||
),
|
||||
frame_i16: [0; INPUT_FRAME_10MS],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WebRtcFallbackVad {
|
||||
fn zero_crossing_rate(samples: &[f32]) -> f32 {
|
||||
if samples.len() < 2 {
|
||||
return 0.0;
|
||||
}
|
||||
let crossings = samples
|
||||
.windows(2)
|
||||
.filter(|pair| (pair[0] >= 0.0 && pair[1] < 0.0) || (pair[0] < 0.0 && pair[1] >= 0.0))
|
||||
.count();
|
||||
crossings as f32 / (samples.len() - 1) as f32
|
||||
}
|
||||
|
||||
fn peak_to_rms(samples: &[f32], rms: f32) -> f32 {
|
||||
if rms <= 0.000_001 {
|
||||
return 0.0;
|
||||
}
|
||||
let peak = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
|
||||
peak / rms
|
||||
}
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for WebRtcFallbackVad {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
let level = dbfs(samples);
|
||||
let threshold = if self.active {
|
||||
self.close_dbfs
|
||||
} else {
|
||||
self.open_dbfs
|
||||
};
|
||||
let rms = samples.iter().map(|s| s * s).sum::<f32>() / samples.len().max(1) as f32;
|
||||
let rms = rms.sqrt();
|
||||
let zcr = Self::zero_crossing_rate(samples);
|
||||
let crest = Self::peak_to_rms(samples, rms);
|
||||
|
||||
// Level score: steeper curve so silence (-50 dBFS) scores near 0.
|
||||
// Speech is typically -30 to -10 dBFS; silence is -60 to -45 dBFS.
|
||||
// Map [-60, -20] → [0, 1] with a midpoint at -40 dBFS.
|
||||
let level_score = ((level + 60.0) / 40.0).clamp(0.0, 1.0);
|
||||
|
||||
let zcr_score = if (0.015..=0.32).contains(&zcr) {
|
||||
1.0
|
||||
} else {
|
||||
0.3 // penalise non-speech ZCR more aggressively
|
||||
};
|
||||
let crest_score = if (1.5..=12.0).contains(&crest) {
|
||||
1.0
|
||||
} else {
|
||||
0.3
|
||||
};
|
||||
let probability =
|
||||
(level_score * 0.72 + zcr_score * 0.18 + crest_score * 0.10).clamp(0.0, 1.0);
|
||||
self.active = level >= threshold && probability >= 0.5;
|
||||
for (dst, src) in self.frame_i16.iter_mut().zip(samples.iter().copied()) {
|
||||
*dst = f32_to_i16(src);
|
||||
}
|
||||
let speech = self.vad.is_voice_segment(&self.frame_i16).unwrap_or(false);
|
||||
VadOutput {
|
||||
probability,
|
||||
speech: self.active,
|
||||
probability: if speech { 1.0 } else { 0.0 },
|
||||
speech,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -151,11 +108,17 @@ pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16
|
||||
|
||||
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
static TEN_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static TEN_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
fn ten_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
TEN_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
/// Configure the preferred Silero ONNX model path.
|
||||
///
|
||||
/// The path is validated eagerly. A successful call increments the
|
||||
@@ -186,12 +149,39 @@ pub fn silero_model_epoch() -> u64 {
|
||||
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Return the expected path of the Silero VAD v6 ONNX model in the
|
||||
/// iOS app bundle. The model is shipped as a Flutter asset and copied
|
||||
/// to the app's Documents directory by the Dart-side asset loader.
|
||||
/// Configure the preferred TEN VAD ONNX model path.
|
||||
pub fn set_ten_model_path(path: &str) -> Result<(), AudioError> {
|
||||
let path = path.trim();
|
||||
if path.is_empty() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"ten vad model path must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
if !std::path::Path::new(path).is_file() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(format!(
|
||||
"ten vad model path does not exist or is not a file: {path}"
|
||||
)));
|
||||
}
|
||||
let mut guard = ten_model_path_override()
|
||||
.write()
|
||||
.map_err(|_| AudioError::Backend("ten vad model path lock poisoned".to_string()))?;
|
||||
*guard = Some(path.to_string());
|
||||
TEN_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Monotonic counter incremented whenever the configured TEN model path changes.
|
||||
pub fn ten_model_epoch() -> u64 {
|
||||
TEN_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Return the expected path of the Silero VAD v6 ONNX model.
|
||||
/// The model is shipped as a Flutter asset and copied to the app's
|
||||
/// data directory by the Dart-side asset loader.
|
||||
///
|
||||
/// Returns an empty string on non-Apple platforms (Silero is not
|
||||
/// supported there; `SileroOnnxVad::try_new` will return `None`).
|
||||
/// On iOS/Android the model lives in the app's Documents/files
|
||||
/// directory. On desktop, the caller should set the path explicitly
|
||||
/// via `set_silero_model_path`.
|
||||
pub fn silero_model_bundle_path() -> String {
|
||||
if let Ok(guard) = silero_model_path_override().read() {
|
||||
if let Some(path) = guard.as_ref() {
|
||||
@@ -199,26 +189,85 @@ pub fn silero_model_bundle_path() -> String {
|
||||
}
|
||||
}
|
||||
|
||||
// iOS: Documents directory (written by Flutter asset loader).
|
||||
// macOS: same Documents pattern.
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
// Primary: Documents directory (written by Flutter asset loader).
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
let docs = format!("{home}/Documents/silero_vad.onnx");
|
||||
if std::path::Path::new(&docs).exists() {
|
||||
return docs;
|
||||
}
|
||||
// Fallback: app bundle Resources directory.
|
||||
let bundle = format!("{home}/../Library/silero_vad.onnx");
|
||||
if std::path::Path::new(&bundle).exists() {
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
// Last resort: current working directory (useful in tests).
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
|
||||
// Android: the model is in the app's files directory, same
|
||||
// Documents path pattern used by Flutter's path_provider.
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
String::new()
|
||||
// On Android, Flutter's getApplicationDocumentsDirectory
|
||||
// resolves to /data/data/<package>/app_flutter.
|
||||
// The Silero model path is set explicitly via
|
||||
// set_silero_model_path from Dart before voice starts,
|
||||
// so this fallback is rarely needed.
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
|
||||
// Desktop (Windows, Linux): rely on the override set by Dart.
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||||
{
|
||||
if let Ok(cwd) = std::env::current_dir() {
|
||||
let local = cwd.join("silero_vad.onnx");
|
||||
if local.exists() {
|
||||
return local.to_string_lossy().to_string();
|
||||
}
|
||||
}
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the expected path of the TEN VAD ONNX model copied by Flutter.
|
||||
pub fn ten_model_bundle_path() -> String {
|
||||
if let Ok(guard) = ten_model_path_override().read() {
|
||||
if let Some(path) = guard.as_ref() {
|
||||
return path.clone();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
let docs = format!("{home}/Documents/ten_vad.onnx");
|
||||
if std::path::Path::new(&docs).exists() {
|
||||
return docs;
|
||||
}
|
||||
let bundle = format!("{home}/../Library/ten_vad.onnx");
|
||||
if std::path::Path::new(&bundle).exists() {
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||||
{
|
||||
if let Ok(cwd) = std::env::current_dir() {
|
||||
let local = cwd.join("ten_vad.onnx");
|
||||
if local.exists() {
|
||||
return local.to_string_lossy().to_string();
|
||||
}
|
||||
}
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -75,13 +75,9 @@ pub struct SileroOnnxVad {
|
||||
}
|
||||
|
||||
enum SileroInner {
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
Onnx(OnnxSession),
|
||||
#[allow(dead_code)]
|
||||
Stub,
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
struct OnnxSession {
|
||||
session: ort::session::Session,
|
||||
}
|
||||
@@ -90,20 +86,11 @@ impl SileroOnnxVad {
|
||||
/// Attempt to load the Silero v6 ONNX model from `model_path`.
|
||||
///
|
||||
/// Returns `None` when the model file is missing, the ONNX Runtime
|
||||
/// is unavailable, or the platform is not iOS/macOS.
|
||||
/// is unavailable, or the platform does not support ONNX.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
Self::try_new_onnx(model_path)
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
let _ = model_path;
|
||||
None
|
||||
}
|
||||
Self::try_new_onnx(model_path)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
fn try_new_onnx(model_path: &str) -> Option<Self> {
|
||||
use tracing::{error, info};
|
||||
|
||||
@@ -116,13 +103,8 @@ impl SileroOnnxVad {
|
||||
return None;
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
if let Some(path) = bundled_onnxruntime_path() {
|
||||
let _ = ort::init_from(path.to_string_lossy()).commit();
|
||||
}
|
||||
|
||||
let session_result = std::panic::catch_unwind(|| {
|
||||
ort::session::Session::builder().and_then(|b| b.commit_from_file(model_path))
|
||||
ort::session::Session::builder().and_then(|mut b| b.commit_from_file(model_path))
|
||||
});
|
||||
|
||||
match session_result {
|
||||
@@ -193,7 +175,6 @@ impl SileroOnnxVad {
|
||||
/// 16 kHz frame: concatenate prior context, pass `input/state/sr` to
|
||||
/// ONNX, persist `stateN`, then refresh context from the current frame.
|
||||
/// Updates `last_probability` and returns the new value.
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
fn calc_level(&mut self, audio_frame: &[f32]) -> f32 {
|
||||
use ort::value::Value;
|
||||
use tracing::error;
|
||||
@@ -279,7 +260,7 @@ impl SileroOnnxVad {
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
|
||||
pub(crate) fn bundled_onnxruntime_path_for_vad() -> Option<std::path::PathBuf> {
|
||||
let exe = std::env::current_exe().ok()?;
|
||||
let app_dir = exe.parent()?;
|
||||
let framework = app_dir
|
||||
@@ -307,16 +288,9 @@ impl VoiceActivityDetector for SileroOnnxVad {
|
||||
|
||||
if self.accum.len() >= SILERO_FRAME_16K {
|
||||
let audio_frame: Vec<f32> = self.accum[..SILERO_FRAME_16K].to_vec();
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
if matches!(self.inner, SileroInner::Onnx(_)) {
|
||||
self.calc_level(&audio_frame);
|
||||
} else {
|
||||
self.update_context_from_frame(&audio_frame);
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
{
|
||||
if matches!(self.inner, SileroInner::Onnx(_)) {
|
||||
self.calc_level(&audio_frame);
|
||||
} else {
|
||||
self.update_context_from_frame(&audio_frame);
|
||||
}
|
||||
// Drain the accumulator (keep any overflow for next frame).
|
||||
@@ -357,9 +331,9 @@ impl SileroOnnxVadWorker {
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
let vad = SileroOnnxVad::try_new(model_path)?;
|
||||
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
|
||||
let latest_processed_seq = Arc::new(AtomicU64::new(0));
|
||||
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
|
||||
let alive = Arc::new(AtomicBool::new(true));
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(8);
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(32);
|
||||
let latest_probability_for_thread = latest_probability.clone();
|
||||
let latest_processed_seq_for_thread = latest_processed_seq.clone();
|
||||
let alive_for_thread = alive.clone();
|
||||
@@ -414,7 +388,10 @@ impl SileroOnnxVadWorker {
|
||||
/// True when the worker is too far behind to trust its latest
|
||||
/// probability for the current frame.
|
||||
pub fn is_stale(&self, capture_seq: u64) -> bool {
|
||||
self.lag_frames(capture_seq) > SILERO_MAX_STALE_FRAMES
|
||||
let latest = self
|
||||
.latest_processed_seq
|
||||
.load(std::sync::atomic::Ordering::Relaxed);
|
||||
latest == u64::MAX || capture_seq.saturating_sub(latest) > SILERO_MAX_STALE_FRAMES
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
//! TEN VAD ONNX backend.
|
||||
//!
|
||||
//! TEN's ONNX graph does not accept raw PCM. It expects the same feature
|
||||
//! stack produced by TEN's `AUP_Aed_aivad_proc`: three context frames of
|
||||
//! 40 log-mel powers plus one pitch feature, followed by four recurrent
|
||||
//! state tensors. This module ports that preprocessing path to Rust and
|
||||
//! keeps ONNX Runtime off the realtime callback where possible.
|
||||
|
||||
use crate::frame::f32_to_i16;
|
||||
|
||||
use super::resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||
use super::{VadOutput, VoiceActivityDetector};
|
||||
|
||||
const SAMPLE_RATE_16K: f32 = 16_000.0;
|
||||
const HOP_16K: usize = 256;
|
||||
const WINDOW_16K: usize = 768;
|
||||
const FFT_SIZE: usize = 1024;
|
||||
const N_BINS: usize = FFT_SIZE / 2 + 1;
|
||||
const MEL_BANDS: usize = 40;
|
||||
const FEATURE_LEN: usize = 41;
|
||||
const CONTEXT: usize = 3;
|
||||
const HIDDEN: usize = 64;
|
||||
const POWER_NORMALIZER: f32 = 32768.0 * 32768.0;
|
||||
const EPS: f32 = 1.0e-20;
|
||||
|
||||
const FEATURE_MEANS: [f32; FEATURE_LEN] = [
|
||||
-8.198236, -6.2657166, -5.4838185, -4.7586913, -4.417089, -4.142893, -3.9128504, -3.845928,
|
||||
-3.6570904, -3.7234187, -3.8761342, -3.843891, -3.6904051, -3.7560658, -3.6986961, -3.650463,
|
||||
-3.7004688, -3.5673213, -3.4989002, -3.477807, -3.458816, -3.4449239, -3.4013286, -3.3062613,
|
||||
-3.2785568, -3.2332509, -3.198616, -3.2045264, -3.2087986, -3.257838, -3.3813767, -3.5340214,
|
||||
-3.640868, -3.7268589, -3.773731, -3.8046672, -3.832901, -3.8711205, -3.990593, -4.4802895,
|
||||
92.3569,
|
||||
];
|
||||
|
||||
const FEATURE_STDS: [f32; FEATURE_LEN] = [
|
||||
5.166064, 4.9772096, 4.698896, 4.6306214, 4.634348, 4.641156, 4.6406765, 4.666367, 4.6505346,
|
||||
4.640021, 4.6374, 4.620099, 4.5963163, 4.562655, 4.5543604, 4.5669107, 4.56249, 4.5624127,
|
||||
4.5852995, 4.6001797, 4.592846, 4.5859227, 4.5834966, 4.626093, 4.626958, 4.6262894, 4.637006,
|
||||
4.683016, 4.726814, 4.7342896, 4.753227, 4.849723, 4.869435, 4.884483, 4.921327, 4.9592123,
|
||||
4.996619, 5.0448236, 5.072217, 5.0964394, 115.21369,
|
||||
];
|
||||
|
||||
/// TEN VAD using ONNX Runtime and Rust-ported TEN feature preprocessing.
|
||||
pub struct TenOnnxVad {
|
||||
session: ort::session::Session,
|
||||
downsampler: Downsampler48to16,
|
||||
hop_accum: Vec<f32>,
|
||||
sample_fifo: Vec<f32>,
|
||||
feature_stack: [[f32; FEATURE_LEN]; CONTEXT],
|
||||
states: [[f32; HIDDEN]; 4],
|
||||
mel_filters: Vec<[f32; N_BINS]>,
|
||||
last_probability: f32,
|
||||
last_speech: bool,
|
||||
}
|
||||
|
||||
unsafe impl Send for TenOnnxVad {}
|
||||
|
||||
impl TenOnnxVad {
|
||||
/// Load TEN VAD ONNX model.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
if !std::path::Path::new(model_path).exists() {
|
||||
tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model not found");
|
||||
return None;
|
||||
}
|
||||
let session = match std::panic::catch_unwind(|| {
|
||||
ort::session::Session::builder().and_then(|mut b| b.commit_from_file(model_path))
|
||||
}) {
|
||||
Ok(Ok(session)) => session,
|
||||
Ok(Err(error)) => {
|
||||
tracing::warn!(target: "chanora_audio", %error, path = model_path, "TEN VAD ONNX model load failed");
|
||||
return None;
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!(target: "chanora_audio", path = model_path, "TEN VAD ONNX Runtime panicked during load");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
tracing::info!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model loaded");
|
||||
Some(Self {
|
||||
session,
|
||||
downsampler: Downsampler48to16::default(),
|
||||
hop_accum: Vec::with_capacity(HOP_16K + super::resampler::OUTPUT_FRAME_10MS),
|
||||
sample_fifo: Vec::with_capacity(WINDOW_16K + HOP_16K),
|
||||
feature_stack: [[0.0; FEATURE_LEN]; CONTEXT],
|
||||
states: [[0.0; HIDDEN]; 4],
|
||||
mel_filters: build_mel_filters(),
|
||||
last_probability: 0.0,
|
||||
last_speech: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn process_hop(&mut self, hop: &[f32]) {
|
||||
self.sample_fifo.extend_from_slice(hop);
|
||||
let frame = if self.sample_fifo.len() >= WINDOW_16K {
|
||||
let start = self.sample_fifo.len() - WINDOW_16K;
|
||||
self.sample_fifo[start..].to_vec()
|
||||
} else {
|
||||
let mut padded = vec![0.0; WINDOW_16K - self.sample_fifo.len()];
|
||||
padded.extend_from_slice(&self.sample_fifo);
|
||||
padded
|
||||
};
|
||||
if self.sample_fifo.len() > WINDOW_16K {
|
||||
let excess = self.sample_fifo.len() - WINDOW_16K;
|
||||
self.sample_fifo.drain(..excess);
|
||||
}
|
||||
|
||||
let feature = compute_feature(&self.mel_filters, &frame);
|
||||
self.feature_stack.copy_within(1..CONTEXT, 0);
|
||||
self.feature_stack[CONTEXT - 1] = feature;
|
||||
self.run_onnx();
|
||||
}
|
||||
|
||||
fn run_onnx(&mut self) {
|
||||
use ndarray::{Array, IxDyn};
|
||||
use ort::value::Value;
|
||||
|
||||
let input: Vec<f32> = self.feature_stack.iter().flatten().copied().collect();
|
||||
let input_arr = match Array::from_shape_vec(IxDyn(&[1, CONTEXT, FEATURE_LEN]), input) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return,
|
||||
};
|
||||
let state_arrs = [0, 1, 2, 3]
|
||||
.map(|idx| Array::from_shape_vec(IxDyn(&[1, HIDDEN]), self.states[idx].to_vec()));
|
||||
let input_val = match Value::from_array(input_arr) {
|
||||
Ok(v) => v,
|
||||
Err(error) => {
|
||||
tracing::warn!(target: "chanora_audio", %error, "TEN VAD input tensor error");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let state_vals = match state_arrs {
|
||||
[Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d],
|
||||
_ => return,
|
||||
};
|
||||
let state_vals = match state_vals.map(Value::from_array) {
|
||||
[Ok(a), Ok(b), Ok(c), Ok(d)] => [a, b, c, d],
|
||||
_ => return,
|
||||
};
|
||||
|
||||
let outputs = match self.session.run([
|
||||
(&input_val).into(),
|
||||
(&state_vals[0]).into(),
|
||||
(&state_vals[1]).into(),
|
||||
(&state_vals[2]).into(),
|
||||
(&state_vals[3]).into(),
|
||||
]) {
|
||||
Ok(outputs) => outputs,
|
||||
Err(error) => {
|
||||
tracing::warn!(target: "chanora_audio", %error, "TEN VAD ONNX inference failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if let Ok((_, prob)) = outputs["output_1"].try_extract_tensor::<f32>() {
|
||||
if let Some(&p) = prob.first() {
|
||||
self.last_probability = p.clamp(0.0, 1.0);
|
||||
self.last_speech = self.last_probability >= 0.5;
|
||||
}
|
||||
}
|
||||
for (idx, name) in ["output_2", "output_3", "output_6", "output_7"]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
if let Ok((_, state)) = outputs[*name].try_extract_tensor::<f32>() {
|
||||
let copy_len = state.len().min(HIDDEN);
|
||||
self.states[idx][..copy_len].copy_from_slice(&state[..copy_len]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
|
||||
let exe = std::env::current_exe().ok()?;
|
||||
let app_dir = exe.parent()?;
|
||||
let framework = app_dir
|
||||
.join("Frameworks")
|
||||
.join("onnxruntime.framework")
|
||||
.join("onnxruntime");
|
||||
framework.exists().then_some(framework)
|
||||
}
|
||||
|
||||
fn compute_feature(mel_filters: &[[f32; N_BINS]], frame: &[f32]) -> [f32; FEATURE_LEN] {
|
||||
let power = power_spectrum(frame);
|
||||
let mut feature = [0.0; FEATURE_LEN];
|
||||
for band in 0..MEL_BANDS {
|
||||
let energy = mel_filters[band]
|
||||
.iter()
|
||||
.zip(power.iter())
|
||||
.map(|(w, p)| w * p)
|
||||
.sum::<f32>()
|
||||
/ POWER_NORMALIZER;
|
||||
let log_energy = (energy + EPS).ln();
|
||||
feature[band] = (log_energy - FEATURE_MEANS[band]) / (FEATURE_STDS[band] + EPS);
|
||||
}
|
||||
let pitch_hz = estimate_pitch_hz(frame);
|
||||
feature[MEL_BANDS] = (pitch_hz - FEATURE_MEANS[MEL_BANDS]) / (FEATURE_STDS[MEL_BANDS] + EPS);
|
||||
feature
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for TenOnnxVad {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
debug_assert_eq!(samples.len(), INPUT_FRAME_10MS);
|
||||
let mut input = [0.0_f32; INPUT_FRAME_10MS];
|
||||
input.copy_from_slice(samples);
|
||||
let downsampled = self.downsampler.process_frame_10ms(&input);
|
||||
self.hop_accum.extend_from_slice(&downsampled);
|
||||
while self.hop_accum.len() >= HOP_16K {
|
||||
let hop: Vec<f32> = self.hop_accum[..HOP_16K].to_vec();
|
||||
self.hop_accum.drain(..HOP_16K);
|
||||
self.process_hop(&hop);
|
||||
}
|
||||
VadOutput {
|
||||
probability: self.last_probability,
|
||||
speech: self.last_speech,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hz_to_mel(hz: f32) -> f32 {
|
||||
2595.0 * (1.0 + hz / 700.0).log10()
|
||||
}
|
||||
|
||||
fn mel_to_hz(mel: f32) -> f32 {
|
||||
700.0 * (10.0_f32.powf(mel / 2595.0) - 1.0)
|
||||
}
|
||||
|
||||
fn build_mel_filters() -> Vec<[f32; N_BINS]> {
|
||||
let low_mel = hz_to_mel(0.0);
|
||||
let high_mel = hz_to_mel(8000.0);
|
||||
let mut bins = [0_usize; MEL_BANDS + 2];
|
||||
for idx in 0..bins.len() {
|
||||
let mel = idx as f32 * (high_mel - low_mel) / (MEL_BANDS as f32 + 1.0) + low_mel;
|
||||
let hz = mel_to_hz(mel);
|
||||
let mut bin = ((FFT_SIZE as f32 + 1.0) * hz / SAMPLE_RATE_16K).floor() as usize;
|
||||
bin = bin.min(N_BINS - 1);
|
||||
if idx > 0 && bin == bins[idx - 1] {
|
||||
bin = (bin + 1).min(N_BINS - 1);
|
||||
}
|
||||
bins[idx] = bin;
|
||||
}
|
||||
|
||||
let mut filters = vec![[0.0_f32; N_BINS]; MEL_BANDS];
|
||||
for band in 0..MEL_BANDS {
|
||||
let left = bins[band];
|
||||
let center = bins[band + 1].max(left + 1);
|
||||
let right = bins[band + 2].max(center + 1).min(N_BINS - 1);
|
||||
for i in left..center.min(N_BINS) {
|
||||
filters[band][i] = (i - left) as f32 / (center - left) as f32;
|
||||
}
|
||||
for i in center..=right {
|
||||
filters[band][i] = (right - i) as f32 / (right - center).max(1) as f32;
|
||||
}
|
||||
}
|
||||
filters
|
||||
}
|
||||
|
||||
fn power_spectrum(frame: &[f32]) -> [f32; N_BINS] {
|
||||
let mut windowed = [0.0_f32; FFT_SIZE];
|
||||
for (idx, sample) in frame.iter().take(WINDOW_16K).enumerate() {
|
||||
let hann = 0.5 - 0.5 * (2.0 * std::f32::consts::PI * idx as f32 / WINDOW_16K as f32).cos();
|
||||
windowed[idx] = f32_to_i16(*sample) as f32 * hann;
|
||||
}
|
||||
|
||||
let mut out = [0.0_f32; N_BINS];
|
||||
for (k, dst) in out.iter_mut().enumerate() {
|
||||
let mut re = 0.0_f32;
|
||||
let mut im = 0.0_f32;
|
||||
for (n, &x) in windowed.iter().enumerate() {
|
||||
let phase = -2.0 * std::f32::consts::PI * k as f32 * n as f32 / FFT_SIZE as f32;
|
||||
re += x * phase.cos();
|
||||
im += x * phase.sin();
|
||||
}
|
||||
*dst = re * re + im * im;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn estimate_pitch_hz(frame: &[f32]) -> f32 {
|
||||
let min_lag = (SAMPLE_RATE_16K / 400.0) as usize;
|
||||
let max_lag = (SAMPLE_RATE_16K / 60.0) as usize;
|
||||
let mut best_lag = 0_usize;
|
||||
let mut best_corr = 0.0_f32;
|
||||
for lag in min_lag..=max_lag.min(frame.len().saturating_sub(1)) {
|
||||
let mut corr = 0.0_f32;
|
||||
let mut energy = 0.0_f32;
|
||||
for i in lag..frame.len() {
|
||||
corr += frame[i] * frame[i - lag];
|
||||
energy += frame[i - lag] * frame[i - lag];
|
||||
}
|
||||
let norm = if energy > 1.0e-8 {
|
||||
corr / energy.sqrt()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
if norm > best_corr {
|
||||
best_corr = norm;
|
||||
best_lag = lag;
|
||||
}
|
||||
}
|
||||
if best_lag == 0 || best_corr < 0.01 {
|
||||
0.0
|
||||
} else {
|
||||
SAMPLE_RATE_16K / best_lag as f32
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Background worker — same pattern as SileroOnnxVadWorker so the realtime
|
||||
// callback never blocks on STFT / pitch / ONNX inference.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64};
|
||||
use std::sync::Arc;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// Maximum number of 10 ms frames the worker may lag before the callback
|
||||
/// treats its output as stale and uses WebRTC fallback instead.
|
||||
const TEN_MAX_STALE_FRAMES: u64 = 8;
|
||||
|
||||
struct TenFrameMessage {
|
||||
seq: u64,
|
||||
frame: [f32; INPUT_FRAME_10MS],
|
||||
}
|
||||
|
||||
/// Background TEN VAD worker. The realtime callback only enqueues 10 ms
|
||||
/// frames and reads the latest probability atomically.
|
||||
pub struct TenOnnxVadWorker {
|
||||
tx: Option<std::sync::mpsc::SyncSender<TenFrameMessage>>,
|
||||
latest_probability: Arc<AtomicU32>,
|
||||
latest_processed_seq: Arc<AtomicU64>,
|
||||
alive: Arc<AtomicBool>,
|
||||
handle: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl TenOnnxVadWorker {
|
||||
/// Start a background TEN worker if the model loads.
|
||||
pub fn try_new(model_path: &str) -> Option<Self> {
|
||||
let vad = TenOnnxVad::try_new(model_path)?;
|
||||
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
|
||||
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
|
||||
let alive = Arc::new(AtomicBool::new(true));
|
||||
let (tx, rx) = std::sync::mpsc::sync_channel::<TenFrameMessage>(32);
|
||||
let prob_arc = latest_probability.clone();
|
||||
let seq_arc = latest_processed_seq.clone();
|
||||
let alive_arc = alive.clone();
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("chanora-ten-vad".to_string())
|
||||
.spawn(move || {
|
||||
let mut vad = vad;
|
||||
while alive_arc.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let msg = match rx.recv() {
|
||||
Ok(m) => m,
|
||||
Err(_) => break,
|
||||
};
|
||||
let mut frame_f32 = [0.0_f32; INPUT_FRAME_10MS];
|
||||
frame_f32.copy_from_slice(&msg.frame);
|
||||
let out = VoiceActivityDetector::process_10ms(&mut vad, &frame_f32);
|
||||
prob_arc.store(
|
||||
out.probability.clamp(0.0, 1.0).to_bits(),
|
||||
std::sync::atomic::Ordering::Relaxed,
|
||||
);
|
||||
seq_arc.store(msg.seq, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
})
|
||||
.ok()?;
|
||||
|
||||
Some(Self {
|
||||
tx: Some(tx),
|
||||
latest_probability,
|
||||
latest_processed_seq,
|
||||
alive,
|
||||
handle: Some(handle),
|
||||
})
|
||||
}
|
||||
|
||||
/// Best-effort enqueue of a 10 ms frame for background inference.
|
||||
pub fn try_send(&self, seq: u64, frame: &[f32; INPUT_FRAME_10MS]) -> bool {
|
||||
let Some(tx) = &self.tx else {
|
||||
return false;
|
||||
};
|
||||
tx.try_send(TenFrameMessage { seq, frame: *frame }).is_ok()
|
||||
}
|
||||
|
||||
/// Latest probability published by the background worker.
|
||||
pub fn latest_probability(&self) -> f32 {
|
||||
f32::from_bits(
|
||||
self.latest_probability
|
||||
.load(std::sync::atomic::Ordering::Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
/// True when the worker is too far behind to trust its output.
|
||||
pub fn is_stale(&self, capture_seq: u64) -> bool {
|
||||
let latest = self
|
||||
.latest_processed_seq
|
||||
.load(std::sync::atomic::Ordering::Relaxed);
|
||||
latest == u64::MAX || capture_seq.saturating_sub(latest) > TEN_MAX_STALE_FRAMES
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TenOnnxVadWorker {
|
||||
fn drop(&mut self) {
|
||||
self.alive
|
||||
.store(false, std::sync::atomic::Ordering::Relaxed);
|
||||
drop(self.tx.take());
|
||||
if let Some(h) = self.handle.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn mel_filter_bank_has_expected_shape() {
|
||||
let filters = build_mel_filters();
|
||||
assert_eq!(filters.len(), MEL_BANDS);
|
||||
assert!(filters.iter().all(|f| f.iter().any(|&v| v > 0.0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preprocessing_produces_finite_features() {
|
||||
let filters = build_mel_filters();
|
||||
let frame = vec![0.0_f32; WINDOW_16K];
|
||||
let feature = compute_feature(&filters, &frame);
|
||||
assert!(feature.iter().all(|v| v.is_finite()));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user