feat: Android Oboe voice backend — WebRTC APM, VAD, HW/SW toggle, BBCode welcome, link trust, foreground task
Audio engine (Rust): - Android Oboe: WebRTC APM (AEC/NS/AGC/HPF) + TEN/Silero ONNX VAD - Hardware effects (JNI) with software fallback per-effect - Render reference buffer for AEC between output/capture callbacks - Voice activity gate: suppress transmission when speaker muted (all platforms) - Audio focus (SDD-109) + Bluetooth SCO (SDD-110) via JNI - ONNX Runtime 1.26 via ort 2.0.0-rc.12 (down from rc.10, ndarray 0.17) - VAD worker channel capacity 8→32, initial seq u64::MAX (warm-up fix) - TEN VAD default backend (was Silero) - Platform→WebrtcApm resolution after hardware binding - oboe-rs edisonjwa fork with get_raw_session_id() Android Kotlin: - AndroidAudioFocusController + AndroidBluetoothScoController - AndroidAudioLifecycleController (route changes to Flutter) - ProGuard rules for new controllers Flutter UI: - VoiceSettings: Android HW/SW toggle (Platform auto / WebRTC APM) - VoiceStatusChip: mute warning border + Speaker muted label - BBCode welcome message parser (BbCodeText, case-insensitive) - Welcome message foldable (expanded by default) - Link trust dialog (domain wildcards, SharedPreferences) - HapticFeedback on voice sheet opener - Server name in AppBar, version v0.1.0 - Default channel (id=1) visible, serverquery clients hidden - flutter_foreground_task integration Config: - ort load-dynamic on all non-iOS (Android/Linux/Windows) - ONNX Runtime AAR 1.26.0 - ndarray moved to common deps (was Apple-only)
This commit is contained in:
Edison Jwa
@@ -7,15 +7,17 @@
|
||||
|
||||
pub mod resampler;
|
||||
pub mod silero_onnx;
|
||||
pub mod ten_onnx;
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{OnceLock, RwLock};
|
||||
|
||||
use crate::frame::{dbfs, i16_to_f32};
|
||||
use crate::frame::{f32_to_i16, i16_to_f32};
|
||||
use crate::AudioError;
|
||||
use resampler::{Downsampler48to16, INPUT_FRAME_10MS};
|
||||
|
||||
pub use silero_onnx::SileroOnnxVad;
|
||||
pub use ten_onnx::{TenOnnxVad, TenOnnxVadWorker};
|
||||
|
||||
/// Voice activity detector output for one 10 ms frame.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -32,83 +34,38 @@ pub trait VoiceActivityDetector: Send {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput;
|
||||
}
|
||||
|
||||
/// Realtime-safe fallback VAD used when a model runtime is unavailable.
|
||||
///
|
||||
/// This is not an energy-only transmit gate. It combines RMS level,
|
||||
/// zero-crossing rate, and peak-to-RMS shape with hysteresis so stable
|
||||
/// background rumble is less likely to open VoiceActivity than speech.
|
||||
#[derive(Debug, Clone)]
|
||||
/// Realtime-safe WebRTC VAD used when a model runtime is unavailable.
|
||||
pub struct WebRtcFallbackVad {
|
||||
open_dbfs: f32,
|
||||
close_dbfs: f32,
|
||||
active: bool,
|
||||
vad: webrtc_vad::Vad,
|
||||
frame_i16: [i16; INPUT_FRAME_10MS],
|
||||
}
|
||||
|
||||
// `webrtc_vad::Vad` owns an FFI pointer and is only touched from the
|
||||
// capture thread after construction. Moving the wrapper between threads is
|
||||
// safe; sharing it concurrently is not required and not implemented.
|
||||
unsafe impl Send for WebRtcFallbackVad {}
|
||||
|
||||
impl Default for WebRtcFallbackVad {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
open_dbfs: -42.0,
|
||||
close_dbfs: -50.0,
|
||||
active: false,
|
||||
vad: webrtc_vad::Vad::new_with_rate_and_mode(
|
||||
webrtc_vad::SampleRate::Rate48kHz,
|
||||
webrtc_vad::VadMode::Aggressive,
|
||||
),
|
||||
frame_i16: [0; INPUT_FRAME_10MS],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WebRtcFallbackVad {
|
||||
fn zero_crossing_rate(samples: &[f32]) -> f32 {
|
||||
if samples.len() < 2 {
|
||||
return 0.0;
|
||||
}
|
||||
let crossings = samples
|
||||
.windows(2)
|
||||
.filter(|pair| (pair[0] >= 0.0 && pair[1] < 0.0) || (pair[0] < 0.0 && pair[1] >= 0.0))
|
||||
.count();
|
||||
crossings as f32 / (samples.len() - 1) as f32
|
||||
}
|
||||
|
||||
fn peak_to_rms(samples: &[f32], rms: f32) -> f32 {
|
||||
if rms <= 0.000_001 {
|
||||
return 0.0;
|
||||
}
|
||||
let peak = samples.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
|
||||
peak / rms
|
||||
}
|
||||
}
|
||||
|
||||
impl VoiceActivityDetector for WebRtcFallbackVad {
|
||||
fn process_10ms(&mut self, samples: &[f32]) -> VadOutput {
|
||||
let level = dbfs(samples);
|
||||
let threshold = if self.active {
|
||||
self.close_dbfs
|
||||
} else {
|
||||
self.open_dbfs
|
||||
};
|
||||
let rms = samples.iter().map(|s| s * s).sum::<f32>() / samples.len().max(1) as f32;
|
||||
let rms = rms.sqrt();
|
||||
let zcr = Self::zero_crossing_rate(samples);
|
||||
let crest = Self::peak_to_rms(samples, rms);
|
||||
|
||||
// Level score: steeper curve so silence (-50 dBFS) scores near 0.
|
||||
// Speech is typically -30 to -10 dBFS; silence is -60 to -45 dBFS.
|
||||
// Map [-60, -20] → [0, 1] with a midpoint at -40 dBFS.
|
||||
let level_score = ((level + 60.0) / 40.0).clamp(0.0, 1.0);
|
||||
|
||||
let zcr_score = if (0.015..=0.32).contains(&zcr) {
|
||||
1.0
|
||||
} else {
|
||||
0.3 // penalise non-speech ZCR more aggressively
|
||||
};
|
||||
let crest_score = if (1.5..=12.0).contains(&crest) {
|
||||
1.0
|
||||
} else {
|
||||
0.3
|
||||
};
|
||||
let probability =
|
||||
(level_score * 0.72 + zcr_score * 0.18 + crest_score * 0.10).clamp(0.0, 1.0);
|
||||
self.active = level >= threshold && probability >= 0.5;
|
||||
for (dst, src) in self.frame_i16.iter_mut().zip(samples.iter().copied()) {
|
||||
*dst = f32_to_i16(src);
|
||||
}
|
||||
let speech = self.vad.is_voice_segment(&self.frame_i16).unwrap_or(false);
|
||||
VadOutput {
|
||||
probability,
|
||||
speech: self.active,
|
||||
probability: if speech { 1.0 } else { 0.0 },
|
||||
speech,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -151,11 +108,17 @@ pub fn process_i16_10ms(detector: &mut dyn VoiceActivityDetector, samples: &[i16
|
||||
|
||||
static SILERO_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static SILERO_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
static TEN_MODEL_PATH_OVERRIDE: OnceLock<RwLock<Option<String>>> = OnceLock::new();
|
||||
static TEN_MODEL_EPOCH: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn silero_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
SILERO_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
fn ten_model_path_override() -> &'static RwLock<Option<String>> {
|
||||
TEN_MODEL_PATH_OVERRIDE.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
/// Configure the preferred Silero ONNX model path.
|
||||
///
|
||||
/// The path is validated eagerly. A successful call increments the
|
||||
@@ -186,12 +149,39 @@ pub fn silero_model_epoch() -> u64 {
|
||||
SILERO_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Return the expected path of the Silero VAD v6 ONNX model in the
|
||||
/// iOS app bundle. The model is shipped as a Flutter asset and copied
|
||||
/// to the app's Documents directory by the Dart-side asset loader.
|
||||
/// Configure the preferred TEN VAD ONNX model path.
|
||||
pub fn set_ten_model_path(path: &str) -> Result<(), AudioError> {
|
||||
let path = path.trim();
|
||||
if path.is_empty() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(
|
||||
"ten vad model path must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
if !std::path::Path::new(path).is_file() {
|
||||
return Err(AudioError::InvalidAudioProcessingConfig(format!(
|
||||
"ten vad model path does not exist or is not a file: {path}"
|
||||
)));
|
||||
}
|
||||
let mut guard = ten_model_path_override()
|
||||
.write()
|
||||
.map_err(|_| AudioError::Backend("ten vad model path lock poisoned".to_string()))?;
|
||||
*guard = Some(path.to_string());
|
||||
TEN_MODEL_EPOCH.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Monotonic counter incremented whenever the configured TEN model path changes.
|
||||
pub fn ten_model_epoch() -> u64 {
|
||||
TEN_MODEL_EPOCH.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Return the expected path of the Silero VAD v6 ONNX model.
|
||||
/// The model is shipped as a Flutter asset and copied to the app's
|
||||
/// data directory by the Dart-side asset loader.
|
||||
///
|
||||
/// Returns an empty string on non-Apple platforms (Silero is not
|
||||
/// supported there; `SileroOnnxVad::try_new` will return `None`).
|
||||
/// On iOS/Android the model lives in the app's Documents/files
|
||||
/// directory. On desktop, the caller should set the path explicitly
|
||||
/// via `set_silero_model_path`.
|
||||
pub fn silero_model_bundle_path() -> String {
|
||||
if let Ok(guard) = silero_model_path_override().read() {
|
||||
if let Some(path) = guard.as_ref() {
|
||||
@@ -199,26 +189,85 @@ pub fn silero_model_bundle_path() -> String {
|
||||
}
|
||||
}
|
||||
|
||||
// iOS: Documents directory (written by Flutter asset loader).
|
||||
// macOS: same Documents pattern.
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
// Primary: Documents directory (written by Flutter asset loader).
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
let docs = format!("{home}/Documents/silero_vad.onnx");
|
||||
if std::path::Path::new(&docs).exists() {
|
||||
return docs;
|
||||
}
|
||||
// Fallback: app bundle Resources directory.
|
||||
let bundle = format!("{home}/../Library/silero_vad.onnx");
|
||||
if std::path::Path::new(&bundle).exists() {
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
// Last resort: current working directory (useful in tests).
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
|
||||
|
||||
// Android: the model is in the app's files directory, same
|
||||
// Documents path pattern used by Flutter's path_provider.
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
String::new()
|
||||
// On Android, Flutter's getApplicationDocumentsDirectory
|
||||
// resolves to /data/data/<package>/app_flutter.
|
||||
// The Silero model path is set explicitly via
|
||||
// set_silero_model_path from Dart before voice starts,
|
||||
// so this fallback is rarely needed.
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
|
||||
// Desktop (Windows, Linux): rely on the override set by Dart.
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||||
{
|
||||
if let Ok(cwd) = std::env::current_dir() {
|
||||
let local = cwd.join("silero_vad.onnx");
|
||||
if local.exists() {
|
||||
return local.to_string_lossy().to_string();
|
||||
}
|
||||
}
|
||||
"silero_vad.onnx".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the expected path of the TEN VAD ONNX model copied by Flutter.
|
||||
pub fn ten_model_bundle_path() -> String {
|
||||
if let Ok(guard) = ten_model_path_override().read() {
|
||||
if let Some(path) = guard.as_ref() {
|
||||
return path.clone();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "ios", target_os = "macos"))]
|
||||
{
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
let docs = format!("{home}/Documents/ten_vad.onnx");
|
||||
if std::path::Path::new(&docs).exists() {
|
||||
return docs;
|
||||
}
|
||||
let bundle = format!("{home}/../Library/ten_vad.onnx");
|
||||
if std::path::Path::new(&bundle).exists() {
|
||||
return bundle;
|
||||
}
|
||||
}
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
|
||||
{
|
||||
if let Ok(cwd) = std::env::current_dir() {
|
||||
let local = cwd.join("ten_vad.onnx");
|
||||
if local.exists() {
|
||||
return local.to_string_lossy().to_string();
|
||||
}
|
||||
}
|
||||
"ten_vad.onnx".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user