edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Android Oboe voice backend — WebRTC APM, VAD, HW/SW toggle, BBCode welcome, link trust, foreground task

Browse Source 
Audio engine (Rust):
- Android Oboe: WebRTC APM (AEC/NS/AGC/HPF) + TEN/Silero ONNX VAD
- Hardware effects (JNI) with software fallback per-effect
- Render reference buffer for AEC between output/capture callbacks
- Voice activity gate: suppress transmission when speaker muted (all platforms)
- Audio focus (SDD-109) + Bluetooth SCO (SDD-110) via JNI
- ONNX Runtime 1.26 via ort 2.0.0-rc.12 (down from rc.10, ndarray 0.17)
- VAD worker channel capacity 8→32, initial seq u64::MAX (warm-up fix)
- TEN VAD default backend (was Silero)
- Platform→WebrtcApm resolution after hardware binding
- oboe-rs edisonjwa fork with get_raw_session_id()

Android Kotlin:
- AndroidAudioFocusController + AndroidBluetoothScoController
- AndroidAudioLifecycleController (route changes to Flutter)
- ProGuard rules for new controllers

Flutter UI:
- VoiceSettings: Android HW/SW toggle (Platform auto / WebRTC APM)
- VoiceStatusChip: mute warning border + Speaker muted label
- BBCode welcome message parser (BbCodeText, case-insensitive)
- Welcome message foldable (expanded by default)
- Link trust dialog (domain wildcards, SharedPreferences)
- HapticFeedback on voice sheet opener
- Server name in AppBar, version v0.1.0
- Default channel (id=1) visible, serverquery clients hidden
- flutter_foreground_task integration

Config:
- ort load-dynamic on all non-iOS (Android/Linux/Windows)
- ONNX Runtime AAR 1.26.0
- ndarray moved to common deps (was Apple-only)
This commit is contained in:
Edison Jwa
2026-05-22 09:29:57 +09:00
parent 6af4ecab0f
commit bf284018e6
37 changed files with 3676 additions and 703 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/ios_voice_unit.rs
+59 -24
View File
@@ -139,6 +139,7 @@ struct IosCaptureState {
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
@@ -146,10 +147,12 @@ struct IosCaptureState {
/// Background Silero worker — enqueues frames off the realtime
/// callback and publishes the latest probability atomically.
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
ten_vad: Option<crate::vad::TenOnnxVadWorker>,
/// Last VAD backend we configured — used to detect backend changes.
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
fallback_warned_backend: Option<crate::VadBackend>,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
sonora_processor: crate::processor::SonoraProcessor,
@@ -173,6 +176,7 @@ impl IosCaptureState {
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
@@ -188,13 +192,16 @@ impl IosCaptureState {
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
output_muted,
frames_sent,
mic_gain,
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
sonora_processor: crate::processor::SonoraProcessor::new(),
@@ -210,12 +217,16 @@ impl IosCaptureState {
})
}
fn disable_failed_vad_backend(&mut self, failed_backend: crate::VadBackend) {
if let Ok(mut cfg) = self.audio_processing_config.try_lock() {
if cfg.disable_failed_vad_backend(failed_backend) {
self.current_vad_backend = crate::VadBackend::WebrtcVad;
}
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend == Some(failed_backend) {
return;
}
self.fallback_warned_backend = Some(failed_backend);
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
/// Consume the i16 mono buffer delivered by VPIO, accumulate
@@ -367,6 +378,7 @@ impl IosCaptureState {
if vad_backend != self.current_vad_backend || silero_model_changed {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
self.silero_model_epoch = silero_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
@@ -383,22 +395,28 @@ impl IosCaptureState {
target: "chanora_audio",
"Silero VAD model not found at {model_path}; falling back to WebRTC VAD"
);
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
}
self.audio_processing_stats
.set_vad_fallback_active(self.silero_vad_worker.is_none());
}
crate::VadBackend::TenVad => {
self.silero_vad_worker = None;
warn!(
target: "chanora_audio",
"TEN VAD selected but native TEN runtime is not bundled; falling back to WebRTC VAD"
);
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
self.audio_processing_stats.set_vad_fallback_active(true);
let model_path = crate::vad::ten_model_bundle_path();
self.ten_vad = crate::vad::TenOnnxVadWorker::try_new(&model_path);
if self.ten_vad.is_none() {
warn!(
target: "chanora_audio",
"TEN VAD ONNX model not available at {model_path}; falling back to WebRTC VAD"
);
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
}
self.audio_processing_stats
.set_vad_fallback_active(self.ten_vad.is_none());
}
_ => {
self.silero_vad_worker = None;
self.ten_vad = None;
self.audio_processing_stats.set_vad_fallback_active(false);
}
}
@@ -419,14 +437,14 @@ impl IosCaptureState {
);
let transmit_active = self.transmit_active.load(Ordering::Relaxed);
// Apply the enabled stages through the SonoraProcessor.
// We reconfigure it on-the-fly to match the current settings.
// VPIO owns AEC. Keep the legacy non-AEC conditioning path here until
// the raw WebRTC APM path is explicitly selected.
if run_ns || run_agc || run_hpf {
use crate::processor::sonora::SonoraConfig;
use crate::processor::AudioProcessor;
let new_cfg = SonoraConfig {
hpf: run_hpf,
aec3: false, // NEVER in VPIO path (INV_009)
aec3: false,
ns: run_ns,
agc2: run_agc,
};
@@ -448,7 +466,8 @@ impl IosCaptureState {
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
if worker.try_send(capture_seq, &frame) && !worker.is_stale(capture_seq) {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
@@ -456,33 +475,48 @@ impl IosCaptureState {
}
} else {
used_fallback_vad = true;
self.silero_vad_worker = None;
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.disable_failed_vad_backend(crate::VadBackend::SileroOnnx);
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
used_fallback_vad = true;
self.disable_failed_vad_backend(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
if let Some(worker) = self.ten_vad.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
};
self.audio_processing_stats
.set_vad_fallback_active(used_fallback_vad);
let gate_open = self.vad_state.update(vad.speech);
let output_muted = self.output_muted.load(Ordering::Relaxed);
if let Some(selector) = &self.voice_activity_selector {
selector.set_voice_activity_open(gate_open);
selector.set_voice_activity_open(gate_open && !output_muted);
}
self.audio_processing_stats.update_capture(
input_dbfs,
crate::frame::dbfs(&frame),
vad.probability,
gate_open,
gate_open && !output_muted,
transmit_active,
);
@@ -700,6 +734,7 @@ impl IosVoiceUnit {
let mut capture_state = IosCaptureState::new(
voice_out_tx,
transmit_active,
output_muted.clone(),
frames_sent,
mic_gain,
voice_activity_selector,