edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize voice activity and audio routing

Browse Source
This commit is contained in:
Edison Jwa
2026-05-25 01:19:09 +09:00
parent eb9014cd81
commit 5515ff6643
34 changed files with 3054 additions and 1751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_bridge/src/api.rs
+78 -30
View File
@@ -145,7 +145,7 @@ pub(crate) fn publish_permission_state(permission: String, state: PermissionStat
/// flood the diagnostic export during transient packet loss;
/// users can still raise verbosity via `RUST_LOG=info`.
const DEFAULT_LOG_FILTER: &str =
"info,tsproto::resend=error,tsproto::packet_codec=error,tsclientlib=error";
"info,tsproto::resend=error,tsproto::packet_codec=error,tsclientlib=error,ts_bookkeeping::messages::s2c=error";
/// Initialise the bridge. Must be called once on Dart side before
/// any other API call. Sets up panic logging.
@@ -921,8 +921,6 @@ pub enum BridgeAudioBackend {
pub enum BridgeVadBackend {
/// Silero ONNX VAD.
SileroOnnx,
/// TEN VAD.
TenVad,
/// WebRTC fallback VAD.
WebrtcVad,
/// Debug energy VAD.
@@ -1014,6 +1012,16 @@ pub struct BridgeAudioProcessingStats {
pub callback_xruns: u64,
/// Clipped samples.
pub clipped_samples: u64,
/// Number of effectively silent processed capture frames.
pub zero_frames: u64,
/// Number of processed capture frames.
pub capture_frames: u64,
/// Input callbacks carrying 10 ms of audio.
pub callbacks_10ms: u64,
/// Input callbacks carrying 20 ms of audio.
pub callbacks_20ms: u64,
/// Input callbacks carrying other sizes.
pub callbacks_other: u64,
/// Sonora enabled.
pub sonora_enabled: bool,
/// Platform voice processing enabled.
@@ -1092,7 +1100,6 @@ impl From<BridgeVadBackend> for chanora_core::VadBackend {
fn from(backend: BridgeVadBackend) -> Self {
match backend {
BridgeVadBackend::SileroOnnx => Self::SileroOnnx,
BridgeVadBackend::TenVad => Self::TenVad,
BridgeVadBackend::WebrtcVad => Self::WebrtcVad,
BridgeVadBackend::EnergyDebug => Self::EnergyDebug,
BridgeVadBackend::Disabled => Self::Disabled,
@@ -1104,7 +1111,6 @@ impl From<chanora_core::VadBackend> for BridgeVadBackend {
fn from(backend: chanora_core::VadBackend) -> Self {
match backend {
chanora_core::VadBackend::SileroOnnx => Self::SileroOnnx,
chanora_core::VadBackend::TenVad => Self::TenVad,
chanora_core::VadBackend::WebrtcVad => Self::WebrtcVad,
chanora_core::VadBackend::EnergyDebug => Self::EnergyDebug,
chanora_core::VadBackend::Disabled => Self::Disabled,
@@ -1196,6 +1202,11 @@ impl From<chanora_core::AudioProcessingStats> for BridgeAudioProcessingStats {
output_underruns: stats.output_underruns,
callback_xruns: stats.callback_xruns,
clipped_samples: stats.clipped_samples,
zero_frames: stats.zero_frames,
capture_frames: stats.capture_frames,
callbacks_10ms: stats.callbacks_10ms,
callbacks_20ms: stats.callbacks_20ms,
callbacks_other: stats.callbacks_other,
sonora_enabled: stats.sonora_enabled,
platform_voice_processing_enabled: stats.platform_voice_processing_enabled,
}
@@ -1236,8 +1247,31 @@ pub fn export_diagnostics() -> String {
let android_audio_yaml =
chanora_audio::mobile_voice_backend::current_android_audio_diagnostics()
.map(|d| d.to_yaml_fragment());
let audio_health = runtime().block_on(async { session().audio_processing_stats().await.ok() });
let network_info = runtime().block_on(async { session().network_diagnostics_summary().await });
let protocol_events = runtime().block_on(async { session().drain_protocol_events().await });
let android_audio_yaml = android_audio_yaml.map(|mut yaml| {
if let Some(stats) = audio_health {
yaml.push_str(&format!(
" health:\n\
\x20\x20\x20\x20capture_frames: {}\n\
\x20\x20\x20\x20zero_frames: {}\n\
\x20\x20\x20\x20callbacks_10ms: {}\n\
\x20\x20\x20\x20callbacks_20ms: {}\n\
\x20\x20\x20\x20callbacks_other: {}\n\
\x20\x20\x20\x20callback_xruns: {}\n\
\x20\x20\x20\x20clipped_samples: {}\n",
stats.capture_frames,
stats.zero_frames,
stats.callbacks_10ms,
stats.callbacks_20ms,
stats.callbacks_other,
stats.callback_xruns,
stats.clipped_samples,
));
}
yaml
});
match chanora_core::DiagnosticExport::from_sink(log_sink(), metadata) {
Ok(exp) => exp
.with_android_audio(android_audio_yaml)
@@ -1506,6 +1540,11 @@ pub enum BridgeEvent {
/// Target scope (server/channel/private/poke).
target: BridgeMessageTarget,
},
/// Human-readable server activity surfaced from protocol bookkeeping events.
ServerActivity {
/// TeamSpeak-style activity line.
message: String,
},
/// Audio route changed (speaker/earpiece/BT/wired).
AudioRouteChanged {
/// The new audio route.
@@ -1692,6 +1731,9 @@ impl From<chanora_core::SessionEvent> for BridgeEvent {
message,
target: target.into(),
},
chanora_core::SessionEvent::ServerActivity { message } => {
BridgeEvent::ServerActivity { message }
}
chanora_core::SessionEvent::AudioRouteChanged { route } => {
BridgeEvent::AudioRouteChanged {
route: route.into(),
@@ -1848,10 +1890,16 @@ pub async fn audio_processing_stats() -> Result<BridgeAudioProcessingStats, Brid
/// Audio device info from the platform.
#[derive(Debug, Clone)]
pub struct BridgeAudioDevice {
/// Stable platform-reported device identifier.
pub id: String,
/// Human-readable device name.
pub name: String,
/// Additional device details useful for disambiguation.
pub details: String,
/// True if the OS reports this as the default device.
pub is_default: bool,
/// True if Chanora currently has this device pinned.
pub is_selected: bool,
}
/// List of available audio devices.
@@ -1864,14 +1912,26 @@ pub struct BridgeAudioDeviceList {
}
/// List available audio input and output devices from the platform.
pub fn list_audio_devices() -> BridgeAudioDeviceList {
let list = chanora_audio::list_audio_devices();
BridgeAudioDeviceList {
pub async fn list_audio_devices() -> Result<BridgeAudioDeviceList, BridgeError> {
let (list, selected_input, selected_output) = runtime()
.spawn(async move {
let selected_input = session().preferred_input_device().await;
let selected_output = session().preferred_output_device().await;
let list = chanora_audio::list_audio_devices();
(list, selected_input, selected_output)
})
.await
.map_err(|e| task_join_error("list_audio_devices", e))?;
Ok(BridgeAudioDeviceList {
input_devices: list
.input_devices
.into_iter()
.map(|d| BridgeAudioDevice {
is_selected: selected_input.as_ref().is_some_and(|id| id == &d.id),
id: d.id,
name: d.name,
details: d.details,
is_default: d.is_default,
})
.collect(),
@@ -1879,27 +1939,30 @@ pub fn list_audio_devices() -> BridgeAudioDeviceList {
.output_devices
.into_iter()
.map(|d| BridgeAudioDevice {
is_selected: selected_output.as_ref().is_some_and(|id| id == &d.id),
id: d.id,
name: d.name,
details: d.details,
is_default: d.is_default,
})
.collect(),
}
})
}
/// Set the preferred input device by name. Takes effect on next
/// Set the preferred input device by id. Takes effect on next
/// `start_audio`.
pub async fn set_input_device(name: Option<String>) -> Result<(), BridgeError> {
pub async fn set_input_device(id: Option<String>) -> Result<(), BridgeError> {
runtime()
.spawn(async move { session().set_input_device(name).await })
.spawn(async move { session().set_input_device(id).await })
.await
.map_err(|e| task_join_error("set_input_device", e))??;
Ok(())
}
/// Set the preferred output device by name.
pub async fn set_output_device(name: Option<String>) -> Result<(), BridgeError> {
/// Set the preferred output device by id.
pub async fn set_output_device(id: Option<String>) -> Result<(), BridgeError> {
runtime()
.spawn(async move { session().set_output_device(name).await })
.spawn(async move { session().set_output_device(id).await })
.await
.map_err(|e| task_join_error("set_output_device", e))??;
Ok(())
@@ -1919,21 +1982,6 @@ pub async fn set_vad_model_path(path: String) -> Result<(), BridgeError> {
Ok(())
}
/// Configure the TEN VAD ONNX model path.
pub async fn set_ten_vad_model_path(path: String) -> Result<(), BridgeError> {
if path.trim().is_empty() {
return Err(BridgeError::InvalidCommand(
"ten vad model path must not be empty".to_string(),
));
}
runtime()
.spawn(async move { chanora_audio::vad::set_ten_model_path(&path) })
.await
.map_err(|e| task_join_error("set_ten_vad_model_path", e))?
.map_err(|e| BridgeError::Unmapped(format!("set_ten_vad_model_path: {e}")))?;
Ok(())
}
/// Enable or disable audio debug WAV dumping.
pub async fn enable_audio_debug_wav_dump(enabled: bool) -> Result<(), BridgeError> {
runtime()