edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize voice activity and audio routing

Browse Source
This commit is contained in:
Edison Jwa
2026-05-25 01:19:09 +09:00
parent eb9014cd81
commit 5515ff6643
34 changed files with 3054 additions and 1751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_bridge/src/api.rs
+78 -30
View File
@@ -145,7 +145,7 @@ pub(crate) fn publish_permission_state(permission: String, state: PermissionStat
/// flood the diagnostic export during transient packet loss;
/// users can still raise verbosity via `RUST_LOG=info`.
const DEFAULT_LOG_FILTER: &str =
"info,tsproto::resend=error,tsproto::packet_codec=error,tsclientlib=error";
"info,tsproto::resend=error,tsproto::packet_codec=error,tsclientlib=error,ts_bookkeeping::messages::s2c=error";
/// Initialise the bridge. Must be called once on Dart side before
/// any other API call. Sets up panic logging.
@@ -921,8 +921,6 @@ pub enum BridgeAudioBackend {
pub enum BridgeVadBackend {
/// Silero ONNX VAD.
SileroOnnx,
/// TEN VAD.
TenVad,
/// WebRTC fallback VAD.
WebrtcVad,
/// Debug energy VAD.
@@ -1014,6 +1012,16 @@ pub struct BridgeAudioProcessingStats {
pub callback_xruns: u64,
/// Clipped samples.
pub clipped_samples: u64,
/// Number of effectively silent processed capture frames.
pub zero_frames: u64,
/// Number of processed capture frames.
pub capture_frames: u64,
/// Input callbacks carrying 10 ms of audio.
pub callbacks_10ms: u64,
/// Input callbacks carrying 20 ms of audio.
pub callbacks_20ms: u64,
/// Input callbacks carrying other sizes.
pub callbacks_other: u64,
/// Sonora enabled.
pub sonora_enabled: bool,
/// Platform voice processing enabled.
@@ -1092,7 +1100,6 @@ impl From<BridgeVadBackend> for chanora_core::VadBackend {
fn from(backend: BridgeVadBackend) -> Self {
match backend {
BridgeVadBackend::SileroOnnx => Self::SileroOnnx,
BridgeVadBackend::TenVad => Self::TenVad,
BridgeVadBackend::WebrtcVad => Self::WebrtcVad,
BridgeVadBackend::EnergyDebug => Self::EnergyDebug,
BridgeVadBackend::Disabled => Self::Disabled,
@@ -1104,7 +1111,6 @@ impl From<chanora_core::VadBackend> for BridgeVadBackend {
fn from(backend: chanora_core::VadBackend) -> Self {
match backend {
chanora_core::VadBackend::SileroOnnx => Self::SileroOnnx,
chanora_core::VadBackend::TenVad => Self::TenVad,
chanora_core::VadBackend::WebrtcVad => Self::WebrtcVad,
chanora_core::VadBackend::EnergyDebug => Self::EnergyDebug,
chanora_core::VadBackend::Disabled => Self::Disabled,
@@ -1196,6 +1202,11 @@ impl From<chanora_core::AudioProcessingStats> for BridgeAudioProcessingStats {
output_underruns: stats.output_underruns,
callback_xruns: stats.callback_xruns,
clipped_samples: stats.clipped_samples,
zero_frames: stats.zero_frames,
capture_frames: stats.capture_frames,
callbacks_10ms: stats.callbacks_10ms,
callbacks_20ms: stats.callbacks_20ms,
callbacks_other: stats.callbacks_other,
sonora_enabled: stats.sonora_enabled,
platform_voice_processing_enabled: stats.platform_voice_processing_enabled,
}
@@ -1236,8 +1247,31 @@ pub fn export_diagnostics() -> String {
let android_audio_yaml =
chanora_audio::mobile_voice_backend::current_android_audio_diagnostics()
.map(|d| d.to_yaml_fragment());
let audio_health = runtime().block_on(async { session().audio_processing_stats().await.ok() });
let network_info = runtime().block_on(async { session().network_diagnostics_summary().await });
let protocol_events = runtime().block_on(async { session().drain_protocol_events().await });
let android_audio_yaml = android_audio_yaml.map(|mut yaml| {
if let Some(stats) = audio_health {
yaml.push_str(&format!(
" health:\n\
\x20\x20\x20\x20capture_frames: {}\n\
\x20\x20\x20\x20zero_frames: {}\n\
\x20\x20\x20\x20callbacks_10ms: {}\n\
\x20\x20\x20\x20callbacks_20ms: {}\n\
\x20\x20\x20\x20callbacks_other: {}\n\
\x20\x20\x20\x20callback_xruns: {}\n\
\x20\x20\x20\x20clipped_samples: {}\n",
stats.capture_frames,
stats.zero_frames,
stats.callbacks_10ms,
stats.callbacks_20ms,
stats.callbacks_other,
stats.callback_xruns,
stats.clipped_samples,
));
}
yaml
});
match chanora_core::DiagnosticExport::from_sink(log_sink(), metadata) {
Ok(exp) => exp
.with_android_audio(android_audio_yaml)
@@ -1506,6 +1540,11 @@ pub enum BridgeEvent {
/// Target scope (server/channel/private/poke).
target: BridgeMessageTarget,
},
/// Human-readable server activity surfaced from protocol bookkeeping events.
ServerActivity {
/// TeamSpeak-style activity line.
message: String,
},
/// Audio route changed (speaker/earpiece/BT/wired).
AudioRouteChanged {
/// The new audio route.
@@ -1692,6 +1731,9 @@ impl From<chanora_core::SessionEvent> for BridgeEvent {
message,
target: target.into(),
},
chanora_core::SessionEvent::ServerActivity { message } => {
BridgeEvent::ServerActivity { message }
}
chanora_core::SessionEvent::AudioRouteChanged { route } => {
BridgeEvent::AudioRouteChanged {
route: route.into(),
@@ -1848,10 +1890,16 @@ pub async fn audio_processing_stats() -> Result<BridgeAudioProcessingStats, Brid
/// Audio device info from the platform.
#[derive(Debug, Clone)]
pub struct BridgeAudioDevice {
/// Stable platform-reported device identifier.
pub id: String,
/// Human-readable device name.
pub name: String,
/// Additional device details useful for disambiguation.
pub details: String,
/// True if the OS reports this as the default device.
pub is_default: bool,
/// True if Chanora currently has this device pinned.
pub is_selected: bool,
}
/// List of available audio devices.
@@ -1864,14 +1912,26 @@ pub struct BridgeAudioDeviceList {
}
/// List available audio input and output devices from the platform.
pub fn list_audio_devices() -> BridgeAudioDeviceList {
let list = chanora_audio::list_audio_devices();
BridgeAudioDeviceList {
pub async fn list_audio_devices() -> Result<BridgeAudioDeviceList, BridgeError> {
let (list, selected_input, selected_output) = runtime()
.spawn(async move {
let selected_input = session().preferred_input_device().await;
let selected_output = session().preferred_output_device().await;
let list = chanora_audio::list_audio_devices();
(list, selected_input, selected_output)
})
.await
.map_err(|e| task_join_error("list_audio_devices", e))?;
Ok(BridgeAudioDeviceList {
input_devices: list
.input_devices
.into_iter()
.map(|d| BridgeAudioDevice {
is_selected: selected_input.as_ref().is_some_and(|id| id == &d.id),
id: d.id,
name: d.name,
details: d.details,
is_default: d.is_default,
})
.collect(),
@@ -1879,27 +1939,30 @@ pub fn list_audio_devices() -> BridgeAudioDeviceList {
.output_devices
.into_iter()
.map(|d| BridgeAudioDevice {
is_selected: selected_output.as_ref().is_some_and(|id| id == &d.id),
id: d.id,
name: d.name,
details: d.details,
is_default: d.is_default,
})
.collect(),
}
})
}
/// Set the preferred input device by name. Takes effect on next
/// Set the preferred input device by id. Takes effect on next
/// `start_audio`.
pub async fn set_input_device(name: Option<String>) -> Result<(), BridgeError> {
pub async fn set_input_device(id: Option<String>) -> Result<(), BridgeError> {
runtime()
.spawn(async move { session().set_input_device(name).await })
.spawn(async move { session().set_input_device(id).await })
.await
.map_err(|e| task_join_error("set_input_device", e))??;
Ok(())
}
/// Set the preferred output device by name.
pub async fn set_output_device(name: Option<String>) -> Result<(), BridgeError> {
/// Set the preferred output device by id.
pub async fn set_output_device(id: Option<String>) -> Result<(), BridgeError> {
runtime()
.spawn(async move { session().set_output_device(name).await })
.spawn(async move { session().set_output_device(id).await })
.await
.map_err(|e| task_join_error("set_output_device", e))??;
Ok(())
@@ -1919,21 +1982,6 @@ pub async fn set_vad_model_path(path: String) -> Result<(), BridgeError> {
Ok(())
}
/// Configure the TEN VAD ONNX model path.
pub async fn set_ten_vad_model_path(path: String) -> Result<(), BridgeError> {
if path.trim().is_empty() {
return Err(BridgeError::InvalidCommand(
"ten vad model path must not be empty".to_string(),
));
}
runtime()
.spawn(async move { chanora_audio::vad::set_ten_model_path(&path) })
.await
.map_err(|e| task_join_error("set_ten_vad_model_path", e))?
.map_err(|e| BridgeError::Unmapped(format!("set_ten_vad_model_path: {e}")))?;
Ok(())
}
/// Enable or disable audio debug WAV dumping.
pub async fn enable_audio_debug_wav_dump(enabled: bool) -> Result<(), BridgeError> {
runtime()
crates/chanora_bridge/src/frb_generated.rs
+76 -68
View File
@@ -38,7 +38,7 @@ flutter_rust_bridge::frb_generated_boilerplate!(
default_rust_auto_opaque = RustAutoOpaqueMoi,
);
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.12.0";
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = 1433826599;
pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -560177922;
// Section: executor
@@ -743,7 +743,7 @@ fn wire__crate__api__list_audio_devices_impl(
rust_vec_len_: i32,
data_len_: i32,
) {
FLUTTER_RUST_BRIDGE_HANDLER.wrap_normal::<flutter_rust_bridge::for_generated::SseCodec, _, _>(
FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
flutter_rust_bridge::for_generated::TaskInfo {
debug_name: "list_audio_devices",
port: Some(port_),
@@ -760,11 +760,14 @@ fn wire__crate__api__list_audio_devices_impl(
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
deserializer.end();
move |context| {
transform_result_sse::<_, ()>((move || {
let output_ok = Result::<_, ()>::Ok(crate::api::list_audio_devices())?;
Ok(output_ok)
})())
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::list_audio_devices().await?;
Ok(output_ok)
})()
.await,
)
}
},
)
@@ -1141,12 +1144,12 @@ fn wire__crate__api__set_input_device_impl(
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
let api_name = <Option<String>>::sse_decode(&mut deserializer);
let api_id = <Option<String>>::sse_decode(&mut deserializer);
deserializer.end();
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::set_input_device(api_name).await?;
let output_ok = crate::api::set_input_device(api_id).await?;
Ok(output_ok)
})()
.await,
@@ -1282,12 +1285,12 @@ fn wire__crate__api__set_output_device_impl(
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
let api_name = <Option<String>>::sse_decode(&mut deserializer);
let api_id = <Option<String>>::sse_decode(&mut deserializer);
deserializer.end();
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::set_output_device(api_name).await?;
let output_ok = crate::api::set_output_device(api_id).await?;
Ok(output_ok)
})()
.await,
@@ -1478,42 +1481,6 @@ fn wire__crate__api__set_release_tail_ms_impl(
},
)
}
fn wire__crate__api__set_ten_vad_model_path_impl(
port_: flutter_rust_bridge::for_generated::MessagePort,
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
rust_vec_len_: i32,
data_len_: i32,
) {
FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::<flutter_rust_bridge::for_generated::SseCodec, _, _, _>(
flutter_rust_bridge::for_generated::TaskInfo {
debug_name: "set_ten_vad_model_path",
port: Some(port_),
mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal,
},
move || {
let message = unsafe {
flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(
ptr_,
rust_vec_len_,
data_len_,
)
};
let mut deserializer =
flutter_rust_bridge::for_generated::SseDeserializer::new(message);
let api_path = <String>::sse_decode(&mut deserializer);
deserializer.end();
move |context| async move {
transform_result_sse::<_, crate::BridgeError>(
(move || async move {
let output_ok = crate::api::set_ten_vad_model_path(api_path).await?;
Ok(output_ok)
})()
.await,
)
}
},
)
}
fn wire__crate__api__set_transmit_mode_impl(
port_: flutter_rust_bridge::for_generated::MessagePort,
ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr,
@@ -1783,11 +1750,17 @@ impl SseDecode for crate::api::BridgeAudioBackend {
impl SseDecode for crate::api::BridgeAudioDevice {
// Codec=Sse (Serialization based), see doc to use other codecs
fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self {
let mut var_id = <String>::sse_decode(deserializer);
let mut var_name = <String>::sse_decode(deserializer);
let mut var_details = <String>::sse_decode(deserializer);
let mut var_isDefault = <bool>::sse_decode(deserializer);
let mut var_isSelected = <bool>::sse_decode(deserializer);
return crate::api::BridgeAudioDevice {
id: var_id,
name: var_name,
details: var_details,
is_default: var_isDefault,
is_selected: var_isSelected,
};
}
}
@@ -1859,6 +1832,11 @@ impl SseDecode for crate::api::BridgeAudioProcessingStats {
let mut var_outputUnderruns = <u64>::sse_decode(deserializer);
let mut var_callbackXruns = <u64>::sse_decode(deserializer);
let mut var_clippedSamples = <u64>::sse_decode(deserializer);
let mut var_zeroFrames = <u64>::sse_decode(deserializer);
let mut var_captureFrames = <u64>::sse_decode(deserializer);
let mut var_callbacks10Ms = <u64>::sse_decode(deserializer);
let mut var_callbacks20Ms = <u64>::sse_decode(deserializer);
let mut var_callbacksOther = <u64>::sse_decode(deserializer);
let mut var_sonoraEnabled = <bool>::sse_decode(deserializer);
let mut var_platformVoiceProcessingEnabled = <bool>::sse_decode(deserializer);
return crate::api::BridgeAudioProcessingStats {
@@ -1879,6 +1857,11 @@ impl SseDecode for crate::api::BridgeAudioProcessingStats {
output_underruns: var_outputUnderruns,
callback_xruns: var_callbackXruns,
clipped_samples: var_clippedSamples,
zero_frames: var_zeroFrames,
capture_frames: var_captureFrames,
callbacks_10ms: var_callbacks10Ms,
callbacks_20ms: var_callbacks20Ms,
callbacks_other: var_callbacksOther,
sonora_enabled: var_sonoraEnabled,
platform_voice_processing_enabled: var_platformVoiceProcessingEnabled,
};
@@ -2147,6 +2130,12 @@ impl SseDecode for crate::api::BridgeEvent {
};
}
12 => {
let mut var_message = <String>::sse_decode(deserializer);
return crate::api::BridgeEvent::ServerActivity {
message: var_message,
};
}
13 => {
let mut var_route = <crate::api::BridgeAudioRoute>::sse_decode(deserializer);
return crate::api::BridgeEvent::AudioRouteChanged { route: var_route };
}
@@ -2291,10 +2280,9 @@ impl SseDecode for crate::api::BridgeVadBackend {
let mut inner = <i32>::sse_decode(deserializer);
return match inner {
0 => crate::api::BridgeVadBackend::SileroOnnx,
1 => crate::api::BridgeVadBackend::TenVad,
2 => crate::api::BridgeVadBackend::WebrtcVad,
3 => crate::api::BridgeVadBackend::EnergyDebug,
4 => crate::api::BridgeVadBackend::Disabled,
1 => crate::api::BridgeVadBackend::WebrtcVad,
2 => crate::api::BridgeVadBackend::EnergyDebug,
3 => crate::api::BridgeVadBackend::Disabled,
_ => unreachable!("Invalid variant for BridgeVadBackend: {}", inner),
};
}
@@ -2544,13 +2532,12 @@ fn pde_ffi_dispatcher_primary_impl(
39 => wire__crate__api__set_ptt_impl(port, ptr, rust_vec_len, data_len),
40 => wire__crate__api__set_ptt_binding_impl(port, ptr, rust_vec_len, data_len),
41 => wire__crate__api__set_release_tail_ms_impl(port, ptr, rust_vec_len, data_len),
42 => wire__crate__api__set_ten_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
43 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
44 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
45 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len),
46 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len),
47 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len),
48 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len),
42 => wire__crate__api__set_transmit_mode_impl(port, ptr, rust_vec_len, data_len),
43 => wire__crate__api__set_vad_model_path_impl(port, ptr, rust_vec_len, data_len),
44 => wire__crate__api__snapshot_impl(port, ptr, rust_vec_len, data_len),
45 => wire__crate__api__update_bookmark_impl(port, ptr, rust_vec_len, data_len),
46 => wire__crate__api__voice_join_impl(port, ptr, rust_vec_len, data_len),
47 => wire__crate__api__voice_leave_impl(port, ptr, rust_vec_len, data_len),
_ => unreachable!(),
}
}
@@ -2609,8 +2596,11 @@ impl flutter_rust_bridge::IntoIntoDart<crate::api::BridgeAudioBackend>
impl flutter_rust_bridge::IntoDart for crate::api::BridgeAudioDevice {
fn into_dart(self) -> flutter_rust_bridge::for_generated::DartAbi {
[
self.id.into_into_dart().into_dart(),
self.name.into_into_dart().into_dart(),
self.details.into_into_dart().into_dart(),
self.is_default.into_into_dart().into_dart(),
self.is_selected.into_into_dart().into_dart(),
]
.into_dart()
}
@@ -2697,6 +2687,11 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeAudioProcessingStats {
self.output_underruns.into_into_dart().into_dart(),
self.callback_xruns.into_into_dart().into_dart(),
self.clipped_samples.into_into_dart().into_dart(),
self.zero_frames.into_into_dart().into_dart(),
self.capture_frames.into_into_dart().into_dart(),
self.callbacks_10ms.into_into_dart().into_dart(),
self.callbacks_20ms.into_into_dart().into_dart(),
self.callbacks_other.into_into_dart().into_dart(),
self.sonora_enabled.into_into_dart().into_dart(),
self.platform_voice_processing_enabled
.into_into_dart()
@@ -2973,8 +2968,11 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeEvent {
target.into_into_dart().into_dart(),
]
.into_dart(),
crate::api::BridgeEvent::ServerActivity { message } => {
[12.into_dart(), message.into_into_dart().into_dart()].into_dart()
}
crate::api::BridgeEvent::AudioRouteChanged { route } => {
[12.into_dart(), route.into_into_dart().into_dart()].into_dart()
[13.into_dart(), route.into_into_dart().into_dart()].into_dart()
}
_ => {
unimplemented!("");
@@ -3170,10 +3168,9 @@ impl flutter_rust_bridge::IntoDart for crate::api::BridgeVadBackend {
fn into_dart(self) -> flutter_rust_bridge::for_generated::DartAbi {
match self {
Self::SileroOnnx => 0.into_dart(),
Self::TenVad => 1.into_dart(),
Self::WebrtcVad => 2.into_dart(),
Self::EnergyDebug => 3.into_dart(),
Self::Disabled => 4.into_dart(),
Self::WebrtcVad => 1.into_dart(),
Self::EnergyDebug => 2.into_dart(),
Self::Disabled => 3.into_dart(),
_ => unreachable!(),
}
}
@@ -3313,8 +3310,11 @@ impl SseEncode for crate::api::BridgeAudioBackend {
impl SseEncode for crate::api::BridgeAudioDevice {
// Codec=Sse (Serialization based), see doc to use other codecs
fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) {
<String>::sse_encode(self.id, serializer);
<String>::sse_encode(self.name, serializer);
<String>::sse_encode(self.details, serializer);
<bool>::sse_encode(self.is_default, serializer);
<bool>::sse_encode(self.is_selected, serializer);
}
}
@@ -3368,6 +3368,11 @@ impl SseEncode for crate::api::BridgeAudioProcessingStats {
<u64>::sse_encode(self.output_underruns, serializer);
<u64>::sse_encode(self.callback_xruns, serializer);
<u64>::sse_encode(self.clipped_samples, serializer);
<u64>::sse_encode(self.zero_frames, serializer);
<u64>::sse_encode(self.capture_frames, serializer);
<u64>::sse_encode(self.callbacks_10ms, serializer);
<u64>::sse_encode(self.callbacks_20ms, serializer);
<u64>::sse_encode(self.callbacks_other, serializer);
<bool>::sse_encode(self.sonora_enabled, serializer);
<bool>::sse_encode(self.platform_voice_processing_enabled, serializer);
}
@@ -3595,8 +3600,12 @@ impl SseEncode for crate::api::BridgeEvent {
<String>::sse_encode(message, serializer);
<crate::api::BridgeMessageTarget>::sse_encode(target, serializer);
}
crate::api::BridgeEvent::AudioRouteChanged { route } => {
crate::api::BridgeEvent::ServerActivity { message } => {
<i32>::sse_encode(12, serializer);
<String>::sse_encode(message, serializer);
}
crate::api::BridgeEvent::AudioRouteChanged { route } => {
<i32>::sse_encode(13, serializer);
<crate::api::BridgeAudioRoute>::sse_encode(route, serializer);
}
_ => {
@@ -3734,10 +3743,9 @@ impl SseEncode for crate::api::BridgeVadBackend {
<i32>::sse_encode(
match self {
crate::api::BridgeVadBackend::SileroOnnx => 0,
crate::api::BridgeVadBackend::TenVad => 1,
crate::api::BridgeVadBackend::WebrtcVad => 2,
crate::api::BridgeVadBackend::EnergyDebug => 3,
crate::api::BridgeVadBackend::Disabled => 4,
crate::api::BridgeVadBackend::WebrtcVad => 1,
crate::api::BridgeVadBackend::EnergyDebug => 2,
crate::api::BridgeVadBackend::Disabled => 3,
_ => {
unimplemented!("");
}