edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): add iOS VAD runtime support

Browse Source
This commit is contained in:
Edison Jwa
2026-05-21 20:51:45 +09:00
parent 171baf6e41
commit 6af4ecab0f
73 changed files with 11529 additions and 1249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/chanora_core/src/lib.rs
+102 -32
View File
@@ -54,8 +54,9 @@ use chanora_state::channel_join::{
pub mod ptt;
pub use chanora_audio::{
AudioEngine, AudioEngineConfig, AudioTransmitGate, PttBackendDescriptor, PttCapabilityLevel,
ReleaseTailTimer, TransmitMode, TransmitModeSelector,
AudioBackend, AudioEngine, AudioEngineConfig, AudioProcessingConfig, AudioProcessingStats,
AudioRoute, AudioTransmitGate, EffectOwner, IosVoiceProcessingMode, PttBackendDescriptor,
PttCapabilityLevel, ReleaseTailTimer, TransmitMode, TransmitModeSelector, VadBackend,
};
pub use chanora_audio::{PttBinding, PttInputClass};
pub use chanora_diagnostics::{
@@ -704,7 +705,7 @@ impl ChanoraSession {
/// Fails if not connected. Idempotent — calling twice replaces
/// the engine. Stores the config so the supervisor can restart
/// audio after a reconnect.
pub async fn start_audio(&self, cfg: AudioEngineConfig) -> Result<(), CoreError> {
pub async fn start_audio(&self, mut cfg: AudioEngineConfig) -> Result<(), CoreError> {
let mut guard = self.inner.lock().await;
let state = guard.as_mut().ok_or(CoreError::NotConnected)?;
@@ -725,6 +726,7 @@ impl ChanoraSession {
.take_voice_in()
.ok_or(CoreError::Invariant("voice_in already taken"))?;
let gate = AudioTransmitGate::new(cfg.ptt_initial);
cfg.voice_activity_selector = Some(self.voice_selector.clone());
let new_engine = match chanora_audio::AudioEngine::start_with_gate(
cfg.clone(),
voice_out,
@@ -996,10 +998,10 @@ impl ChanoraSession {
}
/// Update self-mute state. `input` mutes the microphone, `output`
/// mutes the local speaker for remote clients. Pass `None` to
/// leave a field unchanged. Adjusting the local output mute also
/// updates the audio engine's master output gain so playback
/// silences immediately, independent of the server's broadcast.
/// mutes the local speaker. Pass `None` to leave a field
/// unchanged. Adjusting the local output mute also updates the
/// audio engine's master output gain so playback silences
/// immediately, independent of the server's broadcast.
pub async fn set_self_muted(
&self,
input: Option<bool>,
@@ -1007,37 +1009,24 @@ impl ChanoraSession {
) -> Result<(), CoreError> {
let mut guard = self.inner.lock().await;
let state = guard.as_mut().ok_or(CoreError::NotConnected)?;
// Server-side output mute/deafen makes TeamSpeak/tsclientlib
// consider the client unable to send audio. That is correct
// for a server-visible "deafened" state, but our P0 speaker
// button is a local playback mute. Keep output mute off the
// server-side output/deafen flag, but fold it into the mic
// disabled state below because P0 product semantics are:
// speaker disabled also means microphone disabled.
if let Some(muted) = input {
state.local_input_muted = muted;
}
if let Some(muted) = output {
state.local_output_muted = muted;
}
let mic_disabled = state.local_input_muted || state.local_output_muted;
state.protocol.set_muted(Some(mic_disabled), None).await?;
state.protocol.set_muted(input, output).await?;
if let Some(muted) = output {
if let Some(audio) = state.audio.as_ref() {
audio.set_output_muted(muted);
}
}
// When server-side input mute is engaged we must ALSO stop
// producing outbound voice frames locally — otherwise the
// Opus encoder happily writes packets, the protocol layer
// hands them to tsclientlib, tsclientlib refuses them
// because its own ClientMuted flag is set, and logs
// "Sending audio while muted" once per 20 ms frame. That
// flooded the log to 200 MB on the Korean test host.
// Clamp the transmit-mode selector's hard_mute input so
// the gate goes false too.
self.voice_selector
.set_hard_mute(mic_disabled);
// Input mute must also stop local outbound voice production
// so the transmit selector stays in sync with the server-side
// mic mute. Output mute is playback-only and must not affect
// the mic gate.
let mic_disabled = state.local_input_muted;
self.voice_selector.set_hard_mute(mic_disabled);
Ok(())
}
@@ -1059,18 +1048,98 @@ impl ChanoraSession {
Ok((audio.frames_sent(), audio.frames_received(), audio.ptt()))
}
/// iOS route-change hook (SDD-100). No-op when audio is not
/// running.
pub async fn ios_handle_route_change(&self) -> Result<(), CoreError> {
/// Apply the Rust-owned P1 audio-processing configuration.
pub async fn set_audio_processing_config(
&self,
config: AudioProcessingConfig,
) -> Result<(), CoreError> {
let guard = self.inner.lock().await;
if let Some(state) = guard.as_ref() {
if let Some(audio) = state.audio.as_ref() {
let state = guard.as_ref().ok_or(CoreError::NotConnected)?;
let audio = state.audio.as_ref().ok_or(CoreError::AudioNotStarted)?;
let new_config = config.clone();
let current = audio.audio_processing_config_snapshot();
audio.set_audio_processing_config(config)?;
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
if current.route != new_config.route
|| current.ios_mode != new_config.ios_mode
|| current.processing_backend != new_config.processing_backend
{
audio.ios_restart_voice_unit()?;
}
}
Ok(())
}
/// Read the current audio-processing configuration.
///
/// Returns the live config snapshot from the audio engine, or a
/// default config when no session / audio engine is active.
pub async fn get_audio_processing_config(&self) -> Result<AudioProcessingConfig, CoreError> {
let guard = self.inner.lock().await;
if let Some(state) = guard.as_ref() {
if let Some(audio) = state.audio.as_ref() {
return Ok(audio.audio_processing_config_snapshot());
}
}
Ok(AudioProcessingConfig::default())
}
/// Read P1 audio-processing diagnostics.
pub async fn audio_processing_stats(&self) -> Result<AudioProcessingStats, CoreError> {
let guard = self.inner.lock().await;
let state = guard.as_ref().ok_or(CoreError::NotConnected)?;
let audio = state.audio.as_ref().ok_or(CoreError::AudioNotStarted)?;
Ok(audio.audio_processing_stats())
}
/// Configure the preferred Silero ONNX VAD model path.
///
/// This does not require an active connection. Running iOS audio
/// backends observe the model-path epoch and reload on the next
/// capture frame when Silero is selected.
pub async fn set_vad_model_path(&self, path: String) -> Result<(), CoreError> {
chanora_audio::vad::set_silero_model_path(&path)?;
Ok(())
}
/// iOS route-change hook (SDD-100). No-op when audio is not running.
pub async fn ios_handle_route_change(&self, route: AudioRoute) -> Result<(), CoreError> {
let guard = self.inner.lock().await;
if let Some(state) = guard.as_ref() {
if let Some(audio) = state.audio.as_ref() {
let mut config = audio.audio_processing_config_snapshot();
config.route = route;
audio.set_audio_processing_config(config)?;
}
}
Ok(())
}
/// Enable or disable async WAV debug dump for the active audio
/// session (DIAG_002 / DIAG_003). No-op when audio is not started.
pub async fn set_audio_debug_wav_dump(&self, enabled: bool) -> Result<(), CoreError> {
let guard = self.inner.lock().await;
if let Some(state) = guard.as_ref() {
if let Some(audio) = state.audio.as_ref() {
let mut config = audio.audio_processing_config_snapshot();
config.debug_wav_dump_enabled = enabled;
audio.set_audio_processing_config(config)?;
}
}
Ok(())
}
/// iOS media-services-reset hook (SDD-101). Rebuilds the audio
/// unit using the supplied route so the processing policy is
/// correct after the OS-level media reset.
pub async fn ios_handle_media_services_reset(
&self,
route: AudioRoute,
) -> Result<(), CoreError> {
self.ios_handle_route_change(route).await
}
/// iOS interruption-began hook (SDD-101). No-op when audio is
/// not running.
pub async fn ios_handle_interruption_began(&self) -> Result<(), CoreError> {
@@ -1519,6 +1588,7 @@ const WATCHDOG_PROBE_TIMEOUT: Duration = Duration::from_secs(4);
/// declares the connection lost.
const WATCHDOG_MAX_MISSES: u32 = 3;
#[allow(clippy::too_many_arguments)]
async fn supervisor_loop(
state_arc: Arc<Mutex<Option<ConnectedState>>>,
events_tx: broadcast::Sender<SessionEvent>,