edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate chat voice and diagnostics client

Browse Source
This commit is contained in:
Edison Jwa
2026-05-23 06:51:55 +09:00
parent 7e28791ec2
commit 7d5d8c2c90
93 changed files with 10273 additions and 3347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/Cargo.toml
+1 -8
View File
@@ -28,6 +28,7 @@ audiopus = "0.3.0-rc.0"
# Connection type stays inside chanora_protocol.
tsclientlib = { git = "https://github.com/ReSpeak/tsclientlib.git", rev = "04aa2491", default-features = false, features = ["audio"] }
tokio = { version = "1", features = ["sync", "rt", "macros", "time"] }
rustfft = "6.2.0"
[target.'cfg(all(not(target_os = "android"), not(target_os = "ios"), not(target_os = "macos")))'.dependencies]
# Desktop audio I/O for Windows capture/playback and Linux capture.
@@ -35,11 +36,6 @@ tokio = { version = "1", features = ["sync", "rt", "macros", "time"] }
# AudioUnits via `coreaudio-rs` for the voice path.
cpal = "0.17.3"
[target.'cfg(not(target_os = "android"))'.dependencies]
# Desktop/iOS: native TLS maps to the platform TLS backend (Security.framework
# on Apple, SChannel on Windows, system OpenSSL on Linux/BSD).
reqwest = { version = "0.13", default-features = false, features = ["charset", "http2", "native-tls"] }
[target.'cfg(any(target_os = "ios", target_os = "macos"))'.dependencies]
# Direct CoreAudio AudioUnit access on Apple platforms (DEC-011 follow-up).
# cpal's Apple path does not expose the voice-processing controls Chanora
@@ -64,9 +60,6 @@ ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "n
ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "api-24"] }
[target.'cfg(target_os = "android")'.dependencies]
# Android cross-builds should not pull OpenSSL. Use rustls here while keeping
# native-tls for Apple targets where aws-lc/rustls is problematic for iOS.
reqwest = { version = "0.13", default-features = false, features = ["charset", "http2", "rustls"] }
# JNI bindings to flip Android's AudioManager into MODE_IN_COMMUNICATION
# when the voice-comm preset is requested. ndk_context is initialised
# by the bridge crate's android_init shim.
crates/chanora_audio/benches/realtime_capture.rs
+1 -1
View File
@@ -95,7 +95,7 @@ fn bench_capture_alloc_count(c: &mut Criterion) {
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let _ = std::fs::write(&path, format!("{}", delta));
let _ = std::fs::write(&path, delta.to_string());
}
}
crates/chanora_audio/src/android_voice_unit.rs
+53 -77
View File
@@ -50,7 +50,7 @@ use crate::mobile_voice_backend::{
AchievedPerformanceMode, AchievedSharingMode, AndroidAudioDiagnostics,
AndroidVoiceStreamConfig, AudioSessionId, BackendError, BackendEvent, BackendEventRx,
BackendEventTx, EffectEngagement, EffectEngine, InputPresetChoice, MobileVoiceAudioBackend,
SharingModeChoice,
SharingModeChoice, VoiceAudioParams,
};
use chanora_protocol::OutPacket;
use tsclientlib::audio::AudioHandler;
@@ -102,8 +102,8 @@ impl RenderReferenceBuffer {
fn write(&self, frame: &[f32; RENDER_REF_SAMPLES]) {
let idx = self.write_idx.load(Ordering::Relaxed);
unsafe {
let slot =
&self.buf[idx] as *const [f32; RENDER_REF_SAMPLES] as *mut [f32; RENDER_REF_SAMPLES];
let slot = &self.buf[idx] as *const [f32; RENDER_REF_SAMPLES]
as *mut [f32; RENDER_REF_SAMPLES];
(*slot).copy_from_slice(frame);
}
self.write_idx
@@ -148,6 +148,7 @@ struct AndroidCaptureState {
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
ten_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
@@ -208,6 +209,7 @@ impl AndroidCaptureState {
ten_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
@@ -227,8 +229,7 @@ impl AndroidCaptureState {
fn ingest_i16(&mut self, samples: &[i16]) {
let mut offset = 0;
while offset < samples.len() {
let remaining =
crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let take = remaining.min(samples.len() - offset);
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
.copy_from_slice(&samples[offset..offset + take]);
@@ -249,11 +250,8 @@ impl AndroidCaptureState {
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(
&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES],
);
self.pcm_accum
.drain(..crate::frame::FRAME_20MS_SAMPLES);
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
crate::opus_voice::send_voip_frame(
@@ -289,18 +287,26 @@ impl AndroidCaptureState {
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend != Some(failed_backend) {
self.fallback_warned_backend = Some(failed_backend);
warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"android: VAD backend unavailable; using WebRTC fallback for runtime detection"
);
// Warm-up period: ONNX workers need ~100ms to process first frame.
// Don't flag as a problem if the capture has just started.
if self.capture_frame_seq < 128 {
info!(
target: "chanora_audio",
backend = failed_backend.as_str(),
seq = self.capture_frame_seq,
"android: VAD backend warming up; using WebRTC fallback"
);
} else {
warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"android: VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
}
}
fn process_10ms_capture_frame(
&mut self,
samples: &[i16; crate::frame::FRAME_10MS_SAMPLES],
) {
fn process_10ms_capture_frame(&mut self, samples: &[i16; crate::frame::FRAME_10MS_SAMPLES]) {
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
*dst = crate::frame::i16_to_f32(src);
@@ -327,11 +333,15 @@ impl AndroidCaptureState {
// VAD backend switching (mirrors iOS Raw path).
let silero_epoch = crate::vad::silero_model_epoch();
let silero_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed {
let ten_epoch = crate::vad::ten_model_epoch();
let silero_changed =
vad_backend == crate::VadBackend::SileroOnnx && silero_epoch != self.silero_model_epoch;
let ten_changed =
vad_backend == crate::VadBackend::TenVad && ten_epoch != self.ten_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed || ten_changed {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.ten_model_epoch = ten_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
@@ -376,52 +386,52 @@ impl AndroidCaptureState {
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
// Warm-up: worker dispatched but hasn't finished yet.
// Default to no-speech until first result arrives (~100ms).
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
@@ -572,37 +582,6 @@ impl AudioOutputCallback for OutputCallback {
}
}
/// Bundle of engine-owned state shared with the Oboe audio callbacks.
/// Mirrors the parameter set that iOS `IosVoiceUnit::start()` receives
/// from the engine (SDD-120 amendment: Android Oboe-only audio path).
pub struct VoiceAudioParams {
/// Opus-encoded voice packets sent on this channel toward the
/// protocol layer.
pub voice_out_tx: mpsc::Sender<OutPacket>,
/// PTT transmission gate — true when the user holds the PTT key.
pub transmit_active: Arc<AtomicBool>,
/// Counter incremented per encoded frame sent.
pub frames_sent: Arc<AtomicU32>,
/// Pre-encode amplitude scale (1.0 = unity).
pub mic_gain: f32,
/// AudioHandler that inbound decode+mix feeds into; the Oboe output
/// callback pulls mixed stereo f32 from it.
pub handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
/// Master output gain (f32 bits stored in AtomicU32 for lock-free
/// cross-thread read from the realtime audio callback).
pub output_gain: Arc<AtomicU32>,
/// True = output silence regardless of incoming voice frames.
pub output_muted: Arc<AtomicBool>,
/// Optional TransmitModeSelector for VoiceActivity transmit mode.
/// The capture callback calls set_voice_activity_open on this when
/// VAD detects speech. None means VoiceActivity mode is disabled.
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
/// Shared audio-processing config (WebRTC APM flags, VAD backend).
pub audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
/// Shared audio-processing statistics for diagnostics.
pub audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
}
// --- The backend itself ------------------------------------------
/// Android voice-audio backend (SDD-111). Owns one input + one
@@ -647,8 +626,7 @@ impl AndroidVoiceUnit {
/// `params` bundles the engine-owned state shared with the Oboe
/// audio callbacks (SDD-120 amendment: Android Oboe-only audio
/// path — capture pipeline, playback pull, and PTT gate).
#[allow(clippy::too_many_arguments)]
pub fn open(
pub(crate) fn open(
cfg: &AndroidVoiceStreamConfig,
params: VoiceAudioParams,
) -> Result<Self, BackendError> {
@@ -761,9 +739,7 @@ impl AndroidVoiceUnit {
//
// For now: hardware effects require the system session ID.
// WebRTC APM software processing handles AEC/NS/AGC/HPF.
let session_id: Option<i32> = input_stream
.as_ref()
.and_then(|s| s.get_raw_session_id());
let session_id: Option<i32> = input_stream.as_ref().and_then(|s| s.get_raw_session_id());
// --- Open output stream (SDD-112) --------------------------
let output_builder = AudioStreamBuilder::default()
@@ -1558,7 +1534,7 @@ fn load_app_class<'local>(
Ok(c) => return Some(c),
Err(e) => {
let _ = env.exception_clear();
warn!(target: "chanora_audio", error = %e, class = slash_name, "android: find_class failed; retrying with app ClassLoader");
debug!(target: "chanora_audio", error = %e, class = slash_name, "android: find_class failed; retrying with app ClassLoader");
}
}
crates/chanora_audio/src/audio_processing.rs
+7 -7
View File
@@ -207,12 +207,12 @@ impl AudioProcessingConfig {
.to_string(),
));
}
if self.ios_mode == IosVoiceProcessingMode::SonoraExperimental {
if self.processing_backend != AudioBackend::WebrtcApm {
return Err(AudioError::InvalidAudioProcessingConfig(
"ios raw processing mode requires the WebRTC APM backend".to_string(),
));
}
if self.ios_mode == IosVoiceProcessingMode::SonoraExperimental
&& self.processing_backend != AudioBackend::WebrtcApm
{
return Err(AudioError::InvalidAudioProcessingConfig(
"ios raw processing mode requires the WebRTC APM backend".to_string(),
));
}
Ok(())
}
@@ -246,7 +246,7 @@ mod tests {
assert_eq!(config.aec, EffectOwner::Platform);
assert_eq!(config.ns, EffectOwner::Platform);
assert_eq!(config.agc, EffectOwner::Platform);
assert_eq!(config.vad_backend, VadBackend::SileroOnnx);
assert_eq!(config.vad_backend, VadBackend::TenVad);
}
#[test]
crates/chanora_audio/src/engine.rs
+235 -167
View File
@@ -55,17 +55,98 @@ use audiopus::coder::Encoder as OpusEncoder;
pub struct SessionAudioId(pub u64);
/// Audio framing: 48 kHz mono, 20 ms = 960 samples per frame.
/// These constants are framing invariants of the engine and are
/// referenced from per-platform helpers (`try_open_capture` and
/// the CaptureState on cpal platforms; `ios_voice_unit` on iOS once
/// commits 3+4 land). The `allow(dead_code)` is here because in
/// the current commit the iOS VPIO callbacks are still no-op stubs
/// and don't reach these constants yet — they will in commit 3
/// when the input callback wires into CaptureState.
#[allow(dead_code)]
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
const SAMPLE_RATE: u32 = 48_000;
#[allow(dead_code)]
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
const FRAME_SAMPLES: usize = 48_000 / 50; // 960
/// List of available audio devices from the platform.
#[derive(Debug, Clone)]
pub struct AudioDeviceList {
/// Available input (capture) devices.
pub input_devices: Vec<AudioDeviceInfo>,
/// Available output (playback) devices.
pub output_devices: Vec<AudioDeviceInfo>,
}
/// Info about a single audio device.
#[derive(Debug, Clone)]
pub struct AudioDeviceInfo {
/// Human-readable device name from the OS.
pub name: String,
/// True if the OS reports this as the default device.
pub is_default: bool,
}
/// Enumerate available audio input and output devices.
/// On mobile platforms (iOS, Android) returns an empty list because
/// device selection is managed by the OS audio session.
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
pub fn list_audio_devices() -> AudioDeviceList {
use cpal::traits::HostTrait;
let mut list = AudioDeviceList {
input_devices: Vec::new(),
output_devices: Vec::new(),
};
let Ok(host) = cpal::default_host() else {
return list;
};
let default_in = host.default_input_device();
let default_out = host.default_output_device();
if let Ok(devices) = host.input_devices() {
for d in devices {
let name = d
.description()
.map(|n| n.name().to_owned())
.unwrap_or_default();
if !name.is_empty() {
let is_default = default_in
.as_ref()
.is_some_and(|di| di.description().is_ok_and(|dn| dn.name() == name.as_str()));
list.input_devices
.push(AudioDeviceInfo { name, is_default });
}
}
}
if let Ok(devices) = host.output_devices() {
for d in devices {
let name = d
.description()
.map(|n| n.name().to_owned())
.unwrap_or_default();
if !name.is_empty() {
let is_default = default_out
.as_ref()
.is_some_and(|di| di.description().is_ok_and(|dn| dn.name() == name.as_str()));
list.output_devices
.push(AudioDeviceInfo { name, is_default });
}
}
}
list
}
#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
/// Enumerate available audio input and output devices.
pub fn list_audio_devices() -> AudioDeviceList {
AudioDeviceList {
input_devices: Vec::new(),
output_devices: Vec::new(),
}
}
/// Engine configuration.
#[derive(Clone)]
pub struct AudioEngineConfig {
@@ -91,6 +172,12 @@ pub struct AudioEngineConfig {
/// is rejected on Android because the P0 path intentionally has
/// no generic mobile-audio fallback.
pub mobile_voice_preset: bool,
/// Optional input device name override. When `None`, the system
/// default input device is used. Set to a device name from
/// [`list_audio_devices`] to pin a specific microphone.
pub input_device_name: Option<String>,
/// Optional output device name override.
pub output_device_name: Option<String>,
/// Optional selector used by P1 VoiceActivity to publish VAD state.
#[doc(hidden)]
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
@@ -103,6 +190,8 @@ impl std::fmt::Debug for AudioEngineConfig {
.field("ptt_initial", &self.ptt_initial)
.field("effects", &self.effects)
.field("mobile_voice_preset", &self.mobile_voice_preset)
.field("input_device_name", &self.input_device_name)
.field("output_device_name", &self.output_device_name)
.field(
"voice_activity_selector",
&self.voice_activity_selector.as_ref().map(|_| "present"),
@@ -118,6 +207,8 @@ impl Default for AudioEngineConfig {
ptt_initial: false,
effects: crate::AudioEffects::default(),
mobile_voice_preset: true,
input_device_name: None,
output_device_name: None,
voice_activity_selector: None,
}
}
@@ -145,7 +236,6 @@ pub struct AudioEngine {
output_muted: Arc<AtomicBool>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
voice_out_tx: mpsc::Sender<OutPacket>,
@@ -204,23 +294,6 @@ pub struct AudioEngine {
/// denied microphone permission), PTT becomes a no-op and
/// `frames_sent` stays at 0.
capture_active: bool,
/// Missed-key-up watchdog (SDD-092). Dropping aborts the task.
/// The watchdog is independent of the PTT input backend — it
/// observes the gate directly. The platform input backend is
/// owned by `chanora_core::ptt::PttController` (SDD-088), not
/// by the engine.
///
/// In the post-rc.7 architecture this field is unused: the
/// missed-key-up watchdog now lives on the session and
/// subscribes to `TransmitModeSelector::subscribe_ptt_held`
/// rather than the gate. Watching the gate caused the watchdog
/// to fire in Continuous mode (where the gate is intentionally
/// pinned to `true`) which clearing surfaced as the bug
/// "Continuous transmission disabled after some time". The
/// field stays here as `None` for now to preserve the existing
/// engine-stop teardown flow; a follow-up commit can remove it
/// entirely.
ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog>,
}
// cpal::Stream is not Send. We keep the engine pinned to the thread
@@ -241,7 +314,6 @@ unsafe impl Send for AudioEngine {}
unsafe impl Sync for AudioEngine {}
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[allow(dead_code)]
enum IosVoiceBackend {
Vpio(crate::ios_voice_unit::IosVoiceUnit),
#[cfg(target_os = "ios")]
@@ -260,7 +332,9 @@ impl IosVoiceBackend {
}
#[cfg(target_os = "macos")]
{
Ok(())
match self {
Self::Vpio(_unit) => Ok(()),
}
}
}
@@ -274,41 +348,23 @@ impl IosVoiceBackend {
}
#[cfg(target_os = "macos")]
{
Ok(())
match self {
Self::Vpio(_unit) => Ok(()),
}
}
}
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[allow(clippy::too_many_arguments)]
fn open_ios_voice_backend(
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_flag_for_capture: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
params: crate::mobile_voice_backend::VoiceAudioParams,
) -> Result<IosVoiceBackend, AudioError> {
let _cfg = audio_processing_config.lock().unwrap().clone();
let _cfg = params.audio_processing_config.lock().unwrap().clone();
#[cfg(target_os = "ios")]
{
if _cfg.ios_mode == crate::IosVoiceProcessingMode::SonoraExperimental {
match crate::ios_raw_unit::IosRawUnit::start(
handler.clone(),
output_gain.clone(),
output_muted.clone(),
voice_out_tx.clone(),
transmit_flag_for_capture.clone(),
frames_sent.clone(),
mic_gain,
voice_activity_selector.clone(),
audio_processing_config.clone(),
audio_processing_stats.clone(),
) {
let raw_params = params.clone();
match crate::ios_raw_unit::IosRawUnit::start(raw_params) {
Ok(unit) => {
info!(target: "chanora_audio", "ios: RemoteIO/WebRTC APM backend selected");
return Ok(IosVoiceBackend::Raw(unit));
@@ -324,18 +380,7 @@ fn open_ios_voice_backend(
}
}
let unit = crate::ios_voice_unit::IosVoiceUnit::start(
handler,
output_gain,
output_muted,
voice_out_tx,
transmit_flag_for_capture,
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats,
)?;
let unit = crate::ios_voice_unit::IosVoiceUnit::start(params)?;
Ok(IosVoiceBackend::Vpio(unit))
}
@@ -361,23 +406,40 @@ impl AudioEngine {
voice_in_rx: mpsc::Receiver<InboundVoice>,
transmit_gate: crate::ptt::AudioTransmitGate,
) -> Result<Self, AudioError> {
#[allow(clippy::needless_return)]
// Apple platforms route to a separate backend (VoiceProcessingIO
// via coreaudio-rs) because cpal does not expose the native
// voice-processing AudioUnit controls Chanora needs for VoIP.
// Windows and Linux stay on the cpal / SDL flow below.
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
return Self::start_with_gate_ios(cfg, voice_out_tx, voice_in_rx, transmit_gate);
}
#[cfg(target_os = "android")]
{
return Self::start_with_gate_android(cfg, voice_out_tx, voice_in_rx, transmit_gate);
}
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
{
Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate)
}
Self::start_with_gate_platform(cfg, voice_out_tx, voice_in_rx, transmit_gate)
}
// Apple platforms route to a separate backend (VoiceProcessingIO
// via coreaudio-rs) because cpal does not expose the native
// voice-processing AudioUnit controls Chanora needs for VoIP.
#[cfg(any(target_os = "ios", target_os = "macos"))]
fn start_with_gate_platform(
cfg: AudioEngineConfig,
voice_out_tx: mpsc::Sender<OutPacket>,
voice_in_rx: mpsc::Receiver<InboundVoice>,
transmit_gate: crate::ptt::AudioTransmitGate,
) -> Result<Self, AudioError> {
Self::start_with_gate_ios(cfg, voice_out_tx, voice_in_rx, transmit_gate)
}
#[cfg(target_os = "android")]
fn start_with_gate_platform(
cfg: AudioEngineConfig,
voice_out_tx: mpsc::Sender<OutPacket>,
voice_in_rx: mpsc::Receiver<InboundVoice>,
transmit_gate: crate::ptt::AudioTransmitGate,
) -> Result<Self, AudioError> {
Self::start_with_gate_android(cfg, voice_out_tx, voice_in_rx, transmit_gate)
}
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
fn start_with_gate_platform(
cfg: AudioEngineConfig,
voice_out_tx: mpsc::Sender<OutPacket>,
voice_in_rx: mpsc::Receiver<InboundVoice>,
transmit_gate: crate::ptt::AudioTransmitGate,
) -> Result<Self, AudioError> {
Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate)
}
/// Non-Apple/non-Android implementation: cpal capture + (cpal | SDL2) output.
@@ -398,12 +460,45 @@ impl AudioEngine {
host_id = ?host.id(),
"starting audio engine: cpal host selected"
);
let in_dev = host
.default_input_device()
.ok_or(AudioError::NoInputDevice)?;
let out_dev = host
.default_output_device()
.ok_or(AudioError::NoOutputDevice)?;
/// Helper: find a device by name, falling back to default.
fn find_device(
host: &cpal::Host,
default_fn: fn(&cpal::Host) -> Option<cpal::Device>,
all_fn: fn(&cpal::Host) -> Result<cpal::Devices, cpal::DevicesError>,
prefer: Option<&str>,
) -> Option<cpal::Device> {
if let Some(name) = prefer {
if let Ok(devices) = all_fn(host) {
for d in devices {
let dn = d
.description()
.map(|n| n.name().to_owned())
.unwrap_or_default();
if dn == name {
return Some(d);
}
}
}
}
default_fn(host)
}
let in_dev = find_device(
&host,
cpal::Host::default_input_device,
cpal::Host::input_devices,
cfg.input_device_name.as_deref(),
)
.ok_or(AudioError::NoInputDevice)?;
let out_dev = find_device(
&host,
cpal::Host::default_output_device,
cpal::Host::output_devices,
cfg.output_device_name.as_deref(),
)
.ok_or(AudioError::NoOutputDevice)?;
info!(
target: "chanora_audio",
@@ -611,19 +706,6 @@ impl AudioEngine {
}
});
// Select and arm the desktop PTT backend is no longer the
// engine's job (SDD-088). The PTT controller lives in
// `chanora_core::ptt::PttController`; the engine is
// responsible only for the cpal streams and the
// missed-key-up watchdog (SDD-092).
// The engine no longer spawns a watchdog against the
// gate (see comment on the `ptt_watchdog` field for the
// rationale). The session spawns the watchdog against the
// selector's `ptt_held` signal instead, so it never fires
// in Continuous mode.
let ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog> = None;
Ok(Self {
transmit_gate,
frames_sent,
@@ -632,11 +714,11 @@ impl AudioEngine {
output_muted,
audio_processing_config,
audio_processing_stats,
audio_handler,
_input_stream: Mutex::new(input_stream),
_output_stream: Mutex::new(Some(output_stream)),
shutdown_tx: Some(shutdown_tx),
capture_active,
ptt_watchdog,
})
}
@@ -716,7 +798,7 @@ impl AudioEngine {
effects: cfg.effects,
..Default::default()
};
let params = crate::android_voice_unit::VoiceAudioParams {
let params = crate::mobile_voice_backend::VoiceAudioParams {
voice_out_tx,
transmit_active: transmit_flag_for_capture,
frames_sent: frames_sent.clone(),
@@ -841,8 +923,6 @@ impl AudioEngine {
}
});
let ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog> = None;
Ok(Self {
transmit_gate,
frames_sent,
@@ -851,11 +931,11 @@ impl AudioEngine {
output_muted,
audio_processing_config,
audio_processing_stats,
audio_handler,
_android_voice_unit: Mutex::new(Some(android_voice_unit)),
audio_mode_stack: Mutex::new(audio_mode_stack),
shutdown_tx: Some(shutdown_tx),
capture_active,
ptt_watchdog,
})
}
@@ -907,18 +987,19 @@ impl AudioEngine {
// Construct the live iOS voice backend. Platform VPIO stays
// the default shipping path; Sonora/RemoteIO remains opt-in.
let ios_voice_backend = open_ios_voice_backend(
audio_handler.clone(),
output_gain.clone(),
output_muted.clone(),
voice_out_tx_for_backend,
transmit_flag_for_capture,
frames_sent.clone(),
cfg.mic_gain,
cfg.voice_activity_selector.clone(),
audio_processing_config.clone(),
audio_processing_stats.clone(),
)?;
let ios_voice_backend =
open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams {
handler: audio_handler.clone(),
output_gain: output_gain.clone(),
output_muted: output_muted.clone(),
voice_out_tx: voice_out_tx_for_backend,
transmit_active: transmit_flag_for_capture,
frames_sent: frames_sent.clone(),
mic_gain: cfg.mic_gain,
voice_activity_selector: cfg.voice_activity_selector.clone(),
audio_processing_config: audio_processing_config.clone(),
audio_processing_stats: audio_processing_stats.clone(),
})?;
// Capture is always considered active on iOS — VPIO's
// input element is wired up by the AudioUnit itself, no
@@ -959,8 +1040,6 @@ impl AudioEngine {
}
});
let ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog> = None;
Ok(Self {
transmit_gate,
frames_sent,
@@ -976,7 +1055,6 @@ impl AudioEngine {
_ios_voice_backend: Mutex::new(Some(ios_voice_backend)),
shutdown_tx: Some(shutdown_tx),
capture_active,
ptt_watchdog,
})
}
@@ -985,12 +1063,6 @@ impl AudioEngine {
if let Some(tx) = self.shutdown_tx.take() {
let _ = tx.send(());
}
// The platform PTT backend is no longer owned by the
// engine (SDD-088); its lifecycle is managed by
// `chanora_core::ptt::PttController`. The engine only
// needs to abort its watchdog and drop the audio streams.
// Aborting the watchdog cancels its tokio task.
self.ptt_watchdog.take();
// Drop the streams, which stops their callback threads.
// Each platform has a slightly different backend; the
// common contract is that dropping the wrapper stops
@@ -1088,18 +1160,18 @@ impl AudioEngine {
pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let backend = open_ios_voice_backend(
self.audio_handler.clone(),
self.output_gain.clone(),
self.output_muted.clone(),
self.voice_out_tx.clone(),
self.transmit_gate.flag_arc(),
self.frames_sent.clone(),
self.mic_gain,
self.voice_activity_selector.clone(),
self.audio_processing_config.clone(),
self.audio_processing_stats.clone(),
)?;
let backend = open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams {
handler: self.audio_handler.clone(),
output_gain: self.output_gain.clone(),
output_muted: self.output_muted.clone(),
voice_out_tx: self.voice_out_tx.clone(),
transmit_active: self.transmit_gate.flag_arc(),
frames_sent: self.frames_sent.clone(),
mic_gain: self.mic_gain,
voice_activity_selector: self.voice_activity_selector.clone(),
audio_processing_config: self.audio_processing_config.clone(),
audio_processing_stats: self.audio_processing_stats.clone(),
})?;
let mut guard = self._ios_voice_backend.lock().unwrap();
*guard = Some(backend);
Ok(())
@@ -1166,33 +1238,6 @@ impl AudioEngine {
&self.transmit_gate
}
/// Privacy-safe descriptor of the engine's PTT view. The
/// platform backend lives in `chanora_core::ptt::PttController`
/// (SDD-088); the engine itself no longer owns it. This getter
/// always returns the universal Focused fallback descriptor
/// and is retained only for legacy callers that constructed
/// engines directly without a controller (tests, headless
/// diagnostics).
pub fn ptt_descriptor(&self) -> crate::ptt::PttBackendDescriptor {
crate::ptt::PttBackendDescriptor::focused()
}
/// Legacy alias for [`Self::set_transmit_active`]. Retained so
/// the existing bridge `set_ptt` command and the existing
/// Flutter UI continue to compile during the v0.9.3 PTT
/// migration (SRS-201 splits the conceptual `ptt` flag into
/// `transmit_active` / `capture_active`).
#[doc(hidden)]
pub fn set_ptt(&self, active: bool) {
self.set_transmit_active(active);
}
/// Legacy alias for [`Self::transmit_active`].
#[doc(hidden)]
pub fn ptt(&self) -> bool {
self.transmit_active()
}
/// True if the capture stream opened. When false, the engine
/// runs in playback-only mode and the transmit gate is a
/// no-op (no frames will ever be encoded).
@@ -1268,6 +1313,29 @@ impl AudioEngine {
pub fn output_gain(&self) -> f32 {
f32::from_bits(self.output_gain.load(Ordering::Relaxed))
}
/// Set per-client output volume (SRS-075). `1.0` is unity, `0.0`
/// mutes. Values above `1.0` amplify and may clip. Clamped to
/// `0.0..4.0`.
pub fn set_client_volume(&self, client_id: u64, volume: f32) {
let clamped = volume.clamp(0.0, 4.0);
match self.audio_handler.lock() {
Ok(mut h) => {
if let Some(q) = h.get_mut_queues().get_mut(&SessionAudioId(client_id)) {
q.volume = clamped;
}
}
Err(e) => {
tracing::warn!(
target: "chanora_audio",
client_id,
volume = clamped,
error = %e,
"set_client_volume: audio_handler lock poisoned — volume not applied"
);
}
}
}
}
impl Drop for AudioEngine {
crates/chanora_audio/src/ios_raw_unit.rs
+52 -53
View File
@@ -44,9 +44,8 @@ mod inner {
use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat};
use tokio::sync::mpsc;
use tracing::{info, warn};
use tsclientlib::audio::AudioHandler;
use crate::engine::SessionAudioId;
use crate::mobile_voice_backend::VoiceAudioParams;
use crate::processor::AudioProcessor;
use crate::AudioError;
use chanora_protocol::OutPacket;
@@ -121,6 +120,7 @@ mod inner {
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
ten_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
/// Processing config — retained for route-change reloads.
@@ -136,18 +136,12 @@ mod inner {
impl RawCaptureState {
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
params: &VoiceAudioParams,
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("ios raw")?;
let webrtc_apm_config = audio_processing_config
let webrtc_apm_config = params
.audio_processing_config
.lock()
.map(|cfg| crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg))
.unwrap_or_default();
@@ -155,24 +149,25 @@ mod inner {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
output_muted,
frames_sent,
mic_gain,
voice_activity_selector,
voice_out_tx: params.voice_out_tx.clone(),
transmit_active: params.transmit_active.clone(),
output_muted: params.output_muted.clone(),
frames_sent: params.frames_sent.clone(),
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
audio_processing_config: params.audio_processing_config.clone(),
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
webrtc_apm_config,
)?,
audio_processing_stats,
audio_processing_stats: params.audio_processing_stats.clone(),
render_reference,
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
@@ -184,11 +179,20 @@ mod inner {
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend != Some(failed_backend) {
self.fallback_warned_backend = Some(failed_backend);
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
if self.capture_frame_seq < 128 {
tracing::info!(
target: "chanora_audio",
backend = failed_backend.as_str(),
seq = self.capture_frame_seq,
"VAD backend warming up; using WebRTC fallback"
);
} else {
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
}
}
@@ -286,11 +290,15 @@ mod inner {
// Switch VAD backend when config changes.
let silero_epoch = crate::vad::silero_model_epoch();
let ten_epoch = crate::vad::ten_model_epoch();
let silero_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed {
let ten_changed =
vad_backend == crate::VadBackend::TenVad && ten_epoch != self.ten_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed || ten_changed {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.ten_model_epoch = ten_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
@@ -335,12 +343,17 @@ mod inner {
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
@@ -357,12 +370,17 @@ mod inner {
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
@@ -422,22 +440,10 @@ mod inner {
impl IosRawUnit {
/// Open a RemoteIO AudioUnit, install render + input callbacks, start.
#[allow(clippy::too_many_arguments)]
pub fn start(
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<Self, AudioError> {
pub(crate) fn start(params: VoiceAudioParams) -> Result<Self, AudioError> {
// INV_010: reject if config requests VPIO (that's IosVoiceUnit's job).
{
let cfg = audio_processing_config.lock().unwrap();
let cfg = params.audio_processing_config.lock().unwrap();
if cfg.ios_mode == crate::IosVoiceProcessingMode::PlatformVoiceProcessing {
return Err(AudioError::InvalidAudioProcessingConfig(
"IosRawUnit requires raw WebRTC APM mode".to_string(),
@@ -470,17 +476,7 @@ mod inner {
let render_ref_buf = RenderReferenceBuffer::new();
let render_ref_for_capture = render_ref_buf.clone();
let mut capture_state = RawCaptureState::new(
voice_out_tx,
transmit_active,
output_muted.clone(),
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats.clone(),
render_ref_for_capture,
)?;
let mut capture_state = RawCaptureState::new(&params, render_ref_for_capture)?;
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
capture_state.ingest_i16(args.data.buffer);
@@ -489,7 +485,10 @@ mod inner {
.map_err(|e| AudioError::Backend(format!("remoteio input cb: {e}")))?;
let mut scratch: Vec<f32> = Vec::with_capacity(2048);
let stats_render = audio_processing_stats.clone();
let handler = params.handler.clone();
let output_gain = params.output_gain.clone();
let output_muted = params.output_muted.clone();
let stats_render = params.audio_processing_stats.clone();
unit.set_render_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
let out = args.data.buffer;
crates/chanora_audio/src/ios_voice_unit.rs
+55 -59
View File
@@ -81,9 +81,8 @@ use coreaudio::audio_unit::IOType;
use coreaudio::audio_unit::{AudioUnit, Element, SampleFormat, Scope, StreamFormat};
use tokio::sync::mpsc;
use tracing::{debug, error, info, warn};
use tsclientlib::audio::AudioHandler;
use crate::engine::SessionAudioId;
use crate::mobile_voice_backend::VoiceAudioParams;
use crate::AudioError;
use chanora_protocol::OutPacket;
@@ -152,6 +151,8 @@ struct IosCaptureState {
current_vad_backend: crate::VadBackend,
/// Last observed configured Silero model epoch.
silero_model_epoch: u64,
/// Last observed configured TEN model epoch.
ten_model_epoch: u64,
fallback_warned_backend: Option<crate::VadBackend>,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
@@ -172,16 +173,8 @@ impl IosCaptureState {
/// Encoder configuration is the same as cpal-side
/// `try_open_capture` (engine.rs) so audio quality is platform-
/// neutral.
#[allow(clippy::too_many_arguments)]
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
output_muted: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
params: &VoiceAudioParams,
wav_recorder: Arc<Mutex<Option<Arc<crate::debug_wav::WavDebugRecorder>>>>,
) -> Result<Self, AudioError> {
let encoder = crate::opus_voice::new_voip_encoder("ios VPIO")?;
@@ -190,22 +183,23 @@ impl IosCaptureState {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
output_muted,
frames_sent,
mic_gain,
voice_activity_selector,
voice_out_tx: params.voice_out_tx.clone(),
transmit_active: params.transmit_active.clone(),
output_muted: params.output_muted.clone(),
frames_sent: params.frames_sent.clone(),
mic_gain: params.mic_gain,
voice_activity_selector: params.voice_activity_selector.clone(),
vad_detector: crate::vad::WebRtcFallbackVad::default(),
silero_vad_worker: None,
ten_vad: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
fallback_warned_backend: None,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
audio_processing_config,
audio_processing_config: params.audio_processing_config.clone(),
sonora_processor: crate::processor::SonoraProcessor::new(),
audio_processing_stats,
audio_processing_stats: params.audio_processing_stats.clone(),
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
pending_10ms_len: 0,
pre_roll_buf: [[0_i16; crate::frame::FRAME_10MS_SAMPLES]; PRE_ROLL_FRAMES],
@@ -222,11 +216,20 @@ impl IosCaptureState {
return;
}
self.fallback_warned_backend = Some(failed_backend);
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
if self.capture_frame_seq < 128 {
tracing::info!(
target: "chanora_audio",
backend = failed_backend.as_str(),
seq = self.capture_frame_seq,
"VAD backend warming up; using WebRTC fallback"
);
} else {
tracing::warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
}
/// Consume the i16 mono buffer delivered by VPIO, accumulate
@@ -360,8 +363,11 @@ impl IosCaptureState {
// Switch VAD backend when the config changes.
let silero_model_epoch = crate::vad::silero_model_epoch();
let ten_model_epoch = crate::vad::ten_model_epoch();
let silero_model_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_model_epoch != self.silero_model_epoch;
let ten_model_changed =
vad_backend == crate::VadBackend::TenVad && ten_model_epoch != self.ten_model_epoch;
if let Ok(mut recorder_guard) = self.wav_recorder.try_lock() {
if debug_wav_dump_enabled {
@@ -376,10 +382,11 @@ impl IosCaptureState {
}
}
if vad_backend != self.current_vad_backend || silero_model_changed {
if vad_backend != self.current_vad_backend || silero_model_changed || ten_model_changed {
self.current_vad_backend = vad_backend;
self.fallback_warned_backend = None;
self.silero_model_epoch = silero_model_epoch;
self.ten_model_epoch = ten_model_epoch;
match vad_backend {
crate::VadBackend::SileroOnnx => {
// Attempt to load Silero model from the well-known
@@ -467,12 +474,17 @@ impl IosCaptureState {
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
@@ -486,12 +498,17 @@ impl IosCaptureState {
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let probability = worker.latest_probability();
crate::vad::VadOutput {
probability,
speech: probability >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(crate::VadBackend::TenVad);
@@ -617,19 +634,7 @@ impl IosVoiceUnit {
///
/// Capture wiring landed in commit 3; playback wiring landed
/// in commit 4. Route-change observation is commit 5.
#[allow(clippy::too_many_arguments)]
pub fn start(
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<Self, AudioError> {
pub(crate) fn start(params: VoiceAudioParams) -> Result<Self, AudioError> {
// Construct the VoiceProcessingIO AudioUnit. cpal exposes
// `Default::default()` which on iOS picks the inferior
// RemoteIO unit; we explicitly pick VPIO. `coreaudio-rs`
@@ -721,7 +726,7 @@ impl IosVoiceUnit {
// because the input callback is the sole writer/reader on
// the audio thread.
let wav_recorder = Arc::new(Mutex::new({
let cfg = audio_processing_config.lock().unwrap().clone();
let cfg = params.audio_processing_config.lock().unwrap().clone();
if cfg.debug_wav_dump_enabled {
Some(crate::debug_wav::WavDebugRecorder::start(
cfg.route,
@@ -731,17 +736,7 @@ impl IosVoiceUnit {
None
}
}));
let mut capture_state = IosCaptureState::new(
voice_out_tx,
transmit_active,
output_muted.clone(),
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats.clone(),
wav_recorder.clone(),
)?;
let mut capture_state = IosCaptureState::new(&params, wav_recorder.clone())?;
unit.set_input_callback(move |args: render_callback::Args<data::Interleaved<i16>>| {
// VPIO with our pinned stream format delivers
@@ -806,9 +801,10 @@ impl IosVoiceUnit {
// buffer. Same as Linux/SDL, just stereo-f32 -> mono-i16
// converted at the boundary.
let mut scratch_stereo: Vec<f32> = Vec::with_capacity(2048);
let handler_for_render = handler.clone();
let output_gain_for_render = output_gain.clone();
let output_muted_for_render = output_muted.clone();
let handler_for_render = params.handler.clone();
let output_gain_for_render = params.output_gain.clone();
let output_muted_for_render = params.output_muted.clone();
let audio_processing_stats_for_render = params.audio_processing_stats.clone();
let wav_recorder_for_render = wav_recorder.clone();
// Diagnostic counters (sampled every 100 callbacks ~= 2 s).
let mut cb_count: u64 = 0;
@@ -841,7 +837,7 @@ impl IosVoiceUnit {
let _removed = h.fill_buffer(&mut scratch_stereo[..needed]);
}
Err(std::sync::TryLockError::WouldBlock) => {
audio_processing_stats.increment_callback_xrun();
audio_processing_stats_for_render.increment_callback_xrun();
// scratch_stereo is already zeroed above.
}
Err(std::sync::TryLockError::Poisoned(e)) => {
@@ -859,9 +855,9 @@ impl IosVoiceUnit {
muted,
);
if mix_stats.clipped_samples > 0 {
audio_processing_stats.add_clipped_samples(mix_stats.clipped_samples);
audio_processing_stats_for_render.add_clipped_samples(mix_stats.clipped_samples);
}
audio_processing_stats.update_render(
audio_processing_stats_for_render.update_render(
crate::frame::dbfs(&scratch_stereo[..needed]),
num_frames as u32,
);
@@ -895,7 +891,7 @@ impl IosVoiceUnit {
if mix_stats.peak_i16 > 0 {
callbacks_with_audio = callbacks_with_audio.wrapping_add(1);
} else {
audio_processing_stats.increment_output_underrun();
audio_processing_stats_for_render.increment_output_underrun();
callbacks_with_silence = callbacks_with_silence.wrapping_add(1);
}
crates/chanora_audio/src/lib.rs
+2 -1
View File
@@ -62,7 +62,8 @@ pub use audio_processing::{
AudioBackend, AudioProcessingConfig, AudioProcessingStats, AudioRoute, EffectOwner,
IosVoiceProcessingMode, SharedAudioProcessingStats, VadBackend,
};
pub use engine::{AudioEngine, AudioEngineConfig};
pub use engine::list_audio_devices;
pub use engine::{AudioDeviceInfo, AudioDeviceList, AudioEngine, AudioEngineConfig};
// SDD-120 §3 bench seam — `#[doc(hidden)]` re-export so the criterion
// bench harness under `crates/chanora_audio/benches/` can construct a
crates/chanora_audio/src/mobile_voice_backend.rs
+52 -13
View File
@@ -17,13 +17,16 @@
// so the engine can hold a single `Box<dyn MobileVoiceAudioBackend>`
// across iOS and Android.
#![allow(dead_code)]
use std::fmt;
use std::sync::atomic::{AtomicBool, AtomicU32};
use std::sync::{Arc, Mutex};
use tokio::sync::mpsc;
use tsclientlib::audio::AudioHandler;
use crate::engine::SessionAudioId;
use crate::AudioEffects;
use chanora_protocol::OutPacket;
/// Events the audio-callback thread or platform JNI listener can post
/// to the tokio side of the engine (SDD-115). Audio callbacks MUST
@@ -63,6 +66,36 @@ pub type BackendEventTx = mpsc::UnboundedSender<BackendEvent>;
/// against the active capture session.
pub type AudioSessionId = i32;
/// Engine-owned state shared with mobile voice audio callbacks.
#[derive(Clone)]
pub(crate) struct VoiceAudioParams {
/// Opus-encoded voice packets sent on this channel toward the
/// protocol layer.
pub voice_out_tx: mpsc::Sender<OutPacket>,
/// PTT transmission gate — true when the user holds the PTT key.
pub transmit_active: Arc<AtomicBool>,
/// Counter incremented per encoded frame sent.
pub frames_sent: Arc<AtomicU32>,
/// Pre-encode amplitude scale (1.0 = unity).
pub mic_gain: f32,
/// AudioHandler that inbound decode+mix feeds into; the output
/// callback pulls mixed stereo f32 from it.
pub handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
/// Master output gain (f32 bits stored in AtomicU32 for lock-free
/// cross-thread read from the realtime audio callback).
pub output_gain: Arc<AtomicU32>,
/// True = output silence regardless of incoming voice frames.
pub output_muted: Arc<AtomicBool>,
/// Optional TransmitModeSelector for VoiceActivity transmit mode.
/// The capture callback calls set_voice_activity_open on this when
/// VAD detects speech. None means VoiceActivity mode is disabled.
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
/// Shared audio-processing config (WebRTC APM flags, VAD backend).
pub audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
/// Shared audio-processing statistics for diagnostics.
pub audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
}
/// Achieved performance-mode reported by the platform after stream
/// open (SDD-112 item 4).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -734,12 +767,18 @@ mod tests {
/// open()/start()/stop()/close()).
#[test]
fn swe4_uv_047_backend_error_display() {
assert!(format!("{}", BackendError::OpenFailed("x".into())).contains("open failed"));
assert!(
format!("{}", BackendError::LifecycleFailed("y".into())).contains("lifecycle failed")
);
assert!(format!("{}", BackendError::ErrorDisconnected).contains("disconnected"));
assert!(format!("{}", BackendError::Platform("z".into())).contains("platform"));
assert!(BackendError::OpenFailed("x".into())
.to_string()
.contains("open failed"));
assert!(BackendError::LifecycleFailed("y".into())
.to_string()
.contains("lifecycle failed"));
assert!(BackendError::ErrorDisconnected
.to_string()
.contains("disconnected"));
assert!(BackendError::Platform("z".into())
.to_string()
.contains("platform"));
}
/// SWE4-UV-047: achieved enums Display stably (used in
@@ -748,15 +787,15 @@ mod tests {
#[test]
fn swe4_uv_047_achieved_enum_display_is_stable() {
assert_eq!(
format!("{}", AchievedPerformanceMode::LowLatency),
AchievedPerformanceMode::LowLatency.to_string(),
"LowLatency"
);
assert_eq!(
format!("{}", AchievedPerformanceMode::PowerSaving),
AchievedPerformanceMode::PowerSaving.to_string(),
"PowerSaving"
);
assert_eq!(format!("{}", AchievedPerformanceMode::None), "None");
assert_eq!(format!("{}", AchievedSharingMode::Exclusive), "Exclusive");
assert_eq!(format!("{}", AchievedSharingMode::Shared), "Shared");
assert_eq!(AchievedPerformanceMode::None.to_string(), "None");
assert_eq!(AchievedSharingMode::Exclusive.to_string(), "Exclusive");
assert_eq!(AchievedSharingMode::Shared.to_string(), "Shared");
}
}
crates/chanora_audio/src/sdl_output.rs
-9
View File
@@ -136,15 +136,6 @@ impl SdlOutput {
_subsystem: subsystem,
})
}
/// Pause the SDL device. Used by the engine on hard mute /
/// shutdown if we ever want to stop the callback firing while
/// keeping the device handle alive. Not currently invoked —
/// the engine drops `SdlOutput` entirely on stop.
#[allow(dead_code)]
pub fn pause(&self) {
self.device.pause();
}
}
impl Drop for SdlOutput {
crates/chanora_audio/src/vad/silero_onnx.rs
+9 -14
View File
@@ -76,6 +76,8 @@ pub struct SileroOnnxVad {
enum SileroInner {
Onnx(OnnxSession),
#[cfg(test)]
Stub,
}
struct OnnxSession {
@@ -179,9 +181,13 @@ impl SileroOnnxVad {
use ort::value::Value;
use tracing::error;
let SileroInner::Onnx(ref mut inner) = self.inner else {
return self.last_probability;
#[cfg(test)]
let inner = match self.inner {
SileroInner::Onnx(ref mut inner) => inner,
SileroInner::Stub => return self.last_probability,
};
#[cfg(not(test))]
let SileroInner::Onnx(ref mut inner) = self.inner;
debug_assert_eq!(audio_frame.len(), SILERO_FRAME_16K);
@@ -259,17 +265,6 @@ impl SileroOnnxVad {
}
}
#[cfg(target_os = "macos")]
pub(crate) fn bundled_onnxruntime_path_for_vad() -> Option<std::path::PathBuf> {
let exe = std::env::current_exe().ok()?;
let app_dir = exe.parent()?;
let framework = app_dir
.join("Frameworks")
.join("onnxruntime.framework")
.join("onnxruntime");
framework.exists().then_some(framework)
}
impl VoiceActivityDetector for SileroOnnxVad {
/// Accept one 10 ms **16 kHz** f32 mono frame (160 samples).
///
@@ -333,7 +328,7 @@ impl SileroOnnxVadWorker {
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(32);
let (tx, rx) = std::sync::mpsc::sync_channel::<SileroFrameMessage>(64);
let latest_probability_for_thread = latest_probability.clone();
let latest_processed_seq_for_thread = latest_processed_seq.clone();
let alive_for_thread = alive.clone();
crates/chanora_audio/src/vad/ten_onnx.rs
+52 -32
View File
@@ -8,6 +8,8 @@
use crate::frame::f32_to_i16;
use rustfft::{num_complex::Complex32, FftPlanner};
use super::resampler::{Downsampler48to16, INPUT_FRAME_10MS};
use super::{VadOutput, VoiceActivityDetector};
@@ -49,6 +51,8 @@ pub struct TenOnnxVad {
feature_stack: [[f32; FEATURE_LEN]; CONTEXT],
states: [[f32; HIDDEN]; 4],
mel_filters: Vec<[f32; N_BINS]>,
fft: std::sync::Arc<dyn rustfft::Fft<f32>>,
fft_buffer: Vec<Complex32>,
last_probability: f32,
last_speech: bool,
}
@@ -75,6 +79,8 @@ impl TenOnnxVad {
return None;
}
};
let mut fft_planner = FftPlanner::<f32>::new();
let fft = fft_planner.plan_fft_forward(FFT_SIZE);
tracing::info!(target: "chanora_audio", path = model_path, "TEN VAD ONNX model loaded");
Some(Self {
session,
@@ -84,6 +90,8 @@ impl TenOnnxVad {
feature_stack: [[0.0; FEATURE_LEN]; CONTEXT],
states: [[0.0; HIDDEN]; 4],
mel_filters: build_mel_filters(),
fft,
fft_buffer: vec![Complex32::ZERO; FFT_SIZE],
last_probability: 0.0,
last_speech: false,
})
@@ -104,7 +112,12 @@ impl TenOnnxVad {
self.sample_fifo.drain(..excess);
}
let feature = compute_feature(&self.mel_filters, &frame);
let feature = compute_feature(
&self.mel_filters,
self.fft.as_ref(),
&mut self.fft_buffer,
&frame,
);
self.feature_stack.copy_within(1..CONTEXT, 0);
self.feature_stack[CONTEXT - 1] = feature;
self.run_onnx();
@@ -169,19 +182,13 @@ impl TenOnnxVad {
}
}
#[cfg(target_os = "macos")]
fn bundled_onnxruntime_path() -> Option<std::path::PathBuf> {
let exe = std::env::current_exe().ok()?;
let app_dir = exe.parent()?;
let framework = app_dir
.join("Frameworks")
.join("onnxruntime.framework")
.join("onnxruntime");
framework.exists().then_some(framework)
}
fn compute_feature(mel_filters: &[[f32; N_BINS]], frame: &[f32]) -> [f32; FEATURE_LEN] {
let power = power_spectrum(frame);
fn compute_feature(
mel_filters: &[[f32; N_BINS]],
fft: &dyn rustfft::Fft<f32>,
fft_buffer: &mut [Complex32],
frame: &[f32],
) -> [f32; FEATURE_LEN] {
let power = power_spectrum(fft, fft_buffer, frame);
let mut feature = [0.0; FEATURE_LEN];
for band in 0..MEL_BANDS {
let energy = mel_filters[band]
@@ -245,33 +252,43 @@ fn build_mel_filters() -> Vec<[f32; N_BINS]> {
let left = bins[band];
let center = bins[band + 1].max(left + 1);
let right = bins[band + 2].max(center + 1).min(N_BINS - 1);
for i in left..center.min(N_BINS) {
filters[band][i] = (i - left) as f32 / (center - left) as f32;
for (i, weight) in filters[band]
.iter_mut()
.enumerate()
.take(center.min(N_BINS))
.skip(left)
{
*weight = (i - left) as f32 / (center - left) as f32;
}
for i in center..=right {
filters[band][i] = (right - i) as f32 / (right - center).max(1) as f32;
for (i, weight) in filters[band]
.iter_mut()
.enumerate()
.take(right + 1)
.skip(center)
{
*weight = (right - i) as f32 / (right - center).max(1) as f32;
}
}
filters
}
fn power_spectrum(frame: &[f32]) -> [f32; N_BINS] {
let mut windowed = [0.0_f32; FFT_SIZE];
fn power_spectrum(
fft: &dyn rustfft::Fft<f32>,
fft_buffer: &mut [Complex32],
frame: &[f32],
) -> [f32; N_BINS] {
debug_assert_eq!(fft_buffer.len(), FFT_SIZE);
fft_buffer.fill(Complex32::ZERO);
for (idx, sample) in frame.iter().take(WINDOW_16K).enumerate() {
let hann = 0.5 - 0.5 * (2.0 * std::f32::consts::PI * idx as f32 / WINDOW_16K as f32).cos();
windowed[idx] = f32_to_i16(*sample) as f32 * hann;
fft_buffer[idx].re = f32_to_i16(*sample) as f32 * hann;
}
fft.process(fft_buffer);
let mut out = [0.0_f32; N_BINS];
for (k, dst) in out.iter_mut().enumerate() {
let mut re = 0.0_f32;
let mut im = 0.0_f32;
for (n, &x) in windowed.iter().enumerate() {
let phase = -2.0 * std::f32::consts::PI * k as f32 * n as f32 / FFT_SIZE as f32;
re += x * phase.cos();
im += x * phase.sin();
}
*dst = re * re + im * im;
for (dst, bin) in out.iter_mut().zip(fft_buffer.iter()) {
*dst = bin.norm_sqr();
}
out
}
@@ -340,7 +357,7 @@ impl TenOnnxVadWorker {
let latest_probability = Arc::new(AtomicU32::new(0.0_f32.to_bits()));
let latest_processed_seq = Arc::new(AtomicU64::new(u64::MAX));
let alive = Arc::new(AtomicBool::new(true));
let (tx, rx) = std::sync::mpsc::sync_channel::<TenFrameMessage>(32);
let (tx, rx) = std::sync::mpsc::sync_channel::<TenFrameMessage>(128);
let prob_arc = latest_probability.clone();
let seq_arc = latest_processed_seq.clone();
let alive_arc = alive.clone();
@@ -425,8 +442,11 @@ mod tests {
#[test]
fn preprocessing_produces_finite_features() {
let filters = build_mel_filters();
let mut planner = FftPlanner::<f32>::new();
let fft = planner.plan_fft_forward(FFT_SIZE);
let mut fft_buffer = vec![Complex32::ZERO; FFT_SIZE];
let frame = vec![0.0_f32; WINDOW_16K];
let feature = compute_feature(&filters, &frame);
let feature = compute_feature(&filters, fft.as_ref(), &mut fft_buffer, &frame);
assert!(feature.iter().all(|v| v.is_finite()));
}
}