edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): add iOS VAD runtime support

Browse Source
This commit is contained in:
Edison Jwa
2026-05-21 20:51:45 +09:00
parent 171baf6e41
commit 6af4ecab0f
73 changed files with 11529 additions and 1249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/engine.rs
+233 -136
View File
@@ -33,44 +33,21 @@ use tracing::{debug, info};
))]
use tracing::{error, warn};
#[cfg(target_os = "android")]
#[cfg(any(target_os = "ios", target_os = "android"))]
use tracing::warn;
use tsclientlib::audio::AudioHandler;
use chanora_protocol::{InboundVoice, OutPacket};
use crate::AudioError;
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
use audiopus::coder::Encoder as OpusEncoder;
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
use audiopus::{
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
SampleRate as OpusSampleRate,
};
use tsclientlib::audio::AudioHandler;
// `AudioData`, `CodecType`, `OutAudio` are referenced only by the
// cpal capture pipeline's Opus encode path (`CaptureState::encode_and_send`).
// `InboundVoice` + `OutPacket` are used by every platform — the
// inbound forwarder task pumps `InboundVoice` into AudioHandler on
// iOS too, and `OutPacket` flows out of the capture pipeline once
// commit 3 lands. Cfg-gate the cpal-only ones to keep iOS warnings
// clean.
#[cfg(all(
not(target_os = "ios"),
not(target_os = "macos"),
not(target_os = "android")
))]
use chanora_protocol::{AudioData, CodecType, OutAudio};
use chanora_protocol::{InboundVoice, OutPacket};
use crate::AudioError;
/// Stable Chanora-side identifier for AudioHandler bookkeeping.
/// We only ever have one connection at a time (DEC-006), so this is
/// trivially unique.
@@ -89,11 +66,8 @@ pub struct SessionAudioId(pub u64);
const SAMPLE_RATE: u32 = 48_000;
#[allow(dead_code)]
const FRAME_SAMPLES: usize = 48_000 / 50; // 960
#[allow(dead_code)]
const MAX_OPUS_FRAME: usize = 1275;
/// Engine configuration.
#[derive(Debug, Clone)]
#[derive(Clone)]
pub struct AudioEngineConfig {
/// Input gain applied before encoding (1.0 = pass-through).
pub mic_gain: f32,
@@ -117,6 +91,24 @@ pub struct AudioEngineConfig {
/// is rejected on Android because the P0 path intentionally has
/// no generic mobile-audio fallback.
pub mobile_voice_preset: bool,
/// Optional selector used by P1 VoiceActivity to publish VAD state.
#[doc(hidden)]
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
}
impl std::fmt::Debug for AudioEngineConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AudioEngineConfig")
.field("mic_gain", &self.mic_gain)
.field("ptt_initial", &self.ptt_initial)
.field("effects", &self.effects)
.field("mobile_voice_preset", &self.mobile_voice_preset)
.field(
"voice_activity_selector",
&self.voice_activity_selector.as_ref().map(|_| "present"),
)
.finish()
}
}
impl Default for AudioEngineConfig {
@@ -126,6 +118,7 @@ impl Default for AudioEngineConfig {
ptt_initial: false,
effects: crate::AudioEffects::default(),
mobile_voice_preset: true,
voice_activity_selector: None,
}
}
}
@@ -150,6 +143,16 @@ pub struct AudioEngine {
/// independent of the server-side mute the protocol layer
/// broadcasts.
output_muted: Arc<AtomicBool>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
voice_out_tx: mpsc::Sender<OutPacket>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
mic_gain: f32,
// Streams must be dropped to stop audio. Both are `!Send` because
// cpal's Stream isn't Send on some backends; we keep them in an
@@ -177,7 +180,7 @@ pub struct AudioEngine {
))]
_output_stream: Mutex<Option<cpal::Stream>>,
#[cfg(any(target_os = "ios", target_os = "macos"))]
_ios_voice_unit: Mutex<Option<crate::ios_voice_unit::IosVoiceUnit>>,
_ios_voice_backend: Mutex<Option<IosVoiceBackend>>,
/// SDD-111..SDD-115: Android Oboe voice backend. Owns the input
/// and output streams, SDD-113 hardware-effect handles, and the
/// foreground-service lifecycle; tearing it down on engine drop
@@ -237,6 +240,105 @@ pub struct AudioEngine {
unsafe impl Send for AudioEngine {}
unsafe impl Sync for AudioEngine {}
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[allow(dead_code)]
enum IosVoiceBackend {
Vpio(crate::ios_voice_unit::IosVoiceUnit),
#[cfg(target_os = "ios")]
Raw(crate::ios_raw_unit::IosRawUnit),
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
impl IosVoiceBackend {
fn pause(&mut self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
{
match self {
Self::Vpio(unit) => unit.pause(),
Self::Raw(unit) => unit.pause(),
}
}
#[cfg(target_os = "macos")]
{
Ok(())
}
}
fn resume(&mut self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
{
match self {
Self::Vpio(unit) => unit.resume(),
Self::Raw(unit) => unit.resume(),
}
}
#[cfg(target_os = "macos")]
{
Ok(())
}
}
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[allow(clippy::too_many_arguments)]
fn open_ios_voice_backend(
handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_flag_for_capture: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<IosVoiceBackend, AudioError> {
let _cfg = audio_processing_config.lock().unwrap().clone();
#[cfg(target_os = "ios")]
{
if _cfg.ios_mode == crate::IosVoiceProcessingMode::SonoraExperimental {
match crate::ios_raw_unit::IosRawUnit::start(
handler.clone(),
output_gain.clone(),
output_muted.clone(),
voice_out_tx.clone(),
transmit_flag_for_capture.clone(),
frames_sent.clone(),
mic_gain,
voice_activity_selector.clone(),
audio_processing_config.clone(),
audio_processing_stats.clone(),
) {
Ok(unit) => {
info!(target: "chanora_audio", "ios: RemoteIO/Sonora experimental backend selected");
return Ok(IosVoiceBackend::Raw(unit));
}
Err(e) => {
warn!(
target: "chanora_audio",
error = %e,
"ios: RemoteIO/Sonora backend failed; falling back to VoiceProcessingIO"
);
}
}
}
}
let unit = crate::ios_voice_unit::IosVoiceUnit::start(
handler,
output_gain,
output_muted,
voice_out_tx,
transmit_flag_for_capture,
frames_sent,
mic_gain,
voice_activity_selector,
audio_processing_config,
audio_processing_stats,
)?;
Ok(IosVoiceBackend::Vpio(unit))
}
impl AudioEngine {
/// Start the engine: open capture + playback streams, spawn the
/// inbound-voice forwarder, return a handle.
@@ -259,6 +361,7 @@ impl AudioEngine {
voice_in_rx: mpsc::Receiver<InboundVoice>,
transmit_gate: crate::ptt::AudioTransmitGate,
) -> Result<Self, AudioError> {
#[allow(clippy::needless_return)]
// Apple platforms route to a separate backend (VoiceProcessingIO
// via coreaudio-rs) because cpal does not expose the native
// voice-processing AudioUnit controls Chanora needs for VoIP.
@@ -353,6 +456,8 @@ impl AudioEngine {
let frames_received = Arc::new(AtomicU32::new(0));
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
let output_muted = Arc::new(AtomicBool::new(false));
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
// ---------- Capture ----------
// Capture is best-effort. If the platform default input
@@ -525,6 +630,8 @@ impl AudioEngine {
frames_received,
output_gain,
output_muted,
audio_processing_config,
audio_processing_stats,
_input_stream: Mutex::new(input_stream),
_output_stream: Mutex::new(Some(output_stream)),
shutdown_tx: Some(shutdown_tx),
@@ -549,6 +656,8 @@ impl AudioEngine {
let frames_received = Arc::new(AtomicU32::new(0));
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
let output_muted = Arc::new(AtomicBool::new(false));
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
Arc::new(Mutex::new(AudioHandler::new()));
@@ -678,6 +787,8 @@ impl AudioEngine {
frames_received,
output_gain,
output_muted,
audio_processing_config,
audio_processing_stats,
_android_voice_unit: Mutex::new(Some(android_voice_unit)),
audio_mode_stack: Mutex::new(audio_mode_stack),
shutdown_tx: Some(shutdown_tx),
@@ -725,23 +836,26 @@ impl AudioEngine {
let frames_received = Arc::new(AtomicU32::new(0));
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
let output_muted = Arc::new(AtomicBool::new(false));
let audio_processing_config = Arc::new(Mutex::new(crate::AudioProcessingConfig::default()));
let audio_processing_stats = Arc::new(crate::SharedAudioProcessingStats::default());
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
Arc::new(Mutex::new(AudioHandler::new()));
let voice_out_tx_for_backend = voice_out_tx.clone();
// Construct the VPIO unit. Commit 1 ships a no-op callback
// pair; commits 3 + 4 land the real capture + playback
// wiring. Construction failure here is fatal (mirrors how
// the cpal output-stream construction failure is fatal in
// the non-iOS path).
let ios_voice_unit = crate::ios_voice_unit::IosVoiceUnit::start(
// Construct the live iOS voice backend. Platform VPIO stays
// the default shipping path; Sonora/RemoteIO remains opt-in.
let ios_voice_backend = open_ios_voice_backend(
audio_handler.clone(),
output_gain.clone(),
output_muted.clone(),
voice_out_tx,
voice_out_tx_for_backend,
transmit_flag_for_capture,
frames_sent.clone(),
cfg.mic_gain,
cfg.voice_activity_selector.clone(),
audio_processing_config.clone(),
audio_processing_stats.clone(),
)?;
// Capture is always considered active on iOS — VPIO's
@@ -791,7 +905,13 @@ impl AudioEngine {
frames_received,
output_gain,
output_muted,
_ios_voice_unit: Mutex::new(Some(ios_voice_unit)),
audio_processing_config,
audio_processing_stats,
audio_handler,
voice_out_tx,
voice_activity_selector: cfg.voice_activity_selector.clone(),
mic_gain: cfg.mic_gain,
_ios_voice_backend: Mutex::new(Some(ios_voice_backend)),
shutdown_tx: Some(shutdown_tx),
capture_active,
ptt_watchdog,
@@ -826,7 +946,7 @@ impl AudioEngine {
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let _ = self._ios_voice_unit.lock().unwrap().take();
let _ = self._ios_voice_backend.lock().unwrap().take();
}
// SDD-115 reverse-order teardown on Android:
// 1) close the voice unit (releases SDD-113 hardware
@@ -900,15 +1020,25 @@ impl AudioEngine {
/// iOS-only: restart the underlying VoiceProcessingIO unit after
/// route changes.
pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let mut guard = self._ios_voice_unit.lock().unwrap();
let unit = guard
.as_mut()
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
return unit.restart();
let backend = open_ios_voice_backend(
self.audio_handler.clone(),
self.output_gain.clone(),
self.output_muted.clone(),
self.voice_out_tx.clone(),
self.transmit_gate.flag_arc(),
self.frames_sent.clone(),
self.mic_gain,
self.voice_activity_selector.clone(),
self.audio_processing_config.clone(),
self.audio_processing_stats.clone(),
)?;
let mut guard = self._ios_voice_backend.lock().unwrap();
*guard = Some(backend);
Ok(())
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
Ok(())
}
@@ -916,15 +1046,15 @@ impl AudioEngine {
/// iOS-only: pause the underlying VoiceProcessingIO unit.
pub fn ios_pause_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let mut guard = self._ios_voice_unit.lock().unwrap();
let mut guard = self._ios_voice_backend.lock().unwrap();
let unit = guard
.as_mut()
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
return unit.pause();
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
unit.pause()
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
Ok(())
}
@@ -932,15 +1062,15 @@ impl AudioEngine {
/// iOS-only: resume the underlying VoiceProcessingIO unit.
pub fn ios_resume_voice_unit(&self) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
#[cfg(any(target_os = "ios", target_os = "macos"))]
{
let mut guard = self._ios_voice_unit.lock().unwrap();
let mut guard = self._ios_voice_backend.lock().unwrap();
let unit = guard
.as_mut()
.ok_or_else(|| AudioError::Backend("ios voice unit not running".to_string()))?;
return unit.resume();
.ok_or_else(|| AudioError::Backend("ios voice backend not running".to_string()))?;
unit.resume()
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "macos")))]
{
Ok(())
}
@@ -1014,6 +1144,29 @@ impl AudioEngine {
self.frames_received.load(Ordering::Relaxed)
}
/// Current audio-processing config snapshot.
pub fn audio_processing_config_snapshot(&self) -> crate::AudioProcessingConfig {
self.audio_processing_config.lock().unwrap().clone()
}
/// Apply a voice-processing config after validating iOS invariants.
pub fn set_audio_processing_config(
&self,
config: crate::AudioProcessingConfig,
) -> Result<(), AudioError> {
#[cfg(target_os = "ios")]
config.validate_for_ios()?;
let mut guard = self.audio_processing_config.lock().unwrap();
*guard = config;
Ok(())
}
/// Current voice-processing stats snapshot.
pub fn audio_processing_stats(&self) -> crate::AudioProcessingStats {
let config = self.audio_processing_config.lock().unwrap().clone();
self.audio_processing_stats.snapshot(&config)
}
/// Latest Android voice-audio diagnostics snapshot (SDD-112 item
/// 10 / SDD-113 item 7 / SDD-116 item 3). On non-Android targets
/// this always returns `None`. On Android it returns `Some(...)`
@@ -1091,58 +1244,7 @@ fn try_open_capture(
in_stream_cfg.buffer_size = cpal::BufferSize::Default;
}
let mut opus_enc = OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.map_err(|e| AudioError::Opus(format!("encoder new: {e}")))?;
// Opus VOIP tuning. Defaults give us 'auto' bitrate (can drop
// to ~6 kbps during silence \u2014 which sounds garbled when
// talking resumes) and inband FEC disabled. On lossy mobile
// networks (cellular / iPhone WiFi roaming), packet loss
// without FEC produces audible clicks + cut-out frames.
//
// Settings derived from the Opus IETF VoIP recommendations
// (RFC 6716 \u00a7 7.1) and Discord's voice client tuning:
//
// * Bitrate 32 kbps : sweet spot for mono voice. Lower
// than 24 kbps starts to sound watery; higher than
// 64 kbps wastes bandwidth without perceptual gain on a
// human voice. Discord uses 64 kbps; mumble defaults to
// 40 kbps; we pick 32 kbps as a conservative VoIP value
// that survives 100 kbps uplinks comfortably.
// * Complexity 10 : max quality. The CPU cost on a modern
// iPhone (A14+) or any desktop is negligible (~0.5 % of
// a single core for 48 kHz mono).
// * Inband FEC on : opus inserts a low-bitrate redundancy
// copy of the previous frame inside the current packet
// so a single dropped packet can be reconstructed from
// the next one. Essential on lossy mobile.
// * Packet loss perc 5 % : tells the encoder to expect 5 %
// loss and pre-emptively budget bits for FEC. Higher
// values trade audio quality for resilience.
//
// Errors here are non-fatal: log + continue. The encoder
// works with defaults if any setter fails on an exotic
// libopus build.
if let Err(e) = opus_enc.set_bitrate(OpusBitrate::BitsPerSecond(32_000)) {
warn!(target: "chanora_audio", error = %e, "opus: set_bitrate(32000) failed");
}
if let Err(e) = opus_enc.set_complexity(10) {
warn!(target: "chanora_audio", error = %e, "opus: set_complexity(10) failed");
}
if let Err(e) = opus_enc.set_inband_fec(true) {
warn!(target: "chanora_audio", error = %e, "opus: set_inband_fec(true) failed");
}
if let Err(e) = opus_enc.set_packet_loss_perc(5) {
warn!(target: "chanora_audio", error = %e, "opus: set_packet_loss_perc(5) failed");
}
info!(
target: "chanora_audio",
bitrate_bps = 32_000,
complexity = 10,
inband_fec = true,
packet_loss_perc = 5,
"opus encoder tuned for VoIP"
);
let opus_enc = crate::opus_voice::new_voip_encoder("cpal capture")?;
let capture_state = Arc::new(Mutex::new(CaptureState::new(
opus_enc,
@@ -1184,7 +1286,7 @@ struct CaptureState {
/// roughly at the period rate (~100 Hz for a 10 ms period on
/// Linux ALSA defaults).
resample_last: f32,
opus_out: [u8; MAX_OPUS_FRAME],
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
/// The PTT transmission gate. Read once per outbound frame; the
/// CaptureState never mutates this flag.
@@ -1224,7 +1326,7 @@ impl CaptureState {
pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2),
resample_pos: 0.0,
resample_last: 0.0,
opus_out: [0u8; MAX_OPUS_FRAME],
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
frames_sent,
@@ -1310,22 +1412,21 @@ impl CaptureState {
.encode_float(&frame[..], &mut self.opus_out[..])
{
Ok(len) => {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &self.opus_out[..len],
});
match self.voice_out_tx.try_send(packet) {
Ok(()) => {
self.frames_sent.fetch_add(1, Ordering::Relaxed);
}
Err(mpsc::error::TrySendError::Full(_)) => {
warn!(target: "chanora_audio", "voice_out queue full; dropping frame");
}
Err(mpsc::error::TrySendError::Closed(_)) => {
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.frames_sent,
&self.opus_out,
len,
|| {
warn!(
target: "chanora_audio",
"voice_out queue full; dropping frame"
);
},
|| {
warn!(target: "chanora_audio", "voice_out closed; stopping send");
}
}
},
);
}
Err(e) => {
error!(target: "chanora_audio", error = %e, "opus encode failed");
@@ -1851,10 +1952,7 @@ pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> {
#[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
#[doc(hidden)]
pub mod bench_seam {
use super::{
Arc, AtomicBool, AtomicU32, CaptureState, OpusApp, OpusChannels, OpusEncoder,
OpusSampleRate, OutPacket,
};
use super::{Arc, AtomicBool, AtomicU32, CaptureState, OpusEncoder, OutPacket};
use tokio::sync::mpsc;
/// Opaque handle wrapping a CaptureState plus the dummy mpsc
@@ -1880,8 +1978,7 @@ pub mod bench_seam {
/// (typically 1 or 2).
pub fn new(in_sample_rate: u32, in_channels: usize) -> Self {
let encoder =
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.expect("opus encoder init");
crate::opus_voice::new_voip_encoder("cpal bench").expect("opus encoder init");
let (tx, rx) = mpsc::channel::<OutPacket>(64);
let transmit_active = Arc::new(AtomicBool::new(true));
let frames_sent = Arc::new(AtomicU32::new(0));