edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): add iOS VAD runtime support

Browse Source
This commit is contained in:
Edison Jwa
2026-05-21 20:51:45 +09:00
parent 171baf6e41
commit 6af4ecab0f
73 changed files with 11529 additions and 1249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/android_voice_unit.rs
+35 -74
View File
@@ -42,7 +42,6 @@ use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use audiopus::coder::Encoder as OpusEncoder;
use audiopus::{Application as OpusApp, Channels as OpusChannels, SampleRate as OpusSampleRate};
use tracing::{debug, info, warn};
use crate::mobile_voice_backend::{
@@ -53,7 +52,7 @@ use crate::mobile_voice_backend::{
BackendEventTx, EffectEngagement, EffectEngine, InputPresetChoice, MobileVoiceAudioBackend,
SharingModeChoice,
};
use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
use chanora_protocol::OutPacket;
use tsclientlib::audio::AudioHandler;
use crate::{engine::SessionAudioId, AudioError};
@@ -71,19 +70,11 @@ use oboe::{
// `mobile_voice_backend` so the trait can expose `take_event_rx`
// (SDD-111 item 1) cross-platform.
/// 20 ms at 48 kHz mono — one Opus frame's worth of samples.
/// Matches the iOS and desktop constants; duplicated here so this
/// module is fully self-contained and cfg-gate-clean.
const FRAME_SAMPLES: usize = 960;
/// Maximum size of an encoded Opus frame in bytes (RFC 6716 §3.2.1).
const MAX_OPUS_FRAME: usize = 1275;
// --- Capture state for Oboe input callback (SDD-111 / SDD-120) ----
//
// Mirrors the iOS `IosCaptureState` and the cpal-side `CaptureState`.
// Oboe delivers 48 kHz mono i16 PCM; we apply mic gain, accumulate to
// FRAME_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
// FRAME_20MS_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
// and try-send the resulting packet on `voice_out_tx`.
struct AndroidCaptureState {
@@ -91,7 +82,7 @@ struct AndroidCaptureState {
/// Accumulator for 48 kHz mono PCM. 2x capacity to absorb
/// cpal-style buffer-size jitter without reallocating.
pcm_accum: Vec<i16>,
opus_out: [u8; MAX_OPUS_FRAME],
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
@@ -105,33 +96,11 @@ impl AndroidCaptureState {
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
) -> Result<Self, AudioError> {
let mut encoder =
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.map_err(|e| AudioError::Opus(format!("encoder new (android): {e}")))?;
if let Err(e) = encoder.set_bitrate(audiopus::Bitrate::BitsPerSecond(32_000)) {
warn!(target: "chanora_audio", error = %e, "opus(android): set_bitrate(32000) failed");
}
if let Err(e) = encoder.set_complexity(10) {
warn!(target: "chanora_audio", error = %e, "opus(android): set_complexity(10) failed");
}
if let Err(e) = encoder.set_inband_fec(true) {
warn!(target: "chanora_audio", error = %e, "opus(android): set_inband_fec(true) failed");
}
if let Err(e) = encoder.set_packet_loss_perc(5) {
warn!(target: "chanora_audio", error = %e, "opus(android): set_packet_loss_perc(5) failed");
}
info!(
target: "chanora_audio",
bitrate_bps = 32_000,
complexity = 10,
inband_fec = true,
packet_loss_perc = 5,
"android Oboe opus encoder tuned for VoIP"
);
let encoder = crate::opus_voice::new_voip_encoder("android")?;
Ok(Self {
encoder,
pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2),
opus_out: [0u8; MAX_OPUS_FRAME],
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
transmit_active,
frames_sent,
@@ -139,7 +108,7 @@ impl AndroidCaptureState {
})
}
/// Consume i16 mono frames from Oboe, accumulate to FRAME_SAMPLES,
/// Consume i16 mono frames from Oboe, accumulate to FRAME_20MS_SAMPLES,
/// encode + send when PTT is held. Oboe delivers at the device's
/// native sample rate (always 48 kHz for modern Android per SRS-210),
/// so no resampling is needed.
@@ -159,28 +128,30 @@ impl AndroidCaptureState {
}));
}
// Drain complete 20 ms frames.
while self.pcm_accum.len() >= FRAME_SAMPLES {
let mut frame = [0i16; FRAME_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..FRAME_SAMPLES]);
self.pcm_accum.drain(..FRAME_SAMPLES);
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
codec: CodecType::OpusVoice,
data: &self.opus_out[..len],
});
match self.voice_out_tx.try_send(packet) {
Ok(()) => {
self.frames_sent.fetch_add(1, Ordering::Relaxed);
}
Err(mpsc::error::TrySendError::Full(_)) => {
warn!(target: "chanora_audio", "android Oboe: voice_out queue full; dropping frame");
}
Err(mpsc::error::TrySendError::Closed(_)) => {
debug!(target: "chanora_audio", "android Oboe: voice_out closed; capture pipeline stopping");
}
}
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.frames_sent,
&self.opus_out,
len,
|| {
warn!(
target: "chanora_audio",
"android Oboe: voice_out queue full; dropping frame"
);
},
|| {
debug!(
target: "chanora_audio",
"android Oboe: voice_out closed; capture pipeline stopping"
);
},
);
}
Err(e) => {
warn!(target: "chanora_audio", error = %e, "android Oboe opus encode failed");
@@ -266,22 +237,12 @@ impl AudioOutputCallback for OutputCallback {
}
let gain = f32::from_bits(self.output_gain.load(Ordering::Relaxed));
let muted = self.output_muted.load(Ordering::Relaxed);
let mut peak: i16 = 0;
for (i, dst) in frames.iter_mut().enumerate() {
if muted {
*dst = 0;
continue;
}
let l = scratch[i * 2];
let r = scratch[i * 2 + 1];
let mono = (l + r) * 0.5 * gain;
let clamped = mono.clamp(-1.0, 1.0);
let sample = (clamped * i16::MAX as f32) as i16;
*dst = sample;
if sample.unsigned_abs() > peak.unsigned_abs() {
peak = sample;
}
}
let _ = crate::voice_render::downmix_stereo_f32_to_mono_i16(
&scratch[..needed],
frames,
gain,
muted,
);
}));
DataCallbackResult::Continue
}