edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(audio): harden realtime callback paths

Browse Source
This commit is contained in:
Edison Jwa
2026-06-08 19:40:03 +09:00
parent d83539436e
commit 8606eb48c8
12 changed files with 1372 additions and 528 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/android_voice_unit.rs
+153 -143
View File
@@ -53,7 +53,6 @@ use crate::mobile_voice_backend::{
BackendEventTx, EffectEngagement, EffectEngine, InputPresetChoice, MobileVoiceAudioBackend,
SharingModeChoice, VoiceAudioParams,
};
use chanora_protocol::OutPacket;
use tsclientlib::audio::AudioHandler;
use crate::{engine::SessionAudioId, AudioError};
@@ -86,40 +85,11 @@ use crate::processor::AudioProcessor;
const RENDER_REF_SLOTS: usize = 4;
const RENDER_REF_SAMPLES: usize = crate::frame::FRAME_10MS_SAMPLES;
const ANDROID_RENDER_PULL_SAMPLES: usize = crate::frame::FRAME_20MS_SAMPLES * 2;
const ANDROID_RENDER_RING_CAPACITY: usize = ANDROID_RENDER_PULL_SAMPLES * 5;
struct RenderReferenceBuffer {
buf: Box<[[f32; RENDER_REF_SAMPLES]; RENDER_REF_SLOTS]>,
write_idx: std::sync::atomic::AtomicUsize,
}
impl RenderReferenceBuffer {
fn new() -> Arc<Self> {
Arc::new(Self {
buf: Box::new([[0.0_f32; RENDER_REF_SAMPLES]; RENDER_REF_SLOTS]),
write_idx: std::sync::atomic::AtomicUsize::new(0),
})
}
fn write(&self, frame: &[f32; RENDER_REF_SAMPLES]) {
let idx = self.write_idx.load(Ordering::Relaxed);
unsafe {
let slot = &self.buf[idx] as *const [f32; RENDER_REF_SAMPLES]
as *mut [f32; RENDER_REF_SAMPLES];
(*slot).copy_from_slice(frame);
}
self.write_idx
.store((idx + 1) % RENDER_REF_SLOTS, Ordering::Relaxed);
}
fn read_latest(&self) -> [f32; RENDER_REF_SAMPLES] {
let wi = self.write_idx.load(Ordering::Relaxed);
let ri = (wi + RENDER_REF_SLOTS - 1) % RENDER_REF_SLOTS;
self.buf[ri]
}
}
unsafe impl Send for RenderReferenceBuffer {}
unsafe impl Sync for RenderReferenceBuffer {}
type RenderReferenceBuffer =
crate::render_reference::RenderReferenceBuffer<RENDER_REF_SAMPLES, RENDER_REF_SLOTS>;
// --- Capture state for Oboe input callback (SDD-111 / SDD-120) ----
//
@@ -138,9 +108,8 @@ struct AndroidCaptureState {
encoder: OpusEncoder,
pcm_accum: Vec<i16>,
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx: mpsc::Sender<OutPacket>,
voice_out_tx: crate::opus_voice::EncodedVoiceFrameSender,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
vad_detector: crate::vad::WebRtcFallbackVad,
@@ -164,7 +133,7 @@ struct AndroidCaptureState {
impl AndroidCaptureState {
fn new(
voice_out_tx: mpsc::Sender<OutPacket>,
voice_out_tx: mpsc::Sender<chanora_protocol::OutPacket>,
transmit_active: Arc<AtomicBool>,
frames_sent: Arc<AtomicU32>,
mic_gain: f32,
@@ -188,9 +157,12 @@ impl AndroidCaptureState {
encoder,
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
voice_out_tx,
voice_out_tx: crate::opus_voice::start_out_packet_worker(
voice_out_tx,
frames_sent.clone(),
"android",
)?,
transmit_active,
frames_sent,
mic_gain,
voice_activity_selector,
vad_detector: crate::vad::WebRtcFallbackVad::default(),
@@ -221,8 +193,10 @@ impl AndroidCaptureState {
self.audio_processing_stats
.record_callback_frames(samples.len() as u64);
if self.input_sample_rate_hz != crate::frame::SAMPLE_RATE_HZ {
let resampled = self.resample_capture_to_48k(samples);
self.resample_capture_to_48k(samples);
let resampled = std::mem::take(&mut self.resample_scratch);
self.ingest_48k_i16(&resampled);
self.resample_scratch = resampled;
return;
}
self.ingest_48k_i16(samples);
@@ -241,6 +215,7 @@ impl AndroidCaptureState {
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
let frame = self.pending_10ms;
self.process_10ms_capture_frame(&frame);
self.encode_complete_20ms_frames();
self.pending_10ms_len = 0;
}
}
@@ -250,6 +225,10 @@ impl AndroidCaptureState {
return;
}
self.encode_complete_20ms_frames();
}
fn encode_complete_20ms_frames(&mut self) {
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
@@ -258,7 +237,6 @@ impl AndroidCaptureState {
Ok(len) => {
crate::opus_voice::send_voip_frame(
&self.voice_out_tx,
&self.frames_sent,
&self.opus_out,
len,
|| {
@@ -286,35 +264,18 @@ impl AndroidCaptureState {
}
}
fn resample_capture_to_48k(&mut self, samples: &[i16]) -> Vec<i16> {
if samples.is_empty() {
return Vec::new();
fn resample_capture_to_48k(&mut self, samples: &[i16]) -> usize {
let result = crate::capture_resampler::resample_capture_to_48k(
samples,
self.input_sample_rate_hz,
&mut self.resample_pos,
&mut self.resample_last,
&mut self.resample_scratch,
);
if result.dropped {
self.audio_processing_stats.increment_callback_xrun();
}
self.resample_scratch.clear();
let ratio = self.input_sample_rate_hz as f64 / crate::frame::SAMPLE_RATE_HZ as f64;
let mut pos = self.resample_pos;
while pos < samples.len() as f64 {
let i = pos.floor() as isize;
let frac = pos - i as f64;
let a = if i <= 0 {
self.resample_last as f64
} else {
samples[(i - 1) as usize] as f64
};
let b = if i < samples.len() as isize {
samples[i as usize] as f64
} else {
a
};
let value = (a + frac * (b - a))
.round()
.clamp(i16::MIN as f64, i16::MAX as f64) as i16;
self.resample_scratch.push(value);
pos += ratio;
}
self.resample_pos = pos - samples.len() as f64;
self.resample_last = *samples.last().unwrap_or(&self.resample_last);
self.resample_scratch.clone()
result.output_len
}
fn set_input_sample_rate_hz(&mut self, sample_rate_hz: u32) {
@@ -393,15 +354,9 @@ impl AndroidCaptureState {
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
let path = crate::vad::silero_model_bundle_path();
self.silero_vad_worker =
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
if self.silero_vad_worker.is_none() {
warn!(
target: "chanora_audio",
"android: Silero VAD model not found at {path}; falling back to WebRTC VAD"
);
}
self.silero_vad_worker = None;
self.mark_vad_fallback_active(crate::VadBackend::SileroOnnx);
self.audio_processing_stats.set_vad_fallback_active(true);
}
_ => {
self.silero_vad_worker = None;
@@ -420,20 +375,38 @@ impl AndroidCaptureState {
speech: true,
}
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
match crate::vad::callback_vad_worker_policy(
voice_activity_mode,
vad_backend,
self.silero_vad_worker.is_some(),
) {
crate::vad::VadWorkerPolicy::UseWorker => {
let worker = self
.silero_vad_worker
.as_ref()
.expect("policy checked worker");
let enqueued = worker.try_send(capture_seq, &frame);
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
}
crate::vad::VadWorkerPolicy::UseFallback => {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
@@ -441,10 +414,10 @@ impl AndroidCaptureState {
&frame,
)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
crate::vad::VadWorkerPolicy::NotModelBacked => crate::vad::VadOutput {
probability: 1.0,
speech: true,
},
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
@@ -472,21 +445,19 @@ impl AndroidCaptureState {
return;
}
let gain = self.mic_gain;
if (gain - 1.0).abs() < f32::EPSILON {
self.pcm_accum
.extend(frame.iter().copied().map(crate::frame::f32_to_i16));
} else {
self.pcm_accum.extend(frame.iter().copied().map(|s| {
let scaled = (crate::frame::f32_to_i16(s) as f32) * gain;
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
}));
if crate::capture_accumulator::append_processed_i16_bounded(
&mut self.pcm_accum,
&frame,
self.mic_gain,
) {
self.audio_processing_stats.increment_callback_xrun();
}
}
}
struct InputCallback {
state: Arc<Mutex<AndroidCaptureState>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
event_tx: BackendEventTx,
}
@@ -498,9 +469,13 @@ impl AudioInputCallback for InputCallback {
_stream: &mut dyn AudioInputStreamSafe,
frames: &[i16],
) -> DataCallbackResult {
let _ = catch_unwind(AssertUnwindSafe(|| {
if let Ok(mut s) = self.state.lock() {
s.ingest_i16(frames);
let _ = catch_unwind(AssertUnwindSafe(|| match self.state.try_lock() {
Ok(mut s) => s.ingest_i16(frames),
Err(std::sync::TryLockError::WouldBlock) => {
self.audio_processing_stats.increment_callback_xrun();
}
Err(std::sync::TryLockError::Poisoned(e)) => {
warn!(target: "chanora_audio", "android: capture state poisoned: {e}");
}
}));
DataCallbackResult::Continue
@@ -522,8 +497,7 @@ impl AudioInputCallback for InputCallback {
// writes stereo f32 directly to the Oboe output buffer.
struct OutputCallback {
handler: AudioHandler<SessionAudioId>,
event_consumer: crate::audio_event_queue::AudioEventConsumer,
pcm_consumer: crate::android_render_ring::AndroidRenderRingConsumer,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
event_tx: BackendEventTx,
@@ -542,31 +516,8 @@ impl AudioOutputCallback for OutputCallback {
frames: &mut [(f32, f32)],
) -> DataCallbackResult {
let _ = catch_unwind(AssertUnwindSafe(|| {
let buf: &mut [f32] =
bytemuck::cast_slice_mut::<(f32, f32), f32>(frames);
for s in buf.iter_mut() {
*s = 0.0;
}
for cmd in self.event_consumer.drain_controls() {
match cmd {
AudioCommand::SetVolume(id, vol) => {
if let Some(q) = self.handler.get_mut_queues().get_mut(&id) {
q.volume = vol;
}
}
AudioCommand::RemoveClient(id) => {
self.handler.get_mut_queues().remove(&id);
}
}
}
for pkt in self.event_consumer.drain_packets(50) {
if let Err(e) = self.handler.handle_packet(pkt.client_id, pkt.data) {
debug!(target: "chanora_audio", error = %e, "decode failed");
}
}
let _ = self.handler.fill_buffer(buf);
let buf: &mut [f32] = bytemuck::cast_slice_mut::<(f32, f32), f32>(frames);
self.pcm_consumer.drain_into_zero_filling(buf);
let gain = f32::from_bits(self.output_gain.load(Ordering::Relaxed));
let muted = self.output_muted.load(Ordering::Relaxed);
if muted {
@@ -614,6 +565,7 @@ impl AudioOutputCallback for OutputCallback {
pub struct AndroidVoiceUnit {
input: Option<AudioStreamAsync<OboeInput, InputCallback>>,
output: Option<AudioStreamAsync<OboeOutput, OutputCallback>>,
render_producer_shutdown: Arc<AtomicBool>,
// Recorded achieved values (SDD-112).
input_perf: AchievedPerformanceMode,
@@ -706,6 +658,7 @@ impl AndroidVoiceUnit {
let input_cb = InputCallback {
state: capture_state.clone(),
audio_processing_stats: audio_processing_stats.clone(),
event_tx: event_tx.clone(),
};
let input_builder = input_builder.set_callback(input_cb);
@@ -722,7 +675,12 @@ impl AndroidVoiceUnit {
error = ?e,
"android: primary input stream open failed; entering fallback ladder"
);
match Self::open_input_fallback(cfg, &event_tx, capture_state.clone()) {
match Self::open_input_fallback(
cfg,
&event_tx,
capture_state.clone(),
audio_processing_stats.clone(),
) {
Ok(s) => Some(s),
Err(fallback_err) => {
warn!(
@@ -789,9 +747,10 @@ impl AndroidVoiceUnit {
let render_ref_for_output = render_ref_buf.clone();
let event_queue = params.event_producer.queue();
let render_ring =
crate::android_render_ring::AndroidRenderRing::new(ANDROID_RENDER_RING_CAPACITY);
let output_cb = OutputCallback {
handler: params.handler,
event_consumer: AudioEventQueue::consumer(&event_queue),
pcm_consumer: render_ring.consumer(),
output_gain: params.output_gain.clone(),
output_muted: params.output_muted.clone(),
event_tx: event_tx.clone(),
@@ -813,8 +772,7 @@ impl AndroidVoiceUnit {
Self::open_output_fallback(
cfg,
&event_tx,
AudioHandler::new(),
AudioEventQueue::consumer(&event_queue),
render_ring.consumer(),
params.output_gain.clone(),
params.output_muted.clone(),
audio_processing_stats.clone(),
@@ -822,6 +780,11 @@ impl AndroidVoiceUnit {
)?
}
};
let render_producer_shutdown = Self::spawn_render_producer(
params.handler,
AudioEventQueue::consumer(&event_queue),
render_ring.producer(),
);
let output_frames_per_burst = output_stream.get_frames_per_burst();
if output_frames_per_burst > 0 {
@@ -978,6 +941,7 @@ impl AndroidVoiceUnit {
Ok(Self {
input: input_stream,
output: Some(output_stream),
render_producer_shutdown,
input_perf,
input_share,
output_perf,
@@ -995,6 +959,7 @@ impl AndroidVoiceUnit {
cfg: &AndroidVoiceStreamConfig,
event_tx: &BackendEventTx,
capture_state: Arc<Mutex<AndroidCaptureState>>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
) -> Result<AudioStreamAsync<OboeInput, InputCallback>, BackendError> {
// SDD-112 items 6 & 7: explore (preset × sharing) independently
// via the pure helpers in `mobile_voice_backend`. Primary
@@ -1031,6 +996,7 @@ impl AndroidVoiceUnit {
};
let cb = InputCallback {
state: capture_state.clone(),
audio_processing_stats: audio_processing_stats.clone(),
event_tx: event_tx.clone(),
};
let builder = AudioStreamBuilder::default()
@@ -1066,16 +1032,14 @@ impl AndroidVoiceUnit {
fn open_output_fallback(
cfg: &AndroidVoiceStreamConfig,
event_tx: &BackendEventTx,
handler: AudioHandler<SessionAudioId>,
event_consumer: crate::audio_event_queue::AudioEventConsumer,
pcm_consumer: crate::android_render_ring::AndroidRenderRingConsumer,
output_gain: Arc<AtomicU32>,
output_muted: Arc<AtomicBool>,
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
render_reference: Arc<RenderReferenceBuffer>,
) -> Result<AudioStreamAsync<OboeOutput, OutputCallback>, BackendError> {
let cb = OutputCallback {
handler,
event_consumer,
pcm_consumer,
output_gain,
output_muted,
event_tx: event_tx.clone(),
@@ -1100,6 +1064,50 @@ impl AndroidVoiceUnit {
.map_err(|e| BackendError::OpenFailed(format!("output fallback: {e:?}")))
}
fn spawn_render_producer(
mut handler: AudioHandler<SessionAudioId>,
event_consumer: crate::audio_event_queue::AudioEventConsumer,
pcm_producer: crate::android_render_ring::AndroidRenderRingProducer,
) -> Arc<AtomicBool> {
let shutdown = Arc::new(AtomicBool::new(false));
let shutdown_for_task = shutdown.clone();
tokio::spawn(async move {
let mut pull_scratch = vec![0.0_f32; ANDROID_RENDER_PULL_SAMPLES];
let mut interval = tokio::time::interval(std::time::Duration::from_millis(20));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
interval.tick().await;
if shutdown_for_task.load(Ordering::Relaxed) {
break;
}
for cmd in event_consumer.drain_controls() {
match cmd {
AudioCommand::SetVolume(id, vol) => {
if let Some(q) = handler.get_mut_queues().get_mut(&id) {
q.volume = vol;
}
}
AudioCommand::RemoveClient(id) => {
handler.get_mut_queues().remove(&id);
}
}
}
for pkt in event_consumer.drain_packets(50) {
if let Err(e) = handler.handle_packet(pkt.client_id, pkt.data) {
debug!(target: "chanora_audio", error = %e, "decode failed");
}
}
pull_scratch.fill(0.0);
let _ = handler.fill_buffer(&mut pull_scratch);
pcm_producer.push_frame_lossy(&pull_scratch);
}
});
shutdown
}
/// Clone of the event sender, for JNI focus / SCO listeners
/// registered on the engine's behalf.
pub fn event_sender(&self) -> BackendEventTx {
@@ -1151,6 +1159,7 @@ impl MobileVoiceAudioBackend for AndroidVoiceUnit {
fn close(&mut self) -> Result<(), BackendError> {
// SDD-115 reverse order: release hardware effects FIRST,
// then close streams.
self.render_producer_shutdown.store(true, Ordering::Relaxed);
release_hardware_effects(&mut self.hw_effects);
self.stop().ok();
// Dropping the Option drops the underlying AudioStreamAsync
@@ -1208,6 +1217,7 @@ impl Drop for AndroidVoiceUnit {
// Wrap in catch_unwind so a panic during Drop cannot unwind
// into the JVM (SDD-115 callback safety).
let _ = catch_unwind(AssertUnwindSafe(|| {
self.render_producer_shutdown.store(true, Ordering::Relaxed);
release_hardware_effects(&mut self.hw_effects);
// SDD-116: clear the diagnostics slot on Drop too.
clear_android_audio_diagnostics();