edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate chat voice and diagnostics client

Browse Source
This commit is contained in:
Edison Jwa
2026-05-23 06:51:55 +09:00
parent 7e28791ec2
commit 7d5d8c2c90
93 changed files with 10273 additions and 3347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/android_voice_unit.rs
+53 -77
View File
@@ -50,7 +50,7 @@ use crate::mobile_voice_backend::{
AchievedPerformanceMode, AchievedSharingMode, AndroidAudioDiagnostics,
AndroidVoiceStreamConfig, AudioSessionId, BackendError, BackendEvent, BackendEventRx,
BackendEventTx, EffectEngagement, EffectEngine, InputPresetChoice, MobileVoiceAudioBackend,
SharingModeChoice,
SharingModeChoice, VoiceAudioParams,
};
use chanora_protocol::OutPacket;
use tsclientlib::audio::AudioHandler;
@@ -102,8 +102,8 @@ impl RenderReferenceBuffer {
fn write(&self, frame: &[f32; RENDER_REF_SAMPLES]) {
let idx = self.write_idx.load(Ordering::Relaxed);
unsafe {
let slot =
&self.buf[idx] as *const [f32; RENDER_REF_SAMPLES] as *mut [f32; RENDER_REF_SAMPLES];
let slot = &self.buf[idx] as *const [f32; RENDER_REF_SAMPLES]
as *mut [f32; RENDER_REF_SAMPLES];
(*slot).copy_from_slice(frame);
}
self.write_idx
@@ -148,6 +148,7 @@ struct AndroidCaptureState {
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
current_vad_backend: crate::VadBackend,
silero_model_epoch: u64,
ten_model_epoch: u64,
capture_frame_seq: u64,
vad_state: crate::voice_activity::VoiceActivityStateMachine,
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
@@ -208,6 +209,7 @@ impl AndroidCaptureState {
ten_vad_worker: None,
current_vad_backend: crate::VadBackend::WebrtcVad,
silero_model_epoch: crate::vad::silero_model_epoch(),
ten_model_epoch: crate::vad::ten_model_epoch(),
capture_frame_seq: 0,
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
@@ -227,8 +229,7 @@ impl AndroidCaptureState {
fn ingest_i16(&mut self, samples: &[i16]) {
let mut offset = 0;
while offset < samples.len() {
let remaining =
crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let remaining = crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
let take = remaining.min(samples.len() - offset);
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
.copy_from_slice(&samples[offset..offset + take]);
@@ -249,11 +250,8 @@ impl AndroidCaptureState {
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
frame.copy_from_slice(
&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES],
);
self.pcm_accum
.drain(..crate::frame::FRAME_20MS_SAMPLES);
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
Ok(len) => {
crate::opus_voice::send_voip_frame(
@@ -289,18 +287,26 @@ impl AndroidCaptureState {
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
if self.fallback_warned_backend != Some(failed_backend) {
self.fallback_warned_backend = Some(failed_backend);
warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"android: VAD backend unavailable; using WebRTC fallback for runtime detection"
);
// Warm-up period: ONNX workers need ~100ms to process first frame.
// Don't flag as a problem if the capture has just started.
if self.capture_frame_seq < 128 {
info!(
target: "chanora_audio",
backend = failed_backend.as_str(),
seq = self.capture_frame_seq,
"android: VAD backend warming up; using WebRTC fallback"
);
} else {
warn!(
target: "chanora_audio",
backend = failed_backend.as_str(),
"android: VAD backend unavailable; using WebRTC fallback for runtime detection"
);
}
}
}
fn process_10ms_capture_frame(
&mut self,
samples: &[i16; crate::frame::FRAME_10MS_SAMPLES],
) {
fn process_10ms_capture_frame(&mut self, samples: &[i16; crate::frame::FRAME_10MS_SAMPLES]) {
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
*dst = crate::frame::i16_to_f32(src);
@@ -327,11 +333,15 @@ impl AndroidCaptureState {
// VAD backend switching (mirrors iOS Raw path).
let silero_epoch = crate::vad::silero_model_epoch();
let silero_changed = vad_backend == crate::VadBackend::SileroOnnx
&& silero_epoch != self.silero_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed {
let ten_epoch = crate::vad::ten_model_epoch();
let silero_changed =
vad_backend == crate::VadBackend::SileroOnnx && silero_epoch != self.silero_model_epoch;
let ten_changed =
vad_backend == crate::VadBackend::TenVad && ten_epoch != self.ten_model_epoch;
if vad_backend != self.current_vad_backend || silero_changed || ten_changed {
self.current_vad_backend = vad_backend;
self.silero_model_epoch = silero_epoch;
self.ten_model_epoch = ten_epoch;
self.fallback_warned_backend = None;
match vad_backend {
crate::VadBackend::SileroOnnx => {
@@ -376,52 +386,52 @@ impl AndroidCaptureState {
} else if vad_backend == crate::VadBackend::SileroOnnx {
if let Some(worker) = self.silero_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
// Warm-up: worker dispatched but hasn't finished yet.
// Default to no-speech until first result arrives (~100ms).
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else if vad_backend == crate::VadBackend::TenVad {
if let Some(worker) = self.ten_vad_worker.as_ref() {
let enqueued = worker.try_send(capture_seq, &frame);
if enqueued && !worker.is_stale(capture_seq) {
if !worker.is_stale(capture_seq) {
let p = worker.latest_probability();
crate::vad::VadOutput {
probability: p,
speech: p >= 0.5,
}
} else if enqueued {
crate::vad::VadOutput {
probability: 0.0,
speech: false,
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
used_fallback_vad = true;
self.mark_vad_fallback_active(vad_backend);
crate::vad::VoiceActivityDetector::process_10ms(
&mut self.vad_detector,
&frame,
)
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
}
} else {
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
@@ -572,37 +582,6 @@ impl AudioOutputCallback for OutputCallback {
}
}
/// Bundle of engine-owned state shared with the Oboe audio callbacks.
/// Mirrors the parameter set that iOS `IosVoiceUnit::start()` receives
/// from the engine (SDD-120 amendment: Android Oboe-only audio path).
pub struct VoiceAudioParams {
/// Opus-encoded voice packets sent on this channel toward the
/// protocol layer.
pub voice_out_tx: mpsc::Sender<OutPacket>,
/// PTT transmission gate — true when the user holds the PTT key.
pub transmit_active: Arc<AtomicBool>,
/// Counter incremented per encoded frame sent.
pub frames_sent: Arc<AtomicU32>,
/// Pre-encode amplitude scale (1.0 = unity).
pub mic_gain: f32,
/// AudioHandler that inbound decode+mix feeds into; the Oboe output
/// callback pulls mixed stereo f32 from it.
pub handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
/// Master output gain (f32 bits stored in AtomicU32 for lock-free
/// cross-thread read from the realtime audio callback).
pub output_gain: Arc<AtomicU32>,
/// True = output silence regardless of incoming voice frames.
pub output_muted: Arc<AtomicBool>,
/// Optional TransmitModeSelector for VoiceActivity transmit mode.
/// The capture callback calls set_voice_activity_open on this when
/// VAD detects speech. None means VoiceActivity mode is disabled.
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
/// Shared audio-processing config (WebRTC APM flags, VAD backend).
pub audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
/// Shared audio-processing statistics for diagnostics.
pub audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
}
// --- The backend itself ------------------------------------------
/// Android voice-audio backend (SDD-111). Owns one input + one
@@ -647,8 +626,7 @@ impl AndroidVoiceUnit {
/// `params` bundles the engine-owned state shared with the Oboe
/// audio callbacks (SDD-120 amendment: Android Oboe-only audio
/// path — capture pipeline, playback pull, and PTT gate).
#[allow(clippy::too_many_arguments)]
pub fn open(
pub(crate) fn open(
cfg: &AndroidVoiceStreamConfig,
params: VoiceAudioParams,
) -> Result<Self, BackendError> {
@@ -761,9 +739,7 @@ impl AndroidVoiceUnit {
//
// For now: hardware effects require the system session ID.
// WebRTC APM software processing handles AEC/NS/AGC/HPF.
let session_id: Option<i32> = input_stream
.as_ref()
.and_then(|s| s.get_raw_session_id());
let session_id: Option<i32> = input_stream.as_ref().and_then(|s| s.get_raw_session_id());
// --- Open output stream (SDD-112) --------------------------
let output_builder = AudioStreamBuilder::default()
@@ -1558,7 +1534,7 @@ fn load_app_class<'local>(
Ok(c) => return Some(c),
Err(e) => {
let _ = env.exception_clear();
warn!(target: "chanora_audio", error = %e, class = slash_name, "android: find_class failed; retrying with app ClassLoader");
debug!(target: "chanora_audio", error = %e, class = slash_name, "android: find_class failed; retrying with app ClassLoader");
}
}