feat: Android Oboe voice backend — WebRTC APM, VAD, HW/SW toggle, BBCode welcome, link trust, foreground task
Audio engine (Rust): - Android Oboe: WebRTC APM (AEC/NS/AGC/HPF) + TEN/Silero ONNX VAD - Hardware effects (JNI) with software fallback per-effect - Render reference buffer for AEC between output/capture callbacks - Voice activity gate: suppress transmission when speaker muted (all platforms) - Audio focus (SDD-109) + Bluetooth SCO (SDD-110) via JNI - ONNX Runtime 1.26 via ort 2.0.0-rc.12 (down from rc.10, ndarray 0.17) - VAD worker channel capacity 8→32, initial seq u64::MAX (warm-up fix) - TEN VAD default backend (was Silero) - Platform→WebrtcApm resolution after hardware binding - oboe-rs edisonjwa fork with get_raw_session_id() Android Kotlin: - AndroidAudioFocusController + AndroidBluetoothScoController - AndroidAudioLifecycleController (route changes to Flutter) - ProGuard rules for new controllers Flutter UI: - VoiceSettings: Android HW/SW toggle (Platform auto / WebRTC APM) - VoiceStatusChip: mute warning border + Speaker muted label - BBCode welcome message parser (BbCodeText, case-insensitive) - Welcome message foldable (expanded by default) - Link trust dialog (domain wildcards, SharedPreferences) - HapticFeedback on voice sheet opener - Server name in AppBar, version v0.1.0 - Default channel (id=1) visible, serverquery clients hidden - flutter_foreground_task integration Config: - ort load-dynamic on all non-iOS (Android/Linux/Windows) - ONNX Runtime AAR 1.26.0 - ndarray moved to common deps (was Apple-only)
This commit is contained in:
Edison Jwa
@@ -66,72 +66,194 @@ use oboe::{
|
||||
PerformanceMode, SessionId, SharingMode, Usage,
|
||||
};
|
||||
|
||||
use crate::processor::AudioProcessor;
|
||||
|
||||
// `BackendEvent` / `BackendEventRx` / `BackendEventTx` moved to
|
||||
// `mobile_voice_backend` so the trait can expose `take_event_rx`
|
||||
// (SDD-111 item 1) cross-platform.
|
||||
|
||||
// --- Render-reference buffer for AEC (SDD-111 / SDD-120) ---------
|
||||
//
|
||||
// The output (render) callback writes the audio that will be played
|
||||
// into this ring buffer. The capture callback reads the latest render
|
||||
// frame and feeds it to WebRTC APM's `process_render` so AEC can
|
||||
// subtract the speaker output from the microphone input.
|
||||
//
|
||||
// 4 slots × 10 ms × 48 kHz mono f32. One slot is always being written
|
||||
// by the render callback; the capture callback reads the slot that was
|
||||
// most recently completed.
|
||||
|
||||
const RENDER_REF_SLOTS: usize = 4;
|
||||
const RENDER_REF_SAMPLES: usize = crate::frame::FRAME_10MS_SAMPLES;
|
||||
|
||||
struct RenderReferenceBuffer {
|
||||
buf: Box<[[f32; RENDER_REF_SAMPLES]; RENDER_REF_SLOTS]>,
|
||||
write_idx: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
impl RenderReferenceBuffer {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
buf: Box::new([[0.0_f32; RENDER_REF_SAMPLES]; RENDER_REF_SLOTS]),
|
||||
write_idx: std::sync::atomic::AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn write(&self, frame: &[f32; RENDER_REF_SAMPLES]) {
|
||||
let idx = self.write_idx.load(Ordering::Relaxed);
|
||||
unsafe {
|
||||
let slot =
|
||||
&self.buf[idx] as *const [f32; RENDER_REF_SAMPLES] as *mut [f32; RENDER_REF_SAMPLES];
|
||||
(*slot).copy_from_slice(frame);
|
||||
}
|
||||
self.write_idx
|
||||
.store((idx + 1) % RENDER_REF_SLOTS, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn read_latest(&self) -> [f32; RENDER_REF_SAMPLES] {
|
||||
let wi = self.write_idx.load(Ordering::Relaxed);
|
||||
let ri = (wi + RENDER_REF_SLOTS - 1) % RENDER_REF_SLOTS;
|
||||
self.buf[ri]
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl Send for RenderReferenceBuffer {}
|
||||
unsafe impl Sync for RenderReferenceBuffer {}
|
||||
|
||||
// --- Capture state for Oboe input callback (SDD-111 / SDD-120) ----
|
||||
//
|
||||
// Mirrors the iOS `IosCaptureState` and the cpal-side `CaptureState`.
|
||||
// Oboe delivers 48 kHz mono i16 PCM; we apply mic gain, accumulate to
|
||||
// FRAME_20MS_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
|
||||
// and try-send the resulting packet on `voice_out_tx`.
|
||||
// Enhanced with WebRTC APM (AEC/NS/AGC) and VAD (voice activity
|
||||
// detection). Oboe delivers 48 kHz mono i16 PCM in variable-size
|
||||
// chunks. We accumulate into 10 ms frames, then:
|
||||
//
|
||||
// 1. i16 → f32 conversion
|
||||
// 2. Read render reference (for AEC)
|
||||
// 3. WebRtcApmProcessor::process_render + process_capture
|
||||
// 4. VAD → VoiceActivityStateMachine → TransmitModeSelector
|
||||
// 5. f32 → i16 conversion + mic gain
|
||||
// 6. Accumulate to 20 ms → Opus encode → send
|
||||
|
||||
struct AndroidCaptureState {
|
||||
encoder: OpusEncoder,
|
||||
/// Accumulator for 48 kHz mono PCM. 2x capacity to absorb
|
||||
/// cpal-style buffer-size jitter without reallocating.
|
||||
pcm_accum: Vec<i16>,
|
||||
opus_out: [u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
output_muted: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad,
|
||||
silero_vad_worker: Option<crate::vad::silero_onnx::SileroOnnxVadWorker>,
|
||||
ten_vad_worker: Option<crate::vad::TenOnnxVadWorker>,
|
||||
current_vad_backend: crate::VadBackend,
|
||||
silero_model_epoch: u64,
|
||||
capture_frame_seq: u64,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine,
|
||||
webrtc_apm_processor: crate::processor::WebRtcApmProcessor,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
pending_10ms: [i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_10ms_len: usize,
|
||||
fallback_warned_backend: Option<crate::VadBackend>,
|
||||
}
|
||||
|
||||
impl AndroidCaptureState {
|
||||
fn new(
|
||||
voice_out_tx: mpsc::Sender<OutPacket>,
|
||||
transmit_active: Arc<AtomicBool>,
|
||||
output_muted: Arc<AtomicBool>,
|
||||
frames_sent: Arc<AtomicU32>,
|
||||
mic_gain: f32,
|
||||
voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
) -> Result<Self, AudioError> {
|
||||
let encoder = crate::opus_voice::new_voip_encoder("android")?;
|
||||
// Android always uses software WebRTC APM for AEC/NS/AGC/HPF.
|
||||
// The config's EffectOwner fields are resolved by open() AFTER
|
||||
// hardware-effect binding; the processor is constructed here
|
||||
// with all modules enabled regardless, so the resolved config
|
||||
// (Platform vs WebrtcApm) only affects diagnostics, not behaviour.
|
||||
let webrtc_apm_config = audio_processing_config
|
||||
.lock()
|
||||
.map(|cfg| {
|
||||
let mut c = crate::processor::webrtc_apm::WebRtcApmConfig::from_audio_config(&cfg);
|
||||
c.aec = true;
|
||||
c.ns = true;
|
||||
c.agc = true;
|
||||
c
|
||||
})
|
||||
.unwrap_or(crate::processor::webrtc_apm::WebRtcApmConfig {
|
||||
aec: true,
|
||||
ns: true,
|
||||
agc: true,
|
||||
hpf: true,
|
||||
..Default::default()
|
||||
});
|
||||
Ok(Self {
|
||||
encoder,
|
||||
pcm_accum: Vec::with_capacity(crate::frame::FRAME_20MS_SAMPLES * 2),
|
||||
opus_out: [0u8; crate::opus_voice::MAX_OPUS_FRAME],
|
||||
voice_out_tx,
|
||||
transmit_active,
|
||||
output_muted,
|
||||
frames_sent,
|
||||
mic_gain,
|
||||
voice_activity_selector,
|
||||
vad_detector: crate::vad::WebRtcFallbackVad::default(),
|
||||
silero_vad_worker: None,
|
||||
ten_vad_worker: None,
|
||||
current_vad_backend: crate::VadBackend::WebrtcVad,
|
||||
silero_model_epoch: crate::vad::silero_model_epoch(),
|
||||
capture_frame_seq: 0,
|
||||
vad_state: crate::voice_activity::VoiceActivityStateMachine::default(),
|
||||
webrtc_apm_processor: crate::processor::WebRtcApmProcessor::with_config(
|
||||
webrtc_apm_config,
|
||||
)?,
|
||||
audio_processing_config,
|
||||
audio_processing_stats,
|
||||
render_reference,
|
||||
pending_10ms: [0_i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
pending_10ms_len: 0,
|
||||
fallback_warned_backend: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Consume i16 mono frames from Oboe, accumulate to FRAME_20MS_SAMPLES,
|
||||
/// encode + send when PTT is held. Oboe delivers at the device's
|
||||
/// native sample rate (always 48 kHz for modern Android per SRS-210),
|
||||
/// so no resampling is needed.
|
||||
fn ingest(&mut self, samples: &[i16]) {
|
||||
/// Consume i16 mono frames from Oboe. Accumulate to 10 ms chunks,
|
||||
/// process each through WebRTC APM + VAD, then encode 20 ms frames.
|
||||
fn ingest_i16(&mut self, samples: &[i16]) {
|
||||
let mut offset = 0;
|
||||
while offset < samples.len() {
|
||||
let remaining =
|
||||
crate::frame::FRAME_10MS_SAMPLES - self.pending_10ms_len;
|
||||
let take = remaining.min(samples.len() - offset);
|
||||
self.pending_10ms[self.pending_10ms_len..self.pending_10ms_len + take]
|
||||
.copy_from_slice(&samples[offset..offset + take]);
|
||||
self.pending_10ms_len += take;
|
||||
offset += take;
|
||||
|
||||
if self.pending_10ms_len == crate::frame::FRAME_10MS_SAMPLES {
|
||||
let frame = self.pending_10ms;
|
||||
self.process_10ms_capture_frame(&frame);
|
||||
self.pending_10ms_len = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if !self.transmit_active.load(Ordering::Relaxed) {
|
||||
self.pcm_accum.clear();
|
||||
return;
|
||||
}
|
||||
// Mic-gain application.
|
||||
if (self.mic_gain - 1.0).abs() < f32::EPSILON {
|
||||
self.pcm_accum.extend_from_slice(samples);
|
||||
} else {
|
||||
let gain = self.mic_gain;
|
||||
self.pcm_accum.extend(samples.iter().map(|&s| {
|
||||
let scaled = (s as f32) * gain;
|
||||
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
|
||||
}));
|
||||
}
|
||||
// Drain complete 20 ms frames.
|
||||
|
||||
while self.pcm_accum.len() >= crate::frame::FRAME_20MS_SAMPLES {
|
||||
let mut frame = [0i16; crate::frame::FRAME_20MS_SAMPLES];
|
||||
frame.copy_from_slice(&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES]);
|
||||
self.pcm_accum.drain(..crate::frame::FRAME_20MS_SAMPLES);
|
||||
frame.copy_from_slice(
|
||||
&self.pcm_accum[..crate::frame::FRAME_20MS_SAMPLES],
|
||||
);
|
||||
self.pcm_accum
|
||||
.drain(..crate::frame::FRAME_20MS_SAMPLES);
|
||||
match self.encoder.encode(&frame, &mut self.opus_out[..]) {
|
||||
Ok(len) => {
|
||||
crate::opus_voice::send_voip_frame(
|
||||
@@ -154,11 +276,186 @@ impl AndroidCaptureState {
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(target: "chanora_audio", error = %e, "android Oboe opus encode failed");
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
error = %e,
|
||||
"android Oboe opus encode failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_vad_fallback_active(&mut self, failed_backend: crate::VadBackend) {
|
||||
if self.fallback_warned_backend != Some(failed_backend) {
|
||||
self.fallback_warned_backend = Some(failed_backend);
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
backend = failed_backend.as_str(),
|
||||
"android: VAD backend unavailable; using WebRTC fallback for runtime detection"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn process_10ms_capture_frame(
|
||||
&mut self,
|
||||
samples: &[i16; crate::frame::FRAME_10MS_SAMPLES],
|
||||
) {
|
||||
let mut frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||||
for (dst, src) in frame.iter_mut().zip(samples.iter().copied()) {
|
||||
*dst = crate::frame::i16_to_f32(src);
|
||||
}
|
||||
let input_dbfs = crate::frame::dbfs(&frame);
|
||||
|
||||
let render_ref = self.render_reference.read_latest();
|
||||
self.webrtc_apm_processor.process_render(&render_ref);
|
||||
self.webrtc_apm_processor.process_capture(&mut frame);
|
||||
|
||||
let (vad_hangover, vad_backend) = self
|
||||
.audio_processing_config
|
||||
.try_lock()
|
||||
.map(|cfg| (cfg.vad_hangover_ms, cfg.vad_backend))
|
||||
.unwrap_or((
|
||||
crate::voice_activity::VAD_HANGOVER_MS,
|
||||
crate::VadBackend::WebrtcVad,
|
||||
));
|
||||
self.vad_state.configure(
|
||||
crate::voice_activity::VAD_OPEN_AFTER_MS,
|
||||
vad_hangover,
|
||||
crate::voice_activity::VAD_MIN_TX_MS,
|
||||
);
|
||||
|
||||
// VAD backend switching (mirrors iOS Raw path).
|
||||
let silero_epoch = crate::vad::silero_model_epoch();
|
||||
let silero_changed = vad_backend == crate::VadBackend::SileroOnnx
|
||||
&& silero_epoch != self.silero_model_epoch;
|
||||
if vad_backend != self.current_vad_backend || silero_changed {
|
||||
self.current_vad_backend = vad_backend;
|
||||
self.silero_model_epoch = silero_epoch;
|
||||
self.fallback_warned_backend = None;
|
||||
match vad_backend {
|
||||
crate::VadBackend::SileroOnnx => {
|
||||
let path = crate::vad::silero_model_bundle_path();
|
||||
self.silero_vad_worker =
|
||||
crate::vad::silero_onnx::SileroOnnxVadWorker::try_new(&path);
|
||||
self.ten_vad_worker = None;
|
||||
if self.silero_vad_worker.is_none() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android: Silero VAD model not found at {path}; falling back to WebRTC VAD"
|
||||
);
|
||||
}
|
||||
}
|
||||
crate::VadBackend::TenVad => {
|
||||
let path = crate::vad::ten_model_bundle_path();
|
||||
self.ten_vad_worker = crate::vad::TenOnnxVadWorker::try_new(&path);
|
||||
self.silero_vad_worker = None;
|
||||
if self.ten_vad_worker.is_none() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android: TEN VAD ONNX model not found at {path}; falling back to WebRTC VAD"
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
self.silero_vad_worker = None;
|
||||
self.ten_vad_worker = None;
|
||||
}
|
||||
}
|
||||
self.vad_state.reset();
|
||||
}
|
||||
|
||||
self.capture_frame_seq = self.capture_frame_seq.wrapping_add(1);
|
||||
let capture_seq = self.capture_frame_seq;
|
||||
let mut used_fallback_vad = false;
|
||||
let vad = if vad_backend == crate::VadBackend::Disabled {
|
||||
crate::vad::VadOutput {
|
||||
probability: 1.0,
|
||||
speech: true,
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::SileroOnnx {
|
||||
if let Some(worker) = self.silero_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if enqueued && !worker.is_stale(capture_seq) {
|
||||
let p = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability: p,
|
||||
speech: p >= 0.5,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(
|
||||
&mut self.vad_detector,
|
||||
&frame,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(
|
||||
&mut self.vad_detector,
|
||||
&frame,
|
||||
)
|
||||
}
|
||||
} else if vad_backend == crate::VadBackend::TenVad {
|
||||
if let Some(worker) = self.ten_vad_worker.as_ref() {
|
||||
let enqueued = worker.try_send(capture_seq, &frame);
|
||||
if enqueued && !worker.is_stale(capture_seq) {
|
||||
let p = worker.latest_probability();
|
||||
crate::vad::VadOutput {
|
||||
probability: p,
|
||||
speech: p >= 0.5,
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(
|
||||
&mut self.vad_detector,
|
||||
&frame,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
used_fallback_vad = true;
|
||||
self.mark_vad_fallback_active(vad_backend);
|
||||
crate::vad::VoiceActivityDetector::process_10ms(
|
||||
&mut self.vad_detector,
|
||||
&frame,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
crate::vad::VoiceActivityDetector::process_10ms(&mut self.vad_detector, &frame)
|
||||
};
|
||||
self.audio_processing_stats
|
||||
.set_vad_fallback_active(used_fallback_vad);
|
||||
let active = self.vad_state.update(vad.speech);
|
||||
let output_muted = self.output_muted.load(Ordering::Relaxed);
|
||||
if let Some(sel) = &self.voice_activity_selector {
|
||||
sel.set_voice_activity_open(active && !output_muted);
|
||||
}
|
||||
self.audio_processing_stats.update_capture(
|
||||
input_dbfs,
|
||||
crate::frame::dbfs(&frame),
|
||||
vad.probability,
|
||||
active && !output_muted,
|
||||
self.transmit_active.load(Ordering::Relaxed),
|
||||
);
|
||||
|
||||
if !self.transmit_active.load(Ordering::Relaxed) || output_muted {
|
||||
return;
|
||||
}
|
||||
|
||||
let gain = self.mic_gain;
|
||||
if (gain - 1.0).abs() < f32::EPSILON {
|
||||
self.pcm_accum
|
||||
.extend(frame.iter().copied().map(crate::frame::f32_to_i16));
|
||||
} else {
|
||||
self.pcm_accum.extend(frame.iter().copied().map(|s| {
|
||||
let scaled = (crate::frame::f32_to_i16(s) as f32) * gain;
|
||||
scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct InputCallback {
|
||||
@@ -176,7 +473,7 @@ impl AudioInputCallback for InputCallback {
|
||||
) -> DataCallbackResult {
|
||||
let _ = catch_unwind(AssertUnwindSafe(|| {
|
||||
if let Ok(mut s) = self.state.lock() {
|
||||
s.ingest(frames);
|
||||
s.ingest_i16(frames);
|
||||
}
|
||||
}));
|
||||
DataCallbackResult::Continue
|
||||
@@ -203,6 +500,7 @@ struct OutputCallback {
|
||||
output_muted: Arc<AtomicBool>,
|
||||
event_tx: BackendEventTx,
|
||||
scratch: Arc<Mutex<Vec<f32>>>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
}
|
||||
|
||||
impl AudioOutputCallback for OutputCallback {
|
||||
@@ -223,16 +521,17 @@ impl AudioOutputCallback for OutputCallback {
|
||||
*s = 0.0;
|
||||
}
|
||||
}
|
||||
// Non-blocking pull from AudioHandler (same pattern as iOS VPIO).
|
||||
match self.handler.try_lock() {
|
||||
Ok(mut h) => {
|
||||
let _ = h.fill_buffer(&mut scratch[..needed]);
|
||||
}
|
||||
Err(std::sync::TryLockError::WouldBlock) => {
|
||||
// scratch already zeroed above.
|
||||
}
|
||||
Err(std::sync::TryLockError::WouldBlock) => {}
|
||||
Err(std::sync::TryLockError::Poisoned(e)) => {
|
||||
warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {}", e);
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"AudioHandler mutex poisoned: {}",
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
let gain = f32::from_bits(self.output_gain.load(Ordering::Relaxed));
|
||||
@@ -243,6 +542,19 @@ impl AudioOutputCallback for OutputCallback {
|
||||
gain,
|
||||
muted,
|
||||
);
|
||||
|
||||
// Write the first 10 ms of render audio into the reference
|
||||
// buffer for the capture-side AEC.
|
||||
let mono_n = needed / 2;
|
||||
let render_n = mono_n.min(crate::frame::FRAME_10MS_SAMPLES);
|
||||
let mut ref_frame = [0.0_f32; crate::frame::FRAME_10MS_SAMPLES];
|
||||
for (i, chunk) in scratch[..render_n * 2].chunks_exact(2).enumerate() {
|
||||
if i >= render_n {
|
||||
break;
|
||||
}
|
||||
ref_frame[i] = (chunk[0] + chunk[1]) * 0.5;
|
||||
}
|
||||
self.render_reference.write(&ref_frame);
|
||||
}));
|
||||
DataCallbackResult::Continue
|
||||
}
|
||||
@@ -273,7 +585,7 @@ pub struct VoiceAudioParams {
|
||||
pub frames_sent: Arc<AtomicU32>,
|
||||
/// Pre-encode amplitude scale (1.0 = unity).
|
||||
pub mic_gain: f32,
|
||||
/// AudioHandler that inbound解码+混合 feeds into; the Oboe output
|
||||
/// AudioHandler that inbound decode+mix feeds into; the Oboe output
|
||||
/// callback pulls mixed stereo f32 from it.
|
||||
pub handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
|
||||
/// Master output gain (f32 bits stored in AtomicU32 for lock-free
|
||||
@@ -281,6 +593,14 @@ pub struct VoiceAudioParams {
|
||||
pub output_gain: Arc<AtomicU32>,
|
||||
/// True = output silence regardless of incoming voice frames.
|
||||
pub output_muted: Arc<AtomicBool>,
|
||||
/// Optional TransmitModeSelector for VoiceActivity transmit mode.
|
||||
/// The capture callback calls set_voice_activity_open on this when
|
||||
/// VAD detects speech. None means VoiceActivity mode is disabled.
|
||||
pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
|
||||
/// Shared audio-processing config (WebRTC APM flags, VAD backend).
|
||||
pub audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
|
||||
/// Shared audio-processing statistics for diagnostics.
|
||||
pub audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
|
||||
}
|
||||
|
||||
// --- The backend itself ------------------------------------------
|
||||
@@ -334,22 +654,30 @@ impl AndroidVoiceUnit {
|
||||
) -> Result<Self, BackendError> {
|
||||
let (event_tx, event_rx) = mpsc::unbounded_channel();
|
||||
|
||||
// SDD-120: build the capture state that the Oboe input callback
|
||||
// will own via Arc<Mutex>. Same Opus VoIP tuning as iOS and
|
||||
// desktop (32 kbps, complexity 10, inband FEC, 5 % PLC).
|
||||
// Shared render-reference buffer (for AEC). The output
|
||||
// callback writes; the capture callback reads.
|
||||
let render_ref_buf = RenderReferenceBuffer::new();
|
||||
let render_ref_for_capture = render_ref_buf.clone();
|
||||
|
||||
// Clone the APM config Arc before params is partially moved
|
||||
// into the capture state constructor below.
|
||||
let apm_config_clone = params.audio_processing_config.clone();
|
||||
|
||||
let capture_state = Arc::new(Mutex::new(
|
||||
AndroidCaptureState::new(
|
||||
params.voice_out_tx,
|
||||
params.transmit_active,
|
||||
params.output_muted.clone(),
|
||||
params.frames_sent,
|
||||
params.mic_gain,
|
||||
params.voice_activity_selector,
|
||||
params.audio_processing_config,
|
||||
params.audio_processing_stats,
|
||||
render_ref_for_capture,
|
||||
)
|
||||
.map_err(|e| BackendError::OpenFailed(format!("capture state init: {e}")))?,
|
||||
));
|
||||
|
||||
// Scratch buffer for the output callback (realtime-safe
|
||||
// pre-allocation). 8192 floats covers the largest practical
|
||||
// burst size at 48 kHz with headroom.
|
||||
let scratch = Arc::new(Mutex::new(Vec::with_capacity(8192)));
|
||||
|
||||
// --- Open input stream (SDD-112) ---------------------------
|
||||
@@ -423,13 +751,19 @@ impl AndroidVoiceUnit {
|
||||
.as_mut()
|
||||
.map(|s| s.get_frames_per_burst())
|
||||
.unwrap_or(0);
|
||||
let session_id = None;
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android: oboe-rs get_session_id is skipped because oboe 0.6.1 \
|
||||
panics on some Android allocated-session values; hardware \
|
||||
effects are disabled for this stream"
|
||||
);
|
||||
// The oboe-rs fork (edisonjwa/oboe-rs 0.6.2) fixes the
|
||||
// get_session_id() panic with unwrap_or_default(), but the
|
||||
// SessionId enum still only models builder parameters (None =
|
||||
// -1, Allocate = 0). The actual system audio session ID (>0)
|
||||
// written by AAudio to mSessionId after stream open cannot be
|
||||
// expressed in the current enum; get_raw_session_id() is a
|
||||
// follow-up addition to the fork.
|
||||
//
|
||||
// For now: hardware effects require the system session ID.
|
||||
// WebRTC APM software processing handles AEC/NS/AGC/HPF.
|
||||
let session_id: Option<i32> = input_stream
|
||||
.as_ref()
|
||||
.and_then(|s| s.get_raw_session_id());
|
||||
|
||||
// --- Open output stream (SDD-112) --------------------------
|
||||
let output_builder = AudioStreamBuilder::default()
|
||||
@@ -450,12 +784,14 @@ impl AndroidVoiceUnit {
|
||||
.set_usage(Usage::VoiceCommunication)
|
||||
.set_content_type(oboe::ContentType::Speech);
|
||||
|
||||
let render_ref_for_output = render_ref_buf.clone();
|
||||
let output_cb = OutputCallback {
|
||||
handler: params.handler.clone(),
|
||||
output_gain: params.output_gain.clone(),
|
||||
output_muted: params.output_muted.clone(),
|
||||
event_tx: event_tx.clone(),
|
||||
scratch: scratch.clone(),
|
||||
render_reference: render_ref_for_output,
|
||||
};
|
||||
let output_builder = output_builder.set_callback(output_cb);
|
||||
|
||||
@@ -474,6 +810,7 @@ impl AndroidVoiceUnit {
|
||||
params.output_gain.clone(),
|
||||
params.output_muted.clone(),
|
||||
scratch.clone(),
|
||||
render_ref_buf,
|
||||
)?
|
||||
}
|
||||
};
|
||||
@@ -511,6 +848,52 @@ impl AndroidVoiceUnit {
|
||||
HardwareEffectHandles::default()
|
||||
};
|
||||
|
||||
// --- SDD-113 config resolution: hardware-available? -------
|
||||
// Android gives the user a choice between hardware (JNI) and
|
||||
// software (WebRTC APM) effects. The AudioProcessingConfig's
|
||||
// EffectOwner fields encode that choice:
|
||||
// Platform → prefer hardware; software fallback if missing
|
||||
// WebrtcApm → always software WebRTC APM
|
||||
// Off → disable effect entirely
|
||||
//
|
||||
// Here we resolve Platform → WebrtcApm for each effect whose
|
||||
// hardware binding failed (or wasn't attempted). This is read
|
||||
// by the capture callback's WebRtcApmProcessor.
|
||||
{
|
||||
use crate::audio_processing::EffectOwner;
|
||||
let mut apm_cfg = apm_config_clone.lock().unwrap();
|
||||
let hw_aec = hw_effects.aec.is_some();
|
||||
let hw_ns = hw_effects.ns.is_some();
|
||||
let hw_agc = hw_effects.agc.is_some();
|
||||
if apm_cfg.aec == EffectOwner::Platform && !hw_aec {
|
||||
apm_cfg.aec = EffectOwner::WebrtcApm;
|
||||
}
|
||||
if apm_cfg.ns == EffectOwner::Platform && !hw_ns {
|
||||
apm_cfg.ns = EffectOwner::WebrtcApm;
|
||||
}
|
||||
if apm_cfg.agc == EffectOwner::Platform && !hw_agc {
|
||||
apm_cfg.agc = EffectOwner::WebrtcApm;
|
||||
}
|
||||
if apm_cfg.processing_backend
|
||||
== crate::audio_processing::AudioBackend::PlatformVoiceProcessing
|
||||
&& (!hw_aec || !hw_ns || !hw_agc)
|
||||
{
|
||||
apm_cfg.processing_backend = crate::audio_processing::AudioBackend::WebrtcApm;
|
||||
}
|
||||
info!(
|
||||
target: "chanora_audio",
|
||||
aec = ?apm_cfg.aec,
|
||||
ns = ?apm_cfg.ns,
|
||||
agc = ?apm_cfg.agc,
|
||||
hpf = apm_cfg.hpf_enabled,
|
||||
hw_aec,
|
||||
hw_ns,
|
||||
hw_agc,
|
||||
session_id,
|
||||
"android: audio processing config resolved (hardware effects: aec={hw_aec} ns={hw_ns} agc={hw_agc})"
|
||||
);
|
||||
}
|
||||
|
||||
// --- SDD-112 item 10 / SDD-113 item 7 / SDD-116 item 3 ---
|
||||
// Publish the diagnostics snapshot. Per-effect engagement is
|
||||
// derived from (a) the JNI handle (`Hardware`) or (b) the
|
||||
@@ -647,6 +1030,7 @@ impl AndroidVoiceUnit {
|
||||
output_gain: Arc<AtomicU32>,
|
||||
output_muted: Arc<AtomicBool>,
|
||||
scratch: Arc<Mutex<Vec<f32>>>,
|
||||
render_reference: Arc<RenderReferenceBuffer>,
|
||||
) -> Result<AudioStreamAsync<OboeOutput, OutputCallback>, BackendError> {
|
||||
let cb = OutputCallback {
|
||||
handler,
|
||||
@@ -654,6 +1038,7 @@ impl AndroidVoiceUnit {
|
||||
output_muted,
|
||||
event_tx: event_tx.clone(),
|
||||
scratch,
|
||||
render_reference,
|
||||
};
|
||||
let builder = AudioStreamBuilder::default()
|
||||
.set_direction::<OboeOutput>()
|
||||
@@ -806,6 +1191,7 @@ fn perf_from_oboe(p: PerformanceMode) -> AchievedPerformanceMode {
|
||||
match p {
|
||||
PerformanceMode::LowLatency => AchievedPerformanceMode::LowLatency,
|
||||
PerformanceMode::PowerSaving => AchievedPerformanceMode::PowerSaving,
|
||||
PerformanceMode::PowerSavingOffloaded => AchievedPerformanceMode::PowerSaving,
|
||||
PerformanceMode::None => AchievedPerformanceMode::None,
|
||||
}
|
||||
}
|
||||
@@ -1041,6 +1427,51 @@ fn release_hardware_effects_inner(handles: &mut HardwareEffectHandles) {
|
||||
}
|
||||
}
|
||||
|
||||
// --- Process-global BackendEvent sender for JNI callbacks --------
|
||||
//
|
||||
// Kotlin-side listeners (audio focus, Bluetooth SCO, device route
|
||||
// changes) need to publish events into the Rust engine's event
|
||||
// channel. Since the engine's `BackendEventTx` is created at voice
|
||||
// start, we store it here as a process-global so the JNI callbacks
|
||||
// can reach it without holding a direct Rust reference.
|
||||
//
|
||||
// Cleared on voice stop; the Kotlin listeners are idempotent when
|
||||
// no sender is registered (they log and continue).
|
||||
|
||||
static GLOBAL_BACKEND_EVENT_TX: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<tokio::sync::mpsc::UnboundedSender<BackendEvent>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
fn global_event_tx_slot(
|
||||
) -> &'static std::sync::Mutex<Option<tokio::sync::mpsc::UnboundedSender<BackendEvent>>> {
|
||||
GLOBAL_BACKEND_EVENT_TX.get_or_init(|| std::sync::Mutex::new(None))
|
||||
}
|
||||
|
||||
pub(crate) fn register_global_event_sender(tx: BackendEventTx) {
|
||||
if let Ok(mut g) = global_event_tx_slot().lock() {
|
||||
*g = Some(tx);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn clear_global_event_sender() {
|
||||
if let Ok(mut g) = global_event_tx_slot().lock() {
|
||||
*g = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn try_send_backend_event(event: BackendEvent) {
|
||||
if let Ok(g) = global_event_tx_slot().lock() {
|
||||
if let Some(tx) = g.as_ref() {
|
||||
if tx.send(event).is_err() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
"android: global BackendEvent channel closed; event dropped"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- SDD-115 foreground-service JNI helpers ----------------------
|
||||
//
|
||||
// The Kotlin class `AndroidVoiceForegroundService` (Wave 2B-2)
|
||||
@@ -1174,3 +1605,158 @@ fn load_app_class<'local>(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- SDD-109 / SDD-110 JNI callbacks: focus + SCO events ----------
|
||||
//
|
||||
// Kotlin-side OnAudioFocusChangeListener and BroadcastReceiver for
|
||||
// ACTION_SCO_AUDIO_STATE_UPDATED call these Rust entry points via
|
||||
// JNI. Each function marshals the platform event into a BackendEvent
|
||||
// and posts it through the global event sender registered by the
|
||||
// engine at voice start.
|
||||
//
|
||||
// SDD-115 callback safety: every entry point is wrapped in
|
||||
// catch_unwind so a panic in the Rust engine can never unwind
|
||||
// into the JVM.
|
||||
|
||||
/// SDD-109: audio focus change published by Kotlin's
|
||||
/// `AndroidAudioFocusController`. `state` is the `focusChange`
|
||||
/// value from `OnAudioFocusChangeListener`.
|
||||
///
|
||||
/// Symbol naming: JNI function declared in
|
||||
/// `app.chanora.chanora_flutter.AndroidAudioFocusController`.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_app_chanora_chanora_1flutter_AndroidAudioFocusController_publishFocusChange<
|
||||
'local,
|
||||
>(
|
||||
_env: jni::JNIEnv<'local>,
|
||||
_class: jni::objects::JClass<'local>,
|
||||
state: jni::sys::jint,
|
||||
) {
|
||||
let _ = catch_unwind(AssertUnwindSafe(|| {
|
||||
// AUDIOFOCUS_LOSS = -1, LOSS_TRANSIENT = -2, LOSS_TRANSIENT_CAN_DUCK = -3,
|
||||
// GAIN = 1 (AudioManager.AUDIOFOCUS_REQUEST_GRANTED is also 1, but we only
|
||||
// call this from the listener callback so the values are well-known).
|
||||
let event = match state {
|
||||
-1 => BackendEvent::FocusLost,
|
||||
-2 => BackendEvent::FocusTransient,
|
||||
-3 => BackendEvent::FocusTransientCanDuck,
|
||||
1 | 2 | 3 | 4 => BackendEvent::FocusGain,
|
||||
_ => {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
state,
|
||||
"android: unknown audio focus change value; treating as FocusLost"
|
||||
);
|
||||
BackendEvent::FocusLost
|
||||
}
|
||||
};
|
||||
try_send_backend_event(event);
|
||||
}));
|
||||
}
|
||||
|
||||
/// SDD-110: Bluetooth SCO state change published by Kotlin's
|
||||
/// `AndroidBluetoothScoController`. `state` is the `STATE`
|
||||
/// value from `ACTION_SCO_AUDIO_STATE_UPDATED`:
|
||||
/// - `ACTION_SCO_AUDIO_STATE_UPDATED` is always fired with `EXTRA_SCO_AUDIO_STATE`
|
||||
/// - `SCO_STATE_CONNECTING = 0`, `SCO_STATE_CONNECTED = 1`, `SCO_STATE_DISCONNECTED = 2`
|
||||
///
|
||||
/// Symbol naming: JNI function declared in
|
||||
/// `app.chanora.chanora_flutter.AndroidBluetoothScoController`.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_app_chanora_chanora_1flutter_AndroidBluetoothScoController_publishScoStateChange<
|
||||
'local,
|
||||
>(
|
||||
_env: jni::JNIEnv<'local>,
|
||||
_class: jni::objects::JClass<'local>,
|
||||
state: jni::sys::jint,
|
||||
) {
|
||||
let _ = catch_unwind(AssertUnwindSafe(|| {
|
||||
try_send_backend_event(BackendEvent::BluetoothScoStateChanged(state));
|
||||
}));
|
||||
}
|
||||
|
||||
/// SDD-115 integration: start the Android audio focus listener.
|
||||
/// Called by the engine after the voice unit is started. Uses JNI
|
||||
/// to invoke `AndroidAudioFocusController.start(Context)`.
|
||||
pub fn chanora_android_request_audio_focus() -> bool {
|
||||
call_static_void_context(
|
||||
"app/chanora/chanora_flutter/AndroidAudioFocusController",
|
||||
"start",
|
||||
)
|
||||
}
|
||||
|
||||
/// SDD-115 integration: stop the Android audio focus listener.
|
||||
/// Called by the engine on voice stop.
|
||||
pub fn chanora_android_abandon_audio_focus() -> bool {
|
||||
call_static_void_context(
|
||||
"app/chanora/chanora_flutter/AndroidAudioFocusController",
|
||||
"stop",
|
||||
)
|
||||
}
|
||||
|
||||
/// SDD-115 integration: start Bluetooth SCO.
|
||||
/// Called by the engine after the voice unit is started.
|
||||
pub fn chanora_android_start_bluetooth_sco() -> bool {
|
||||
call_static_void_context(
|
||||
"app/chanora/chanora_flutter/AndroidBluetoothScoController",
|
||||
"start",
|
||||
)
|
||||
}
|
||||
|
||||
/// SDD-115 integration: stop Bluetooth SCO.
|
||||
/// Called by the engine on voice stop.
|
||||
pub fn chanora_android_stop_bluetooth_sco() -> bool {
|
||||
call_static_void_context(
|
||||
"app/chanora/chanora_flutter/AndroidBluetoothScoController",
|
||||
"stop",
|
||||
)
|
||||
}
|
||||
|
||||
fn call_static_void_context(fqcn: &str, method: &str) -> bool {
|
||||
use jni::objects::{JObject, JValue};
|
||||
let ctx = ndk_context::android_context();
|
||||
if ctx.vm().is_null() || ctx.context().is_null() {
|
||||
warn!(
|
||||
target: "chanora_audio",
|
||||
class = fqcn,
|
||||
method,
|
||||
"android: ndk_context not initialised; call skipped"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
let jvm = match unsafe { jni::JavaVM::from_raw(ctx.vm() as *mut _) } {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: JavaVM::from_raw failed");
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let mut env = match jvm.attach_current_thread() {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: attach_current_thread failed");
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let context_obj = unsafe { JObject::from_raw(ctx.context() as jni::sys::jobject) };
|
||||
let class = match load_app_class(&mut env, &context_obj, fqcn) {
|
||||
Some(c) => c,
|
||||
None => return false,
|
||||
};
|
||||
match env.call_static_method(
|
||||
&class,
|
||||
method,
|
||||
"(Landroid/content/Context;)V",
|
||||
&[JValue::Object(&context_obj)],
|
||||
) {
|
||||
Ok(_) => {
|
||||
info!(target: "chanora_audio", class = fqcn, method, "android: dispatched");
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = env.exception_clear();
|
||||
warn!(target: "chanora_audio", error = %e, class = fqcn, method, "android: static call failed");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user