edison/chanora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): harden Android audio and channel joins

Browse Source
This commit is contained in:
Edison Jwa
2026-05-19 01:58:07 +09:00
parent 29a553d4e1
commit 8c253f1d4d
23 changed files with 2948 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/chanora_audio/src/engine.rs
+203 -190
View File
@@ -8,9 +8,9 @@
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
#[cfg(not(target_os = "ios"))]
#[cfg(all(not(target_os = "ios"), not(target_os = "android")))]
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
#[cfg(not(target_os = "ios"))]
#[cfg(all(not(target_os = "ios"), not(target_os = "android")))]
use cpal::{SampleFormat, SizedSample};
use tokio::sync::mpsc;
use tracing::{debug, info};
@@ -18,12 +18,15 @@ use tracing::{debug, info};
// playback paths (`build_input_stream`, `build_output_stream`,
// `try_open_capture` log lines). Cfg-gate the imports too so iOS
// builds don't carry an unused-imports warning.
#[cfg(not(target_os = "ios"))]
#[cfg(all(not(target_os = "ios"), not(target_os = "android")))]
use tracing::{error, warn};
#[cfg(not(target_os = "ios"))]
#[cfg(target_os = "android")]
use tracing::warn;
#[cfg(all(not(target_os = "ios"), not(target_os = "android")))]
use audiopus::coder::Encoder as OpusEncoder;
#[cfg(not(target_os = "ios"))]
#[cfg(all(not(target_os = "ios"), not(target_os = "android")))]
use audiopus::{
Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
SampleRate as OpusSampleRate,
@@ -38,7 +41,7 @@ use tsclientlib::audio::AudioHandler;
// iOS too, and `OutPacket` flows out of the capture pipeline once
// commit 3 lands. Cfg-gate the cpal-only ones to keep iOS warnings
// clean.
#[cfg(not(target_os = "ios"))]
#[cfg(all(not(target_os = "ios"), not(target_os = "android")))]
use chanora_protocol::{AudioData, CodecType, OutAudio};
use chanora_protocol::{InboundVoice, OutPacket};
@@ -136,11 +139,15 @@ pub struct AudioEngine {
// VoiceProcessingIO AudioUnit hosts mic + speaker) — cpal is
// unused on iOS for the reasons documented in
// `ios_voice_unit.rs`.
#[cfg(not(target_os = "ios"))]
#[cfg(all(not(target_os = "ios"), not(target_os = "android")))]
_input_stream: Mutex<Option<cpal::Stream>>,
#[cfg(target_os = "linux")]
_output_stream: Mutex<Option<crate::sdl_output::SdlOutput>>,
#[cfg(all(not(target_os = "linux"), not(target_os = "ios")))]
#[cfg(all(
not(target_os = "linux"),
not(target_os = "ios"),
not(target_os = "android")
))]
_output_stream: Mutex<Option<cpal::Stream>>,
#[cfg(target_os = "ios")]
_ios_voice_unit: Mutex<Option<crate::ios_voice_unit::IosVoiceUnit>>,
@@ -236,7 +243,11 @@ impl AudioEngine {
{
return Self::start_with_gate_ios(cfg, voice_out_tx, voice_in_rx, transmit_gate);
}
#[cfg(not(target_os = "ios"))]
#[cfg(target_os = "android")]
{
return Self::start_with_gate_android(cfg, voice_out_tx, voice_in_rx, transmit_gate);
}
#[cfg(not(any(target_os = "ios", target_os = "android")))]
{
Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate)
}
@@ -247,7 +258,7 @@ impl AudioEngine {
/// at the top of `start_with_gate` without dragging a 200-line
/// cfg-gated block. Body is the pre-iOS-port code, unchanged
/// except for the new function name + signature.
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
fn start_with_gate_cpal(
cfg: AudioEngineConfig,
voice_out_tx: mpsc::Sender<OutPacket>,
@@ -313,153 +324,6 @@ impl AudioEngine {
),
}
// SDD-115 lifecycle sequencing on engine start. Forward order:
// 1) bridge -> engine receives voice_join (here we are
// already inside `start`, the engine-side trigger).
// 2) start the Android voice foreground service so that
// the platform records the microphone capture under
// `foregroundServiceType="microphone"` (SDD-107 + SRS-215).
// 3) open the AAudio voice streams (SDD-111 + SDD-112).
// 4) engage `MODE_IN_COMMUNICATION` (SDD-108).
// 5) bind hardware effects (SDD-113) — performed inside
// `AndroidVoiceUnit::open()` once the input stream has a
// session id.
// The reverse order on engine drop is enforced by the field
// drop order (`_android_voice_unit` is dropped before the
// engine returns; `close()` is invoked from its Drop impl).
#[cfg(target_os = "android")]
let mut audio_mode_stack = crate::mode_stack::ModeStack::new();
#[cfg(target_os = "android")]
let _android_voice_unit = {
if cfg.mobile_voice_preset {
// Step 2: foreground service.
if crate::android_voice_unit::chanora_android_start_voice_service() {
info!(
target: "chanora_audio",
"android: voice foreground service start dispatched (SDD-115)"
);
} else {
warn!(
target: "chanora_audio",
"android: foreground service start failed; capture may be denied in background (SDD-115)"
);
}
// Step 4 (mode engage) BEFORE Step 5 (effect bind);
// hardware-effect routing only engages reliably under
// MODE_IN_COMMUNICATION (SDD-113 item 6 / SDD-115).
//
// SDD-108 §1/§2: route through `ModeStack` so the
// 0 → 1 transition snapshots the prior platform mode
// (via `android_get_audio_mode`) and only that
// transition writes `MODE_IN_COMMUNICATION` via
// `android_set_audio_mode`. P0 only ever observes
// refcount {0, 1} per SRS-189 but the composition
// model is in place for P1.
match android_get_audio_mode() {
Ok(prior_now) => {
let outcome = audio_mode_stack.acquire(prior_now);
if let crate::mode_stack::ModeAcquire::FirstAcquire { prior } = outcome {
match android_set_audio_mode(ANDROID_MODE_IN_COMMUNICATION) {
Ok(()) => info!(
target: "chanora_audio",
prior_mode = prior,
"android: AudioManager mode set to MODE_IN_COMMUNICATION (SDD-108)"
),
Err(e) => {
// SDD-108 §5: setMode failed AFTER
// the 0 → 1 ModeStack transition.
// Roll the stack back so refcount
// returns to 0 and the snapshot is
// cleared; otherwise a future
// release would issue an
// unmatched setMode(prior) against
// a system that never had its mode
// changed by us.
warn!(
target: "chanora_audio",
error = %e,
prior_mode = prior,
"android: setMode failed; rolling back ModeStack acquire (SDD-108 §5)"
);
let _ = audio_mode_stack.release();
}
}
}
}
Err(e) => warn!(
target: "chanora_audio",
error = %e,
"android: AudioManager.getMode failed; skipping mode engage (SDD-108)"
),
}
// Step 3 + 5: open streams (SDD-111/112) and bind
// hardware effects (SDD-113). Failure here is logged
// and the engine continues with software AEC/NS/AGC
// via the existing engine path; the cpal data path
// remains the in-flight carrier.
//
// The prior silent-no-op log line at this site
// ("engagement depends on device AEC/NS support
// under MODE_IN_COMMUNICATION") is removed: the
// AndroidVoiceUnit either succeeds in engaging
// hardware effects (SDD-113) or logs the per-effect
// fallback, so engagement is now observable rather
// than rationalised.
let cfg_av = crate::mobile_voice_backend::AndroidVoiceStreamConfig {
effects: cfg.effects,
..Default::default()
};
match crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av) {
Ok(mut unit) => {
use crate::mobile_voice_backend::MobileVoiceAudioBackend;
if let Err(e) = unit.start() {
warn!(
target: "chanora_audio",
error = %e,
"android: AndroidVoiceUnit::start failed — cpal path remains active (SDD-115)"
);
}
Some(unit)
}
Err(e) => {
warn!(
target: "chanora_audio",
error = %e,
"android: AndroidVoiceUnit::open failed — software AEC/NS/AGC fallback engages (SDD-111/SDD-113)"
);
None
}
}
} else {
None
}
};
#[cfg(target_os = "ios")]
{
if cfg.mobile_voice_preset {
// iOS AVAudioSession configuration is performed
// Swift-side in `apps/chanora_flutter/ios/Runner/
// AppDelegate.swift::application(_:didFinishLaunching\
// WithOptions:)` BEFORE Flutter starts its audio
// pipeline. The category/mode set there
// (`.playAndRecord` + `.voiceChat`,
// `defaultToSpeaker | allowBluetooth |
// allowBluetoothA2DP`) is the recommended iOS shape
// for voice clients and engages on-device AEC / NS
// routing where supported. cpal's CoreAudio
// backend then opens its streams against that
// session and inherits the routing. Logging here
// just records that the engine-start path
// acknowledges the request; the actual session
// mutation lives in Swift because it must happen
// before Dart loads.
info!(
target: "chanora_audio",
"ios: voice-chat session mode requested — AVAudioSession configured in AppDelegate"
);
}
}
let transmit_flag_for_capture = transmit_gate.flag_arc();
let frames_sent = Arc::new(AtomicU32::new(0));
let frames_received = Arc::new(AtomicU32::new(0));
@@ -644,9 +508,158 @@ impl AudioEngine {
output_muted,
_input_stream: Mutex::new(input_stream),
_output_stream: Mutex::new(Some(output_stream)),
#[cfg(target_os = "android")]
_android_voice_unit: Mutex::new(_android_voice_unit),
#[cfg(target_os = "android")]
shutdown_tx: Some(shutdown_tx),
capture_active,
ptt_watchdog,
})
}
#[cfg(target_os = "android")]
fn start_with_gate_android(
cfg: AudioEngineConfig,
voice_out_tx: mpsc::Sender<OutPacket>,
mut voice_in_rx: mpsc::Receiver<InboundVoice>,
transmit_gate: crate::ptt::AudioTransmitGate,
) -> Result<Self, AudioError> {
use crate::mobile_voice_backend::{BackendEvent, MobileVoiceAudioBackend};
info!(target: "chanora_audio", "starting audio engine: Android Oboe backend");
let transmit_flag_for_capture = transmit_gate.flag_arc();
let frames_sent = Arc::new(AtomicU32::new(0));
let frames_received = Arc::new(AtomicU32::new(0));
let output_gain = Arc::new(AtomicU32::new(1.0_f32.to_bits()));
let output_muted = Arc::new(AtomicBool::new(false));
let audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>> =
Arc::new(Mutex::new(AudioHandler::new()));
if !cfg.mobile_voice_preset {
return Err(AudioError::Backend(
"android: mobile_voice_preset=false is unsupported for Oboe-only backend"
.to_string(),
));
}
let mut audio_mode_stack = crate::mode_stack::ModeStack::new();
if crate::android_voice_unit::chanora_android_start_voice_service() {
info!(
target: "chanora_audio",
"android: voice foreground service start dispatched (SDD-115)"
);
} else {
warn!(
target: "chanora_audio",
"android: foreground service start failed; capture may be denied in background (SDD-115)"
);
}
match android_get_audio_mode() {
Ok(prior_now) => {
let outcome = audio_mode_stack.acquire(prior_now);
if let crate::mode_stack::ModeAcquire::FirstAcquire { prior } = outcome {
match android_set_audio_mode(ANDROID_MODE_IN_COMMUNICATION) {
Ok(()) => info!(
target: "chanora_audio",
prior_mode = prior,
"android: AudioManager mode set to MODE_IN_COMMUNICATION (SDD-108)"
),
Err(e) => {
warn!(
target: "chanora_audio",
error = %e,
prior_mode = prior,
"android: setMode failed; rolling back ModeStack acquire (SDD-108 §5)"
);
let _ = audio_mode_stack.release();
}
}
}
}
Err(e) => warn!(
target: "chanora_audio",
error = %e,
"android: AudioManager.getMode failed; skipping mode engage (SDD-108)"
),
}
let cfg_av = crate::mobile_voice_backend::AndroidVoiceStreamConfig {
effects: cfg.effects,
..Default::default()
};
let params = crate::android_voice_unit::VoiceAudioParams {
voice_out_tx,
transmit_active: transmit_flag_for_capture,
frames_sent: frames_sent.clone(),
mic_gain: cfg.mic_gain,
handler: audio_handler.clone(),
output_gain: output_gain.clone(),
output_muted: output_muted.clone(),
};
let mut android_voice_unit =
crate::android_voice_unit::AndroidVoiceUnit::open(&cfg_av, params).map_err(|e| {
AudioError::Backend(format!("android: failed to open Oboe voice unit: {e}"))
})?;
if let Err(e) = android_voice_unit.start() {
return Err(AudioError::Backend(format!(
"android: failed to start Oboe voice unit: {e}"
)));
}
if let Some(mut event_rx) = android_voice_unit.take_event_rx() {
tokio::spawn(async move {
while let Some(event) = event_rx.recv().await {
if let BackendEvent::Disconnected = event {
warn!(
target: "chanora_audio",
"android: backend disconnected event received; stream reconnect requires session restart"
);
}
}
});
}
let capture_active = true;
let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
let handler_for_task = audio_handler.clone();
let frames_received_for_task = frames_received.clone();
tokio::spawn(async move {
loop {
tokio::select! {
_ = &mut shutdown_rx => {
debug!(target: "chanora_audio", "inbound forwarder shutting down");
break;
}
item = voice_in_rx.recv() => {
match item {
Some(v) => {
let id = SessionAudioId(v.from_client);
let mut h = handler_for_task.lock().unwrap();
if let Err(e) = h.handle_packet(id, v.packet) {
debug!(target: "chanora_audio", error = %e, "decode failed");
} else {
frames_received_for_task.fetch_add(1, Ordering::Relaxed);
}
}
None => break,
}
}
}
}
});
let ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog> = None;
Ok(Self {
transmit_gate,
frames_sent,
frames_received,
output_gain,
output_muted,
_android_voice_unit: Mutex::new(Some(android_voice_unit)),
audio_mode_stack: Mutex::new(audio_mode_stack),
shutdown_tx: Some(shutdown_tx),
capture_active,
@@ -783,7 +796,7 @@ impl AudioEngine {
// audio. iOS collapses input + output into one
// `IosVoiceUnit` (see `ios_voice_unit.rs`); every other
// platform has separate cpal input + cpal/SDL output.
#[cfg(not(target_os = "ios"))]
#[cfg(all(not(target_os = "ios"), not(target_os = "android")))]
{
let _ = self._input_stream.lock().unwrap().take();
let _ = self._output_stream.lock().unwrap().take();
@@ -1023,7 +1036,7 @@ impl Drop for AudioEngine {
// ---------- Capture pipeline ----------
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
fn try_open_capture(
in_dev: &cpal::Device,
voice_out_tx: mpsc::Sender<OutPacket>,
@@ -1131,7 +1144,7 @@ fn try_open_capture(
Ok(stream)
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
struct CaptureState {
encoder: OpusEncoder,
in_sample_rate: u32,
@@ -1169,7 +1182,7 @@ struct CaptureState {
frame_scratch: Vec<f32>,
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
impl CaptureState {
fn new(
encoder: OpusEncoder,
@@ -1269,7 +1282,10 @@ impl CaptureState {
*s = -1.0;
}
}
match self.encoder.encode_float(&frame[..], &mut self.opus_out[..]) {
match self
.encoder
.encode_float(&frame[..], &mut self.opus_out[..])
{
Ok(len) => {
let packet = OutAudio::new(&AudioData::C2S {
id: 0,
@@ -1350,30 +1366,30 @@ impl CaptureState {
}
/// Per-sample format conversion to f32 in the range [-1.0, 1.0].
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
trait ToF32 {
fn to_f32_sample(self) -> f32;
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
impl ToF32 for f32 {
fn to_f32_sample(self) -> f32 {
self
}
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
impl ToF32 for i16 {
fn to_f32_sample(self) -> f32 {
f32::from(self) / f32::from(i16::MAX)
}
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
impl ToF32 for u16 {
fn to_f32_sample(self) -> f32 {
(f32::from(self) - f32::from(i16::MAX) - 1.0) / f32::from(i16::MAX)
}
}
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
fn build_input_stream<T>(
device: &cpal::Device,
config: &cpal::StreamConfig,
@@ -1401,7 +1417,7 @@ where
// ---------- Playback pipeline ----------
#[cfg(not(target_os = "linux"))]
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
fn build_output_stream<T>(
device: &cpal::Device,
config: &cpal::StreamConfig,
@@ -1590,7 +1606,7 @@ where
/// Resampler state carried across output cpal callbacks. See
/// `build_output_stream` for the rationale.
#[cfg(not(target_os = "linux"))]
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
struct PlaybackResampleState {
pos: f64,
last_l: f32,
@@ -1598,26 +1614,26 @@ struct PlaybackResampleState {
}
#[cfg(not(target_os = "linux"))]
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
trait FromF32 {
fn from_f32_sample(v: f32) -> Self;
}
#[cfg(not(target_os = "linux"))]
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
impl FromF32 for f32 {
fn from_f32_sample(v: f32) -> Self {
v
}
}
#[cfg(not(target_os = "linux"))]
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
impl FromF32 for i16 {
fn from_f32_sample(v: f32) -> Self {
(v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i16
}
}
#[cfg(not(target_os = "linux"))]
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
impl FromF32 for u16 {
fn from_f32_sample(v: f32) -> Self {
let s = (v.clamp(-1.0, 1.0) * f32::from(i16::MAX)) as i32;
@@ -1720,12 +1736,12 @@ where
let context_obj = unsafe { JObject::from_raw(ctx.context() as jni::sys::jobject) };
let service_name: JString = env.new_string("audio").map_err(|e| {
AudioModeError::MethodCallFailed {
method: "new_string",
detail: e.to_string(),
}
})?;
let service_name: JString =
env.new_string("audio")
.map_err(|e| AudioModeError::MethodCallFailed {
method: "new_string",
detail: e.to_string(),
})?;
let audio_manager = env
.call_method(
&context_obj,
@@ -1809,11 +1825,11 @@ pub fn android_set_audio_mode(mode: i32) -> Result<(), AudioModeError> {
// is not a supported external API. Only compiled on non-iOS targets
// because `CaptureState` itself is gated on `cfg(not(target_os = "ios"))`.
// ---------------------------------------------------------------------------
#[cfg(not(target_os = "ios"))]
#[cfg(not(any(target_os = "ios", target_os = "android")))]
#[doc(hidden)]
pub mod bench_seam {
use super::{
AtomicBool, AtomicU32, Arc, CaptureState, OpusApp, OpusChannels, OpusEncoder,
Arc, AtomicBool, AtomicU32, CaptureState, OpusApp, OpusChannels, OpusEncoder,
OpusSampleRate, OutPacket,
};
use tokio::sync::mpsc;
@@ -1840,12 +1856,9 @@ pub mod bench_seam {
/// resampler). `in_channels` selects the channel layout
/// (typically 1 or 2).
pub fn new(in_sample_rate: u32, in_channels: usize) -> Self {
let encoder = OpusEncoder::new(
OpusSampleRate::Hz48000,
OpusChannels::Mono,
OpusApp::Voip,
)
.expect("opus encoder init");
let encoder =
OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
.expect("opus encoder init");
let (tx, rx) = mpsc::channel::<OutPacket>(64);
let transmit_active = Arc::new(AtomicBool::new(true));
let frames_sent = Arc::new(AtomicU32::new(0));