feat(voice): harden Android audio and channel joins

2026-05-19 01:58:07 +09:00
parent 29a553d4e1
commit 8c253f1d4d
23 changed files with 2948 additions and 363 deletions
@@ -36,11 +36,17 @@
 //!    engine-state mutation happens off the audio thread (SDD-115).

 #![cfg(target_os = "android")]
-#![allow(dead_code)]

 use std::panic::{catch_unwind, AssertUnwindSafe};
+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
+use std::sync::{Arc, Mutex};

-use tracing::{info, warn};
+use audiopus::coder::Encoder as OpusEncoder;
+use audiopus::{
+    Application as OpusApp, Bitrate as OpusBitrate, Channels as OpusChannels,
+    SampleRate as OpusSampleRate,
+};
+use tracing::{debug, info, warn};

 use crate::mobile_voice_backend::{
    clear_android_audio_diagnostics, latency_tier_for, next_input_preset_after,
@@ -50,6 +56,10 @@ use crate::mobile_voice_backend::{
    BackendEventTx, EffectEngagement, EffectEngine, InputPresetChoice, MobileVoiceAudioBackend,
    SharingModeChoice,
 };
+use chanora_protocol::{AudioData, CodecType, OutAudio, OutPacket};
+use tsclientlib::audio::AudioHandler;
+
+use crate::{engine::SessionAudioId, AudioError};

 use tokio::sync::mpsc;

@@ -64,21 +74,127 @@ use oboe::{
 // `mobile_voice_backend` so the trait can expose `take_event_rx`
 // (SDD-111 item 1) cross-platform.

-// --- Empty I/O callbacks for the lifecycle skeleton (SDD-111) ----
+/// 20 ms at 48 kHz mono — one Opus frame's worth of samples.
+/// Matches the iOS and desktop constants; duplicated here so this
+/// module is fully self-contained and cfg-gate-clean.
+const FRAME_SAMPLES: usize = 960;
+
+/// Maximum size of an encoded Opus frame in bytes (RFC 6716 §3.2.1).
+const MAX_OPUS_FRAME: usize = 1275;
+
+// --- Capture state for Oboe input callback (SDD-111 / SDD-120) ----
 //
-// Audio data is plumbed through the existing engine paths
-// (cpal-shaped channels feeding the `AudioHandler` mix). The
-// callbacks here exist to (a) satisfy `oboe-rs`'s requirement that
-// each async stream have a callback, and (b) provide the seam where
-// the engine can later inject its capture / playback ring buffers.
-// They are deliberately panic-free: any error path logs through the
-// `tracing` macro and returns `DataCallbackResult::Continue`. A
-// disconnect / error is delivered out-of-band through the error
-// callback that `AudioStreamBuilder::set_error_callback` would
-// install (the safe wrapper exposes this via the callback's
-// `on_error_*` hooks).
+// Mirrors the iOS `IosCaptureState` and the cpal-side `CaptureState`.
+// Oboe delivers 48 kHz mono i16 PCM; we apply mic gain, accumulate to
+// FRAME_SAMPLES, encode to Opus 32 kbps (complexity 10, inband FEC, 5 % PLC),
+// and try-send the resulting packet on `voice_out_tx`.
+
+struct AndroidCaptureState {
+    encoder: OpusEncoder,
+    /// Accumulator for 48 kHz mono PCM. 2x capacity to absorb
+    /// cpal-style buffer-size jitter without reallocating.
+    pcm_accum: Vec<i16>,
+    opus_out: [u8; MAX_OPUS_FRAME],
+    voice_out_tx: mpsc::Sender<OutPacket>,
+    transmit_active: Arc<AtomicBool>,
+    frames_sent: Arc<AtomicU32>,
+    mic_gain: f32,
+}
+
+impl AndroidCaptureState {
+    fn new(
+        voice_out_tx: mpsc::Sender<OutPacket>,
+        transmit_active: Arc<AtomicBool>,
+        frames_sent: Arc<AtomicU32>,
+        mic_gain: f32,
+    ) -> Result<Self, AudioError> {
+        let mut encoder =
+            OpusEncoder::new(OpusSampleRate::Hz48000, OpusChannels::Mono, OpusApp::Voip)
+                .map_err(|e| AudioError::Opus(format!("encoder new (android): {e}")))?;
+        if let Err(e) = encoder.set_bitrate(audiopus::Bitrate::BitsPerSecond(32_000)) {
+            warn!(target: "chanora_audio", error = %e, "opus(android): set_bitrate(32000) failed");
+        }
+        if let Err(e) = encoder.set_complexity(10) {
+            warn!(target: "chanora_audio", error = %e, "opus(android): set_complexity(10) failed");
+        }
+        if let Err(e) = encoder.set_inband_fec(true) {
+            warn!(target: "chanora_audio", error = %e, "opus(android): set_inband_fec(true) failed");
+        }
+        if let Err(e) = encoder.set_packet_loss_perc(5) {
+            warn!(target: "chanora_audio", error = %e, "opus(android): set_packet_loss_perc(5) failed");
+        }
+        info!(
+            target: "chanora_audio",
+            bitrate_bps = 32_000,
+            complexity = 10,
+            inband_fec = true,
+            packet_loss_perc = 5,
+            "android Oboe opus encoder tuned for VoIP"
+        );
+        Ok(Self {
+            encoder,
+            pcm_accum: Vec::with_capacity(FRAME_SAMPLES * 2),
+            opus_out: [0u8; MAX_OPUS_FRAME],
+            voice_out_tx,
+            transmit_active,
+            frames_sent,
+            mic_gain,
+        })
+    }
+
+    /// Consume i16 mono frames from Oboe, accumulate to FRAME_SAMPLES,
+    /// encode + send when PTT is held. Oboe delivers at the device's
+    /// native sample rate (always 48 kHz for modern Android per SRS-210),
+    /// so no resampling is needed.
+    fn ingest(&mut self, samples: &[i16]) {
+        if !self.transmit_active.load(Ordering::Relaxed) {
+            self.pcm_accum.clear();
+            return;
+        }
+        // Mic-gain application.
+        if (self.mic_gain - 1.0).abs() < f32::EPSILON {
+            self.pcm_accum.extend_from_slice(samples);
+        } else {
+            let gain = self.mic_gain;
+            self.pcm_accum.extend(samples.iter().map(|&s| {
+                let scaled = (s as f32) * gain;
+                scaled.clamp(i16::MIN as f32, i16::MAX as f32) as i16
+            }));
+        }
+        // Drain complete 20 ms frames.
+        while self.pcm_accum.len() >= FRAME_SAMPLES {
+            let mut frame = [0i16; FRAME_SAMPLES];
+            frame.copy_from_slice(&self.pcm_accum[..FRAME_SAMPLES]);
+            self.pcm_accum.drain(..FRAME_SAMPLES);
+            match self.encoder.encode(&frame, &mut self.opus_out[..]) {
+                Ok(len) => {
+                    let packet = OutAudio::new(&AudioData::C2S {
+                        id: 0,
+                        codec: CodecType::OpusVoice,
+                        data: &self.opus_out[..len],
+                    });
+                    match self.voice_out_tx.try_send(packet) {
+                        Ok(()) => {
+                            self.frames_sent.fetch_add(1, Ordering::Relaxed);
+                        }
+                        Err(mpsc::error::TrySendError::Full(_)) => {
+                            warn!(target: "chanora_audio", "android Oboe: voice_out queue full; dropping frame");
+                        }
+                        Err(mpsc::error::TrySendError::Closed(_)) => {
+                            debug!(target: "chanora_audio", "android Oboe: voice_out closed; capture pipeline stopping");
+                        }
+                    }
+                }
+                Err(e) => {
+                    warn!(target: "chanora_audio", error = %e, "android Oboe opus encode failed");
+                }
+            }
+        }
+    }
+}

 struct InputCallback {
+    state: Arc<Mutex<AndroidCaptureState>>,
    event_tx: BackendEventTx,
 }

@@ -88,36 +204,37 @@ impl AudioInputCallback for InputCallback {
    fn on_audio_ready(
        &mut self,
        _stream: &mut dyn AudioInputStreamSafe,
-        _frames: &[i16],
+        frames: &[i16],
    ) -> DataCallbackResult {
-        // Catch panics so a logic bug in the future can't abort the
-        // process under `panic=abort`. The audio thread MUST NOT
-        // panic.
        let _ = catch_unwind(AssertUnwindSafe(|| {
-            // Engine wires real capture through the AudioHandler
-            // path; this seam is intentionally a no-op for now.
+            if let Ok(mut s) = self.state.lock() {
+                s.ingest(frames);
+            }
        }));
        DataCallbackResult::Continue
    }

    fn on_error_after_close(&mut self, _stream: &mut dyn AudioInputStreamSafe, error: oboe::Error) {
-        // Oboe reports `ErrorDisconnected` here on route loss.
-        // We never call back into the engine from this method;
-        // instead we marshal a `Disconnected` event.
        if matches!(error, oboe::Error::Disconnected) {
            let _ = self.event_tx.send(BackendEvent::Disconnected);
        } else {
-            warn!(
-                target: "chanora_audio",
-                error = ?error,
-                "android: input stream error_after_close"
-            );
+            warn!(target: "chanora_audio", error = ?error, "android: input stream error_after_close");
        }
    }
 }

+// --- Output callback wiring (SDD-111 / SDD-120) ----
+//
+// Mirrors the iOS VPIO render callback. Pulls mixed 48 kHz stereo f32
+// from `AudioHandler::fill_buffer`, applies output gain + mute, and
+// writes mono i16 to the Oboe output buffer.
+
 struct OutputCallback {
+    handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
+    output_gain: Arc<AtomicU32>,
+    output_muted: Arc<AtomicBool>,
    event_tx: BackendEventTx,
+    scratch: Arc<Mutex<Vec<f32>>>,
 }

 impl AudioOutputCallback for OutputCallback {
@@ -128,13 +245,45 @@ impl AudioOutputCallback for OutputCallback {
        _stream: &mut dyn AudioOutputStreamSafe,
        frames: &mut [i16],
    ) -> DataCallbackResult {
-        // Default to silence. The engine wires real playback through
-        // the existing AudioHandler path; this callback is a seam
-        // where a future commit replaces silence with a ring-buffer
-        // pull. Zeroing is panic-free and lock-free.
        let _ = catch_unwind(AssertUnwindSafe(|| {
-            for s in frames.iter_mut() {
-                *s = 0;
+            let needed = frames.len() * 2; // stereo
+            let scratch = &mut self.scratch.lock().unwrap();
+            if scratch.len() < needed {
+                scratch.resize(needed, 0.0);
+            } else {
+                for s in &mut scratch[..needed] {
+                    *s = 0.0;
+                }
+            }
+            // Non-blocking pull from AudioHandler (same pattern as iOS VPIO).
+            match self.handler.try_lock() {
+                Ok(mut h) => {
+                    let _ = h.fill_buffer(&mut scratch[..needed]);
+                }
+                Err(std::sync::TryLockError::WouldBlock) => {
+                    // scratch already zeroed above.
+                }
+                Err(std::sync::TryLockError::Poisoned(e)) => {
+                    warn!(target: "chanora_audio", "AudioHandler mutex poisoned: {}", e);
+                }
+            }
+            let gain = f32::from_bits(self.output_gain.load(Ordering::Relaxed));
+            let muted = self.output_muted.load(Ordering::Relaxed);
+            let mut peak: i16 = 0;
+            for (i, dst) in frames.iter_mut().enumerate() {
+                if muted {
+                    *dst = 0;
+                    continue;
+                }
+                let l = scratch[i * 2];
+                let r = scratch[i * 2 + 1];
+                let mono = (l + r) * 0.5 * gain;
+                let clamped = mono.clamp(-1.0, 1.0);
+                let sample = (clamped * i16::MAX as f32) as i16;
+                *dst = sample;
+                if sample.unsigned_abs() > peak.unsigned_abs() {
+                    peak = sample;
+                }
            }
        }));
        DataCallbackResult::Continue
@@ -148,15 +297,34 @@ impl AudioOutputCallback for OutputCallback {
        if matches!(error, oboe::Error::Disconnected) {
            let _ = self.event_tx.send(BackendEvent::Disconnected);
        } else {
-            warn!(
-                target: "chanora_audio",
-                error = ?error,
-                "android: output stream error_after_close"
-            );
+            warn!(target: "chanora_audio", error = ?error, "android: output stream error_after_close");
        }
    }
 }

+/// Bundle of engine-owned state shared with the Oboe audio callbacks.
+/// Mirrors the parameter set that iOS `IosVoiceUnit::start()` receives
+/// from the engine (SDD-120 amendment: Android Oboe-only audio path).
+pub struct VoiceAudioParams {
+    /// Opus-encoded voice packets sent on this channel toward the
+    /// protocol layer.
+    pub voice_out_tx: mpsc::Sender<OutPacket>,
+    /// PTT transmission gate — true when the user holds the PTT key.
+    pub transmit_active: Arc<AtomicBool>,
+    /// Counter incremented per encoded frame sent.
+    pub frames_sent: Arc<AtomicU32>,
+    /// Pre-encode amplitude scale (1.0 = unity).
+    pub mic_gain: f32,
+    /// AudioHandler that inbound解码+混合 feeds into; the Oboe output
+    /// callback pulls mixed stereo f32 from it.
+    pub handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
+    /// Master output gain (f32 bits stored in AtomicU32 for lock-free
+    /// cross-thread read from the realtime audio callback).
+    pub output_gain: Arc<AtomicU32>,
+    /// True = output silence regardless of incoming voice frames.
+    pub output_muted: Arc<AtomicBool>,
+}
+
 // --- The backend itself ------------------------------------------

 /// Android voice-audio backend (SDD-111). Owns one input + one
@@ -199,9 +367,35 @@ impl AndroidVoiceUnit {
    /// effects. The engine is expected to have already issued
    /// `setMode(MODE_IN_COMMUNICATION)` (SDD-108) per the SDD-115
    /// sequencing rules.
-    pub fn open(cfg: &AndroidVoiceStreamConfig) -> Result<Self, BackendError> {
+    ///
+    /// `params` bundles the engine-owned state shared with the Oboe
+    /// audio callbacks (SDD-120 amendment: Android Oboe-only audio
+    /// path — capture pipeline, playback pull, and PTT gate).
+    #[allow(clippy::too_many_arguments)]
+    pub fn open(
+        cfg: &AndroidVoiceStreamConfig,
+        params: VoiceAudioParams,
+    ) -> Result<Self, BackendError> {
        let (event_tx, event_rx) = mpsc::unbounded_channel();

+        // SDD-120: build the capture state that the Oboe input callback
+        // will own via Arc<Mutex>.  Same Opus VoIP tuning as iOS and
+        // desktop (32 kbps, complexity 10, inband FEC, 5 % PLC).
+        let capture_state = Arc::new(Mutex::new(
+            AndroidCaptureState::new(
+                params.voice_out_tx,
+                params.transmit_active,
+                params.frames_sent,
+                params.mic_gain,
+            )
+            .map_err(|e| BackendError::OpenFailed(format!("capture state init: {e}")))?,
+        ));
+
+        // Scratch buffer for the output callback (realtime-safe
+        // pre-allocation).  8192 floats covers the largest practical
+        // burst size at 48 kHz with headroom.
+        let scratch = Arc::new(Mutex::new(Vec::with_capacity(8192)));
+
        // --- Open input stream (SDD-112) ---------------------------
        let mut input_builder = AudioStreamBuilder::default()
            .set_direction::<OboeInput>()
@@ -226,6 +420,7 @@ impl AndroidVoiceUnit {
            .set_usage(Usage::VoiceCommunication);

        let input_cb = InputCallback {
+            state: capture_state.clone(),
            event_tx: event_tx.clone(),
        };
        let input_builder = input_builder.set_callback(input_cb);
@@ -242,7 +437,7 @@ impl AndroidVoiceUnit {
                    error = ?e,
                    "android: primary input stream open failed; entering fallback ladder"
                );
-                Self::open_input_fallback(cfg, &event_tx)?
+                Self::open_input_fallback(cfg, &event_tx, capture_state.clone())?
            }
        };

@@ -279,7 +474,11 @@ impl AndroidVoiceUnit {
            .set_content_type(oboe::ContentType::Speech);

        let output_cb = OutputCallback {
+            handler: params.handler.clone(),
+            output_gain: params.output_gain.clone(),
+            output_muted: params.output_muted.clone(),
            event_tx: event_tx.clone(),
+            scratch: scratch.clone(),
        };
        let output_builder = output_builder.set_callback(output_cb);

@@ -291,7 +490,14 @@ impl AndroidVoiceUnit {
                    error = ?e,
                    "android: primary output stream open failed; retrying with Shared sharing mode"
                );
-                Self::open_output_fallback(cfg, &event_tx)?
+                Self::open_output_fallback(
+                    cfg,
+                    &event_tx,
+                    params.handler.clone(),
+                    params.output_gain.clone(),
+                    params.output_muted.clone(),
+                    scratch.clone(),
+                )?
            }
        };

@@ -390,6 +596,7 @@ impl AndroidVoiceUnit {
    fn open_input_fallback(
        cfg: &AndroidVoiceStreamConfig,
        event_tx: &BackendEventTx,
+        capture_state: Arc<Mutex<AndroidCaptureState>>,
    ) -> Result<AudioStreamAsync<OboeInput, InputCallback>, BackendError> {
        // SDD-112 items 6 & 7: explore (preset × sharing) independently
        // via the pure helpers in `mobile_voice_backend`. Primary
@@ -425,6 +632,7 @@ impl AndroidVoiceUnit {
                    SharingModeChoice::Shared => SharingMode::Shared,
                };
                let cb = InputCallback {
+                    state: capture_state.clone(),
                    event_tx: event_tx.clone(),
                };
                let builder = AudioStreamBuilder::default()
@@ -460,9 +668,17 @@ impl AndroidVoiceUnit {
    fn open_output_fallback(
        cfg: &AndroidVoiceStreamConfig,
        event_tx: &BackendEventTx,
+        handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
+        output_gain: Arc<AtomicU32>,
+        output_muted: Arc<AtomicBool>,
+        scratch: Arc<Mutex<Vec<f32>>>,
    ) -> Result<AudioStreamAsync<OboeOutput, OutputCallback>, BackendError> {
        let cb = OutputCallback {
+            handler,
+            output_gain,
+            output_muted,
            event_tx: event_tx.clone(),
+            scratch,
        };
        let builder = AudioStreamBuilder::default()
            .set_direction::<OboeOutput>()
@@ -648,7 +864,9 @@ fn attach_hardware_effects(
    // SDD-115 callback safety: even on the (assumed) non-realtime
    // open/close paths, wrap the JNI body in `catch_unwind` so a
    // panic during teardown cannot unwind into the JVM.
-    let result = catch_unwind(AssertUnwindSafe(|| attach_hardware_effects_inner(session_id, effects)));
+    let result = catch_unwind(AssertUnwindSafe(|| {
+        attach_hardware_effects_inner(session_id, effects)
+    }));
    match result {
        Ok(h) => h,
        Err(_) => {
@@ -690,13 +908,28 @@ fn attach_hardware_effects_inner(

    let mut handles = HardwareEffectHandles::default();
    if effects.aec {
-        handles.aec = create_effect(&mut env, "android/media/audiofx/AcousticEchoCanceler", session_id, "AEC");
+        handles.aec = create_effect(
+            &mut env,
+            "android/media/audiofx/AcousticEchoCanceler",
+            session_id,
+            "AEC",
+        );
    }
    if effects.noise_suppression {
-        handles.ns = create_effect(&mut env, "android/media/audiofx/NoiseSuppressor", session_id, "NS");
+        handles.ns = create_effect(
+            &mut env,
+            "android/media/audiofx/NoiseSuppressor",
+            session_id,
+            "NS",
+        );
    }
    if effects.agc {
-        handles.agc = create_effect(&mut env, "android/media/audiofx/AutomaticGainControl", session_id, "AGC");
+        handles.agc = create_effect(
+            &mut env,
+            "android/media/audiofx/AutomaticGainControl",
+            session_id,
+            "AGC",
+        );
    }
    handles
 }