feat: integrate chat voice and diagnostics client

2026-05-23 06:51:55 +09:00
parent 7e28791ec2
commit 7d5d8c2c90
93 changed files with 10273 additions and 3347 deletions
@@ -55,17 +55,98 @@ use audiopus::coder::Encoder as OpusEncoder;
 pub struct SessionAudioId(pub u64);

 /// Audio framing: 48 kHz mono, 20 ms = 960 samples per frame.
-/// These constants are framing invariants of the engine and are
-/// referenced from per-platform helpers (`try_open_capture` and
-/// the CaptureState on cpal platforms; `ios_voice_unit` on iOS once
-/// commits 3+4 land). The `allow(dead_code)` is here because in
-/// the current commit the iOS VPIO callbacks are still no-op stubs
-/// and don't reach these constants yet — they will in commit 3
-/// when the input callback wires into CaptureState.
-#[allow(dead_code)]
+#[cfg(all(
+    not(target_os = "ios"),
+    not(target_os = "macos"),
+    not(target_os = "android")
+))]
 const SAMPLE_RATE: u32 = 48_000;
-#[allow(dead_code)]
+#[cfg(all(
+    not(target_os = "ios"),
+    not(target_os = "macos"),
+    not(target_os = "android")
+))]
 const FRAME_SAMPLES: usize = 48_000 / 50; // 960
+
+/// List of available audio devices from the platform.
+#[derive(Debug, Clone)]
+pub struct AudioDeviceList {
+    /// Available input (capture) devices.
+    pub input_devices: Vec<AudioDeviceInfo>,
+    /// Available output (playback) devices.
+    pub output_devices: Vec<AudioDeviceInfo>,
+}
+
+/// Info about a single audio device.
+#[derive(Debug, Clone)]
+pub struct AudioDeviceInfo {
+    /// Human-readable device name from the OS.
+    pub name: String,
+    /// True if the OS reports this as the default device.
+    pub is_default: bool,
+}
+
+/// Enumerate available audio input and output devices.
+/// On mobile platforms (iOS, Android) returns an empty list because
+/// device selection is managed by the OS audio session.
+#[cfg(all(
+    not(target_os = "ios"),
+    not(target_os = "macos"),
+    not(target_os = "android")
+))]
+pub fn list_audio_devices() -> AudioDeviceList {
+    use cpal::traits::HostTrait;
+    let mut list = AudioDeviceList {
+        input_devices: Vec::new(),
+        output_devices: Vec::new(),
+    };
+    let Ok(host) = cpal::default_host() else {
+        return list;
+    };
+    let default_in = host.default_input_device();
+    let default_out = host.default_output_device();
+    if let Ok(devices) = host.input_devices() {
+        for d in devices {
+            let name = d
+                .description()
+                .map(|n| n.name().to_owned())
+                .unwrap_or_default();
+            if !name.is_empty() {
+                let is_default = default_in
+                    .as_ref()
+                    .is_some_and(|di| di.description().is_ok_and(|dn| dn.name() == name.as_str()));
+                list.input_devices
+                    .push(AudioDeviceInfo { name, is_default });
+            }
+        }
+    }
+    if let Ok(devices) = host.output_devices() {
+        for d in devices {
+            let name = d
+                .description()
+                .map(|n| n.name().to_owned())
+                .unwrap_or_default();
+            if !name.is_empty() {
+                let is_default = default_out
+                    .as_ref()
+                    .is_some_and(|di| di.description().is_ok_and(|dn| dn.name() == name.as_str()));
+                list.output_devices
+                    .push(AudioDeviceInfo { name, is_default });
+            }
+        }
+    }
+    list
+}
+
+#[cfg(any(target_os = "ios", target_os = "macos", target_os = "android"))]
+/// Enumerate available audio input and output devices.
+pub fn list_audio_devices() -> AudioDeviceList {
+    AudioDeviceList {
+        input_devices: Vec::new(),
+        output_devices: Vec::new(),
+    }
+}
+
 /// Engine configuration.
 #[derive(Clone)]
 pub struct AudioEngineConfig {
@@ -91,6 +172,12 @@ pub struct AudioEngineConfig {
    /// is rejected on Android because the P0 path intentionally has
    /// no generic mobile-audio fallback.
    pub mobile_voice_preset: bool,
+    /// Optional input device name override. When `None`, the system
+    /// default input device is used. Set to a device name from
+    /// [`list_audio_devices`] to pin a specific microphone.
+    pub input_device_name: Option<String>,
+    /// Optional output device name override.
+    pub output_device_name: Option<String>,
    /// Optional selector used by P1 VoiceActivity to publish VAD state.
    #[doc(hidden)]
    pub voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
@@ -103,6 +190,8 @@ impl std::fmt::Debug for AudioEngineConfig {
            .field("ptt_initial", &self.ptt_initial)
            .field("effects", &self.effects)
            .field("mobile_voice_preset", &self.mobile_voice_preset)
+            .field("input_device_name", &self.input_device_name)
+            .field("output_device_name", &self.output_device_name)
            .field(
                "voice_activity_selector",
                &self.voice_activity_selector.as_ref().map(|_| "present"),
@@ -118,6 +207,8 @@ impl Default for AudioEngineConfig {
            ptt_initial: false,
            effects: crate::AudioEffects::default(),
            mobile_voice_preset: true,
+            input_device_name: None,
+            output_device_name: None,
            voice_activity_selector: None,
        }
    }
@@ -145,7 +236,6 @@ pub struct AudioEngine {
    output_muted: Arc<AtomicBool>,
    audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
    audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
-    #[cfg(any(target_os = "ios", target_os = "macos"))]
    audio_handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
    #[cfg(any(target_os = "ios", target_os = "macos"))]
    voice_out_tx: mpsc::Sender<OutPacket>,
@@ -204,23 +294,6 @@ pub struct AudioEngine {
    /// denied microphone permission), PTT becomes a no-op and
    /// `frames_sent` stays at 0.
    capture_active: bool,
-    /// Missed-key-up watchdog (SDD-092). Dropping aborts the task.
-    /// The watchdog is independent of the PTT input backend — it
-    /// observes the gate directly. The platform input backend is
-    /// owned by `chanora_core::ptt::PttController` (SDD-088), not
-    /// by the engine.
-    ///
-    /// In the post-rc.7 architecture this field is unused: the
-    /// missed-key-up watchdog now lives on the session and
-    /// subscribes to `TransmitModeSelector::subscribe_ptt_held`
-    /// rather than the gate. Watching the gate caused the watchdog
-    /// to fire in Continuous mode (where the gate is intentionally
-    /// pinned to `true`) which clearing surfaced as the bug
-    /// "Continuous transmission disabled after some time". The
-    /// field stays here as `None` for now to preserve the existing
-    /// engine-stop teardown flow; a follow-up commit can remove it
-    /// entirely.
-    ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog>,
 }

 // cpal::Stream is not Send. We keep the engine pinned to the thread
@@ -241,7 +314,6 @@ unsafe impl Send for AudioEngine {}
 unsafe impl Sync for AudioEngine {}

 #[cfg(any(target_os = "ios", target_os = "macos"))]
-#[allow(dead_code)]
 enum IosVoiceBackend {
    Vpio(crate::ios_voice_unit::IosVoiceUnit),
    #[cfg(target_os = "ios")]
@@ -260,7 +332,9 @@ impl IosVoiceBackend {
        }
        #[cfg(target_os = "macos")]
        {
-            Ok(())
+            match self {
+                Self::Vpio(_unit) => Ok(()),
+            }
        }
    }

@@ -274,41 +348,23 @@ impl IosVoiceBackend {
        }
        #[cfg(target_os = "macos")]
        {
-            Ok(())
+            match self {
+                Self::Vpio(_unit) => Ok(()),
+            }
        }
    }
 }

 #[cfg(any(target_os = "ios", target_os = "macos"))]
-#[allow(clippy::too_many_arguments)]
 fn open_ios_voice_backend(
-    handler: Arc<Mutex<AudioHandler<SessionAudioId>>>,
-    output_gain: Arc<AtomicU32>,
-    output_muted: Arc<AtomicBool>,
-    voice_out_tx: mpsc::Sender<OutPacket>,
-    transmit_flag_for_capture: Arc<AtomicBool>,
-    frames_sent: Arc<AtomicU32>,
-    mic_gain: f32,
-    voice_activity_selector: Option<Arc<crate::TransmitModeSelector>>,
-    audio_processing_config: Arc<Mutex<crate::AudioProcessingConfig>>,
-    audio_processing_stats: Arc<crate::SharedAudioProcessingStats>,
+    params: crate::mobile_voice_backend::VoiceAudioParams,
 ) -> Result<IosVoiceBackend, AudioError> {
-    let _cfg = audio_processing_config.lock().unwrap().clone();
+    let _cfg = params.audio_processing_config.lock().unwrap().clone();
    #[cfg(target_os = "ios")]
    {
        if _cfg.ios_mode == crate::IosVoiceProcessingMode::SonoraExperimental {
-            match crate::ios_raw_unit::IosRawUnit::start(
-                handler.clone(),
-                output_gain.clone(),
-                output_muted.clone(),
-                voice_out_tx.clone(),
-                transmit_flag_for_capture.clone(),
-                frames_sent.clone(),
-                mic_gain,
-                voice_activity_selector.clone(),
-                audio_processing_config.clone(),
-                audio_processing_stats.clone(),
-            ) {
+            let raw_params = params.clone();
+            match crate::ios_raw_unit::IosRawUnit::start(raw_params) {
                Ok(unit) => {
                    info!(target: "chanora_audio", "ios: RemoteIO/WebRTC APM backend selected");
                    return Ok(IosVoiceBackend::Raw(unit));
@@ -324,18 +380,7 @@ fn open_ios_voice_backend(
        }
    }

-    let unit = crate::ios_voice_unit::IosVoiceUnit::start(
-        handler,
-        output_gain,
-        output_muted,
-        voice_out_tx,
-        transmit_flag_for_capture,
-        frames_sent,
-        mic_gain,
-        voice_activity_selector,
-        audio_processing_config,
-        audio_processing_stats,
-    )?;
+    let unit = crate::ios_voice_unit::IosVoiceUnit::start(params)?;
    Ok(IosVoiceBackend::Vpio(unit))
 }

@@ -361,23 +406,40 @@ impl AudioEngine {
        voice_in_rx: mpsc::Receiver<InboundVoice>,
        transmit_gate: crate::ptt::AudioTransmitGate,
    ) -> Result<Self, AudioError> {
-        #[allow(clippy::needless_return)]
-        // Apple platforms route to a separate backend (VoiceProcessingIO
-        // via coreaudio-rs) because cpal does not expose the native
-        // voice-processing AudioUnit controls Chanora needs for VoIP.
-        // Windows and Linux stay on the cpal / SDL flow below.
-        #[cfg(any(target_os = "ios", target_os = "macos"))]
-        {
-            return Self::start_with_gate_ios(cfg, voice_out_tx, voice_in_rx, transmit_gate);
-        }
-        #[cfg(target_os = "android")]
-        {
-            return Self::start_with_gate_android(cfg, voice_out_tx, voice_in_rx, transmit_gate);
-        }
-        #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
-        {
-            Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate)
-        }
+        Self::start_with_gate_platform(cfg, voice_out_tx, voice_in_rx, transmit_gate)
+    }
+
+    // Apple platforms route to a separate backend (VoiceProcessingIO
+    // via coreaudio-rs) because cpal does not expose the native
+    // voice-processing AudioUnit controls Chanora needs for VoIP.
+    #[cfg(any(target_os = "ios", target_os = "macos"))]
+    fn start_with_gate_platform(
+        cfg: AudioEngineConfig,
+        voice_out_tx: mpsc::Sender<OutPacket>,
+        voice_in_rx: mpsc::Receiver<InboundVoice>,
+        transmit_gate: crate::ptt::AudioTransmitGate,
+    ) -> Result<Self, AudioError> {
+        Self::start_with_gate_ios(cfg, voice_out_tx, voice_in_rx, transmit_gate)
+    }
+
+    #[cfg(target_os = "android")]
+    fn start_with_gate_platform(
+        cfg: AudioEngineConfig,
+        voice_out_tx: mpsc::Sender<OutPacket>,
+        voice_in_rx: mpsc::Receiver<InboundVoice>,
+        transmit_gate: crate::ptt::AudioTransmitGate,
+    ) -> Result<Self, AudioError> {
+        Self::start_with_gate_android(cfg, voice_out_tx, voice_in_rx, transmit_gate)
+    }
+
+    #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "android")))]
+    fn start_with_gate_platform(
+        cfg: AudioEngineConfig,
+        voice_out_tx: mpsc::Sender<OutPacket>,
+        voice_in_rx: mpsc::Receiver<InboundVoice>,
+        transmit_gate: crate::ptt::AudioTransmitGate,
+    ) -> Result<Self, AudioError> {
+        Self::start_with_gate_cpal(cfg, voice_out_tx, voice_in_rx, transmit_gate)
    }

    /// Non-Apple/non-Android implementation: cpal capture + (cpal | SDL2) output.
@@ -398,12 +460,45 @@ impl AudioEngine {
            host_id = ?host.id(),
            "starting audio engine: cpal host selected"
        );
-        let in_dev = host
-            .default_input_device()
-            .ok_or(AudioError::NoInputDevice)?;
-        let out_dev = host
-            .default_output_device()
-            .ok_or(AudioError::NoOutputDevice)?;
+
+        /// Helper: find a device by name, falling back to default.
+        fn find_device(
+            host: &cpal::Host,
+            default_fn: fn(&cpal::Host) -> Option<cpal::Device>,
+            all_fn: fn(&cpal::Host) -> Result<cpal::Devices, cpal::DevicesError>,
+            prefer: Option<&str>,
+        ) -> Option<cpal::Device> {
+            if let Some(name) = prefer {
+                if let Ok(devices) = all_fn(host) {
+                    for d in devices {
+                        let dn = d
+                            .description()
+                            .map(|n| n.name().to_owned())
+                            .unwrap_or_default();
+                        if dn == name {
+                            return Some(d);
+                        }
+                    }
+                }
+            }
+            default_fn(host)
+        }
+
+        let in_dev = find_device(
+            &host,
+            cpal::Host::default_input_device,
+            cpal::Host::input_devices,
+            cfg.input_device_name.as_deref(),
+        )
+        .ok_or(AudioError::NoInputDevice)?;
+
+        let out_dev = find_device(
+            &host,
+            cpal::Host::default_output_device,
+            cpal::Host::output_devices,
+            cfg.output_device_name.as_deref(),
+        )
+        .ok_or(AudioError::NoOutputDevice)?;

        info!(
            target: "chanora_audio",
@@ -611,19 +706,6 @@ impl AudioEngine {
            }
        });

-        // Select and arm the desktop PTT backend is no longer the
-        // engine's job (SDD-088). The PTT controller lives in
-        // `chanora_core::ptt::PttController`; the engine is
-        // responsible only for the cpal streams and the
-        // missed-key-up watchdog (SDD-092).
-
-        // The engine no longer spawns a watchdog against the
-        // gate (see comment on the `ptt_watchdog` field for the
-        // rationale). The session spawns the watchdog against the
-        // selector's `ptt_held` signal instead, so it never fires
-        // in Continuous mode.
-        let ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog> = None;
-
        Ok(Self {
            transmit_gate,
            frames_sent,
@@ -632,11 +714,11 @@ impl AudioEngine {
            output_muted,
            audio_processing_config,
            audio_processing_stats,
+            audio_handler,
            _input_stream: Mutex::new(input_stream),
            _output_stream: Mutex::new(Some(output_stream)),
            shutdown_tx: Some(shutdown_tx),
            capture_active,
-            ptt_watchdog,
        })
    }

@@ -716,7 +798,7 @@ impl AudioEngine {
            effects: cfg.effects,
            ..Default::default()
        };
-        let params = crate::android_voice_unit::VoiceAudioParams {
+        let params = crate::mobile_voice_backend::VoiceAudioParams {
            voice_out_tx,
            transmit_active: transmit_flag_for_capture,
            frames_sent: frames_sent.clone(),
@@ -841,8 +923,6 @@ impl AudioEngine {
            }
        });

-        let ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog> = None;
-
        Ok(Self {
            transmit_gate,
            frames_sent,
@@ -851,11 +931,11 @@ impl AudioEngine {
            output_muted,
            audio_processing_config,
            audio_processing_stats,
+            audio_handler,
            _android_voice_unit: Mutex::new(Some(android_voice_unit)),
            audio_mode_stack: Mutex::new(audio_mode_stack),
            shutdown_tx: Some(shutdown_tx),
            capture_active,
-            ptt_watchdog,
        })
    }

@@ -907,18 +987,19 @@ impl AudioEngine {

        // Construct the live iOS voice backend. Platform VPIO stays
        // the default shipping path; Sonora/RemoteIO remains opt-in.
-        let ios_voice_backend = open_ios_voice_backend(
-            audio_handler.clone(),
-            output_gain.clone(),
-            output_muted.clone(),
-            voice_out_tx_for_backend,
-            transmit_flag_for_capture,
-            frames_sent.clone(),
-            cfg.mic_gain,
-            cfg.voice_activity_selector.clone(),
-            audio_processing_config.clone(),
-            audio_processing_stats.clone(),
-        )?;
+        let ios_voice_backend =
+            open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams {
+                handler: audio_handler.clone(),
+                output_gain: output_gain.clone(),
+                output_muted: output_muted.clone(),
+                voice_out_tx: voice_out_tx_for_backend,
+                transmit_active: transmit_flag_for_capture,
+                frames_sent: frames_sent.clone(),
+                mic_gain: cfg.mic_gain,
+                voice_activity_selector: cfg.voice_activity_selector.clone(),
+                audio_processing_config: audio_processing_config.clone(),
+                audio_processing_stats: audio_processing_stats.clone(),
+            })?;

        // Capture is always considered active on iOS — VPIO's
        // input element is wired up by the AudioUnit itself, no
@@ -959,8 +1040,6 @@ impl AudioEngine {
            }
        });

-        let ptt_watchdog: Option<crate::ptt::MissedKeyUpWatchdog> = None;
-
        Ok(Self {
            transmit_gate,
            frames_sent,
@@ -976,7 +1055,6 @@ impl AudioEngine {
            _ios_voice_backend: Mutex::new(Some(ios_voice_backend)),
            shutdown_tx: Some(shutdown_tx),
            capture_active,
-            ptt_watchdog,
        })
    }

@@ -985,12 +1063,6 @@ impl AudioEngine {
        if let Some(tx) = self.shutdown_tx.take() {
            let _ = tx.send(());
        }
-        // The platform PTT backend is no longer owned by the
-        // engine (SDD-088); its lifecycle is managed by
-        // `chanora_core::ptt::PttController`. The engine only
-        // needs to abort its watchdog and drop the audio streams.
-        // Aborting the watchdog cancels its tokio task.
-        self.ptt_watchdog.take();
        // Drop the streams, which stops their callback threads.
        // Each platform has a slightly different backend; the
        // common contract is that dropping the wrapper stops
@@ -1088,18 +1160,18 @@ impl AudioEngine {
    pub fn ios_restart_voice_unit(&self) -> Result<(), AudioError> {
        #[cfg(any(target_os = "ios", target_os = "macos"))]
        {
-            let backend = open_ios_voice_backend(
-                self.audio_handler.clone(),
-                self.output_gain.clone(),
-                self.output_muted.clone(),
-                self.voice_out_tx.clone(),
-                self.transmit_gate.flag_arc(),
-                self.frames_sent.clone(),
-                self.mic_gain,
-                self.voice_activity_selector.clone(),
-                self.audio_processing_config.clone(),
-                self.audio_processing_stats.clone(),
-            )?;
+            let backend = open_ios_voice_backend(crate::mobile_voice_backend::VoiceAudioParams {
+                handler: self.audio_handler.clone(),
+                output_gain: self.output_gain.clone(),
+                output_muted: self.output_muted.clone(),
+                voice_out_tx: self.voice_out_tx.clone(),
+                transmit_active: self.transmit_gate.flag_arc(),
+                frames_sent: self.frames_sent.clone(),
+                mic_gain: self.mic_gain,
+                voice_activity_selector: self.voice_activity_selector.clone(),
+                audio_processing_config: self.audio_processing_config.clone(),
+                audio_processing_stats: self.audio_processing_stats.clone(),
+            })?;
            let mut guard = self._ios_voice_backend.lock().unwrap();
            *guard = Some(backend);
            Ok(())
@@ -1166,33 +1238,6 @@ impl AudioEngine {
        &self.transmit_gate
    }

-    /// Privacy-safe descriptor of the engine's PTT view. The
-    /// platform backend lives in `chanora_core::ptt::PttController`
-    /// (SDD-088); the engine itself no longer owns it. This getter
-    /// always returns the universal Focused fallback descriptor
-    /// and is retained only for legacy callers that constructed
-    /// engines directly without a controller (tests, headless
-    /// diagnostics).
-    pub fn ptt_descriptor(&self) -> crate::ptt::PttBackendDescriptor {
-        crate::ptt::PttBackendDescriptor::focused()
-    }
-
-    /// Legacy alias for [`Self::set_transmit_active`]. Retained so
-    /// the existing bridge `set_ptt` command and the existing
-    /// Flutter UI continue to compile during the v0.9.3 PTT
-    /// migration (SRS-201 splits the conceptual `ptt` flag into
-    /// `transmit_active` / `capture_active`).
-    #[doc(hidden)]
-    pub fn set_ptt(&self, active: bool) {
-        self.set_transmit_active(active);
-    }
-
-    /// Legacy alias for [`Self::transmit_active`].
-    #[doc(hidden)]
-    pub fn ptt(&self) -> bool {
-        self.transmit_active()
-    }
-
    /// True if the capture stream opened. When false, the engine
    /// runs in playback-only mode and the transmit gate is a
    /// no-op (no frames will ever be encoded).
@@ -1268,6 +1313,29 @@ impl AudioEngine {
    pub fn output_gain(&self) -> f32 {
        f32::from_bits(self.output_gain.load(Ordering::Relaxed))
    }
+
+    /// Set per-client output volume (SRS-075). `1.0` is unity, `0.0`
+    /// mutes. Values above `1.0` amplify and may clip. Clamped to
+    /// `0.0..4.0`.
+    pub fn set_client_volume(&self, client_id: u64, volume: f32) {
+        let clamped = volume.clamp(0.0, 4.0);
+        match self.audio_handler.lock() {
+            Ok(mut h) => {
+                if let Some(q) = h.get_mut_queues().get_mut(&SessionAudioId(client_id)) {
+                    q.volume = clamped;
+                }
+            }
+            Err(e) => {
+                tracing::warn!(
+                    target: "chanora_audio",
+                    client_id,
+                    volume = clamped,
+                    error = %e,
+                    "set_client_volume: audio_handler lock poisoned — volume not applied"
+                );
+            }
+        }
+    }
 }

 impl Drop for AudioEngine {